diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,11803 @@
+{
+  "best_metric": 0.23097696900367737,
+  "best_model_checkpoint": "models/qwen2.5-3b-orpo-mini-fp-no-tools/checkpoint-5000",
+  "epoch": 0.9999360981532366,
+  "eval_steps": 5000,
+  "global_step": 7824,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00012780369352674293,
+      "grad_norm": 17.796366866389164,
+      "learning_rate": 8e-08,
+      "logits/chosen": -1.6875,
+      "logits/rejected": -1.484375,
+      "logps/chosen": -186.0,
+      "logps/rejected": -200.0,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0012780369352674292,
+      "grad_norm": 19.156159317430546,
+      "learning_rate": 8e-07,
+      "logits/chosen": -1.9375,
+      "logits/rejected": -1.4765625,
+      "logps/chosen": -212.0,
+      "logps/rejected": -182.0,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.2638888955116272,
+      "rewards/chosen": 0.005218505859375,
+      "rewards/margins": 0.0024566650390625,
+      "rewards/rejected": 0.0027923583984375,
+      "step": 10
+    },
+    {
+      "epoch": 0.0025560738705348585,
+      "grad_norm": 15.361014523593422,
+      "learning_rate": 1.6e-06,
+      "logits/chosen": -2.015625,
+      "logits/rejected": -1.6640625,
+      "logps/chosen": -224.0,
+      "logps/rejected": -185.0,
+      "loss": 0.6196,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.03369140625,
+      "rewards/margins": 0.1884765625,
+      "rewards/rejected": -0.154296875,
+      "step": 20
+    },
+    {
+      "epoch": 0.0038341108058022877,
+      "grad_norm": 14.538115687223565,
+      "learning_rate": 2.4e-06,
+      "logits/chosen": -2.125,
+      "logits/rejected": -1.8046875,
+      "logps/chosen": -232.0,
+      "logps/rejected": -206.0,
+      "loss": 0.5053,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.447265625,
+      "rewards/margins": 0.85546875,
+      "rewards/rejected": -1.3046875,
+      "step": 30
+    },
+    {
+      "epoch": 0.005112147741069717,
+      "grad_norm": 14.97522508604102,
+      "learning_rate": 3.2e-06,
+      "logits/chosen": -2.0,
+      "logits/rejected": -1.8046875,
+      "logps/chosen": -210.0,
+      "logps/rejected": -199.0,
+      "loss": 0.4729,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.41015625,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -1.8046875,
+      "step": 40
+    },
+    {
+      "epoch": 0.006390184676337146,
+      "grad_norm": 13.171460552236484,
+      "learning_rate": 4e-06,
+      "logits/chosen": -1.9609375,
+      "logits/rejected": -1.640625,
+      "logps/chosen": -230.0,
+      "logps/rejected": -210.0,
+      "loss": 0.4252,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -0.81640625,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -2.609375,
+      "step": 50
+    },
+    {
+      "epoch": 0.007668221611604575,
+      "grad_norm": 10.703538397047184,
+      "learning_rate": 4.8e-06,
+      "logits/chosen": -1.8359375,
+      "logits/rejected": -1.515625,
+      "logps/chosen": -234.0,
+      "logps/rejected": -216.0,
+      "loss": 0.4156,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.625,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -4.28125,
+      "step": 60
+    },
+    {
+      "epoch": 0.008946258546872005,
+      "grad_norm": 13.375098626574099,
+      "learning_rate": 5.6e-06,
+      "logits/chosen": -1.8828125,
+      "logits/rejected": -1.5859375,
+      "logps/chosen": -244.0,
+      "logps/rejected": -232.0,
+      "loss": 0.3928,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.53125,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -4.28125,
+      "step": 70
+    },
+    {
+      "epoch": 0.010224295482139434,
+      "grad_norm": 12.668599957650846,
+      "learning_rate": 6.4e-06,
+      "logits/chosen": -1.890625,
+      "logits/rejected": -1.5390625,
+      "logps/chosen": -248.0,
+      "logps/rejected": -232.0,
+      "loss": 0.3642,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -2.75,
+      "rewards/margins": 1.828125,
+      "rewards/rejected": -4.59375,
+      "step": 80
+    },
+    {
+      "epoch": 0.011502332417406863,
+      "grad_norm": 12.310525233028384,
+      "learning_rate": 7.2e-06,
+      "logits/chosen": -2.0,
+      "logits/rejected": -1.640625,
+      "logps/chosen": -239.0,
+      "logps/rejected": -244.0,
+      "loss": 0.3823,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.40625,
+      "rewards/margins": 2.1875,
+      "rewards/rejected": -5.59375,
+      "step": 90
+    },
+    {
+      "epoch": 0.012780369352674292,
+      "grad_norm": 11.257568570093397,
+      "learning_rate": 8e-06,
+      "logits/chosen": -1.796875,
+      "logits/rejected": -1.375,
+      "logps/chosen": -243.0,
+      "logps/rejected": -239.0,
+      "loss": 0.3532,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -2.921875,
+      "rewards/margins": 2.40625,
+      "rewards/rejected": -5.3125,
+      "step": 100
+    },
+    {
+      "epoch": 0.014058406287941722,
+      "grad_norm": 10.852887372552932,
+      "learning_rate": 7.627700713964738e-06,
+      "logits/chosen": -2.046875,
+      "logits/rejected": -1.6171875,
+      "logps/chosen": -256.0,
+      "logps/rejected": -256.0,
+      "loss": 0.3398,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -4.4375,
+      "rewards/margins": 2.96875,
+      "rewards/rejected": -7.40625,
+      "step": 110
+    },
+    {
+      "epoch": 0.01533644322320915,
+      "grad_norm": 11.604663860150195,
+      "learning_rate": 7.3029674334022146e-06,
+      "logits/chosen": -1.890625,
+      "logits/rejected": -1.46875,
+      "logps/chosen": -253.0,
+      "logps/rejected": -262.0,
+      "loss": 0.319,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -4.75,
+      "rewards/margins": 3.015625,
+      "rewards/rejected": -7.75,
+      "step": 120
+    },
+    {
+      "epoch": 0.01661448015847658,
+      "grad_norm": 10.82907361824836,
+      "learning_rate": 7.016464154456233e-06,
+      "logits/chosen": -1.7578125,
+      "logits/rejected": -1.3515625,
+      "logps/chosen": -248.0,
+      "logps/rejected": -284.0,
+      "loss": 0.3555,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -5.0,
+      "rewards/margins": 2.609375,
+      "rewards/rejected": -7.59375,
+      "step": 130
+    },
+    {
+      "epoch": 0.01789251709374401,
+      "grad_norm": 24.047324041151715,
+      "learning_rate": 6.7612340378281325e-06,
+      "logits/chosen": -1.8671875,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -255.0,
+      "logps/rejected": -254.0,
+      "loss": 0.3331,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -3.78125,
+      "rewards/margins": 2.546875,
+      "rewards/rejected": -6.3125,
+      "step": 140
+    },
+    {
+      "epoch": 0.01917055402901144,
+      "grad_norm": 11.392485733768796,
+      "learning_rate": 6.531972647421809e-06,
+      "logits/chosen": -1.8984375,
+      "logits/rejected": -1.640625,
+      "logps/chosen": -274.0,
+      "logps/rejected": -282.0,
+      "loss": 0.3224,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 3.71875,
+      "rewards/rejected": -10.0,
+      "step": 150
+    },
+    {
+      "epoch": 0.020448590964278868,
+      "grad_norm": 9.926296303181028,
+      "learning_rate": 6.3245553203367575e-06,
+      "logits/chosen": -1.7109375,
+      "logits/rejected": -1.328125,
+      "logps/chosen": -278.0,
+      "logps/rejected": -286.0,
+      "loss": 0.3396,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -5.09375,
+      "rewards/margins": 3.359375,
+      "rewards/rejected": -8.4375,
+      "step": 160
+    },
+    {
+      "epoch": 0.021726627899546297,
+      "grad_norm": 9.725695583265718,
+      "learning_rate": 6.135719910778963e-06,
+      "logits/chosen": -1.7734375,
+      "logits/rejected": -1.28125,
+      "logps/chosen": -280.0,
+      "logps/rejected": -300.0,
+      "loss": 0.2908,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 3.734375,
+      "rewards/rejected": -10.1875,
+      "step": 170
+    },
+    {
+      "epoch": 0.023004664834813726,
+      "grad_norm": 9.382720768394076,
+      "learning_rate": 5.962847939999439e-06,
+      "logits/chosen": -1.7109375,
+      "logits/rejected": -1.2890625,
+      "logps/chosen": -286.0,
+      "logps/rejected": -300.0,
+      "loss": 0.3343,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 3.65625,
+      "rewards/rejected": -10.8125,
+      "step": 180
+    },
+    {
+      "epoch": 0.024282701770081155,
+      "grad_norm": 16.891092124692705,
+      "learning_rate": 5.803810000880094e-06,
+      "logits/chosen": -1.6875,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -290.0,
+      "logps/rejected": -290.0,
+      "loss": 0.2976,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 3.03125,
+      "rewards/rejected": -9.5,
+      "step": 190
+    },
+    {
+      "epoch": 0.025560738705348585,
+      "grad_norm": 9.346122622608238,
+      "learning_rate": 5.65685424949238e-06,
+      "logits/chosen": -1.6328125,
+      "logits/rejected": -1.3046875,
+      "logps/chosen": -260.0,
+      "logps/rejected": -272.0,
+      "loss": 0.274,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 3.28125,
+      "rewards/rejected": -10.6875,
+      "step": 200
+    },
+    {
+      "epoch": 0.026838775640616014,
+      "grad_norm": 12.437482194566748,
+      "learning_rate": 5.5205244747388325e-06,
+      "logits/chosen": -1.765625,
+      "logits/rejected": -1.3515625,
+      "logps/chosen": -298.0,
+      "logps/rejected": -312.0,
+      "loss": 0.3422,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -8.625,
+      "rewards/margins": 3.921875,
+      "rewards/rejected": -12.5625,
+      "step": 210
+    },
+    {
+      "epoch": 0.028116812575883443,
+      "grad_norm": 11.76969573915444,
+      "learning_rate": 5.393598899705936e-06,
+      "logits/chosen": -1.578125,
+      "logits/rejected": -1.2109375,
+      "logps/chosen": -292.0,
+      "logps/rejected": -312.0,
+      "loss": 0.3384,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 3.3125,
+      "rewards/rejected": -10.1875,
+      "step": 220
+    },
+    {
+      "epoch": 0.029394849511150872,
+      "grad_norm": 10.990657347485662,
+      "learning_rate": 5.275043787166296e-06,
+      "logits/chosen": -1.6328125,
+      "logits/rejected": -1.234375,
+      "logps/chosen": -272.0,
+      "logps/rejected": -282.0,
+      "loss": 0.2902,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -5.78125,
+      "rewards/margins": 3.140625,
+      "rewards/rejected": -8.9375,
+      "step": 230
+    },
+    {
+      "epoch": 0.0306728864464183,
+      "grad_norm": 19.102510179342975,
+      "learning_rate": 5.163977794943223e-06,
+      "logits/chosen": -1.7421875,
+      "logits/rejected": -1.375,
+      "logps/chosen": -302.0,
+      "logps/rejected": -304.0,
+      "loss": 0.3615,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 4.3125,
+      "rewards/rejected": -11.625,
+      "step": 240
+    },
+    {
+      "epoch": 0.03195092338168573,
+      "grad_norm": 26.633553117585773,
+      "learning_rate": 5.059644256269407e-06,
+      "logits/chosen": -1.71875,
+      "logits/rejected": -1.3203125,
+      "logps/chosen": -288.0,
+      "logps/rejected": -302.0,
+      "loss": 0.3392,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 3.34375,
+      "rewards/rejected": -11.0,
+      "step": 250
+    },
+    {
+      "epoch": 0.03322896031695316,
+      "grad_norm": 14.094690504033714,
+      "learning_rate": 4.961389383568338e-06,
+      "logits/chosen": -1.6875,
+      "logits/rejected": -1.2421875,
+      "logps/chosen": -274.0,
+      "logps/rejected": -286.0,
+      "loss": 0.2998,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -5.78125,
+      "rewards/margins": 2.78125,
+      "rewards/rejected": -8.5625,
+      "step": 260
+    },
+    {
+      "epoch": 0.03450699725222059,
+      "grad_norm": 11.67667509301692,
+      "learning_rate": 4.8686449556014755e-06,
+      "logits/chosen": -1.765625,
+      "logits/rejected": -1.2265625,
+      "logps/chosen": -274.0,
+      "logps/rejected": -278.0,
+      "loss": 0.2834,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -5.5,
+      "rewards/margins": 3.828125,
+      "rewards/rejected": -9.3125,
+      "step": 270
+    },
+    {
+      "epoch": 0.03578503418748802,
+      "grad_norm": 10.42640331150727,
+      "learning_rate": 4.780914437337574e-06,
+      "logits/chosen": -1.578125,
+      "logits/rejected": -1.140625,
+      "logps/chosen": -270.0,
+      "logps/rejected": -294.0,
+      "loss": 0.2976,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -6.125,
+      "rewards/margins": 4.1875,
+      "rewards/rejected": -10.3125,
+      "step": 280
+    },
+    {
+      "epoch": 0.03706307112275545,
+      "grad_norm": 10.083985572500362,
+      "learning_rate": 4.697761756117627e-06,
+      "logits/chosen": -1.546875,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -282.0,
+      "logps/rejected": -280.0,
+      "loss": 0.3286,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 3.25,
+      "rewards/rejected": -10.1875,
+      "step": 290
+    },
+    {
+      "epoch": 0.03834110805802288,
+      "grad_norm": 7.94956372800409,
+      "learning_rate": 4.618802153517006e-06,
+      "logits/chosen": -1.6171875,
+      "logits/rejected": -1.125,
+      "logps/chosen": -282.0,
+      "logps/rejected": -294.0,
+      "loss": 0.2759,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -5.75,
+      "rewards/margins": 3.765625,
+      "rewards/rejected": -9.5,
+      "step": 300
+    },
+    {
+      "epoch": 0.039619144993290306,
+      "grad_norm": 10.888068658340114,
+      "learning_rate": 4.543694673976518e-06,
+      "logits/chosen": -1.6015625,
+      "logits/rejected": -1.15625,
+      "logps/chosen": -270.0,
+      "logps/rejected": -298.0,
+      "loss": 0.2785,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -6.21875,
+      "rewards/margins": 4.0,
+      "rewards/rejected": -10.1875,
+      "step": 310
+    },
+    {
+      "epoch": 0.040897181928557735,
+      "grad_norm": 12.53461479792773,
+      "learning_rate": 4.472135954999579e-06,
+      "logits/chosen": -1.515625,
+      "logits/rejected": -1.21875,
+      "logps/chosen": -274.0,
+      "logps/rejected": -302.0,
+      "loss": 0.282,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.8125,
+      "rewards/margins": 3.984375,
+      "rewards/rejected": -10.8125,
+      "step": 320
+    },
+    {
+      "epoch": 0.042175218863825165,
+      "grad_norm": 10.391820810011357,
+      "learning_rate": 4.403855060505443e-06,
+      "logits/chosen": -1.6484375,
+      "logits/rejected": -1.2109375,
+      "logps/chosen": -294.0,
+      "logps/rejected": -314.0,
+      "loss": 0.2695,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 4.09375,
+      "rewards/rejected": -11.625,
+      "step": 330
+    },
+    {
+      "epoch": 0.043453255799092594,
+      "grad_norm": 8.066978288276575,
+      "learning_rate": 4.338609156373123e-06,
+      "logits/chosen": -1.6015625,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -298.0,
+      "logps/rejected": -294.0,
+      "loss": 0.2821,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 4.03125,
+      "rewards/rejected": -11.125,
+      "step": 340
+    },
+    {
+      "epoch": 0.04473129273436002,
+      "grad_norm": 9.562512774748992,
+      "learning_rate": 4.27617987059879e-06,
+      "logits/chosen": -1.6953125,
+      "logits/rejected": -1.234375,
+      "logps/chosen": -268.0,
+      "logps/rejected": -306.0,
+      "loss": 0.2863,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 4.21875,
+      "rewards/rejected": -11.3125,
+      "step": 350
+    },
+    {
+      "epoch": 0.04600932966962745,
+      "grad_norm": 12.380924502381642,
+      "learning_rate": 4.216370213557839e-06,
+      "logits/chosen": -1.703125,
+      "logits/rejected": -1.28125,
+      "logps/chosen": -308.0,
+      "logps/rejected": -306.0,
+      "loss": 0.3366,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -8.4375,
+      "rewards/margins": 4.09375,
+      "rewards/rejected": -12.5,
+      "step": 360
+    },
+    {
+      "epoch": 0.04728736660489488,
+      "grad_norm": 11.21079759362238,
+      "learning_rate": 4.15900195928029e-06,
+      "logits/chosen": -1.671875,
+      "logits/rejected": -1.171875,
+      "logps/chosen": -316.0,
+      "logps/rejected": -332.0,
+      "loss": 0.2459,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -9.125,
+      "rewards/margins": 4.78125,
+      "rewards/rejected": -13.9375,
+      "step": 370
+    },
+    {
+      "epoch": 0.04856540354016231,
+      "grad_norm": 11.155157580442184,
+      "learning_rate": 4.103913408340617e-06,
+      "logits/chosen": -1.625,
+      "logits/rejected": -1.1171875,
+      "logps/chosen": -270.0,
+      "logps/rejected": -310.0,
+      "loss": 0.2176,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 4.59375,
+      "rewards/rejected": -12.4375,
+      "step": 380
+    },
+    {
+      "epoch": 0.04984344047542974,
+      "grad_norm": 11.116900640631746,
+      "learning_rate": 4.050957468334666e-06,
+      "logits/chosen": -1.5390625,
+      "logits/rejected": -1.125,
+      "logps/chosen": -326.0,
+      "logps/rejected": -310.0,
+      "loss": 0.2739,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -9.875,
+      "rewards/margins": 4.125,
+      "rewards/rejected": -14.0,
+      "step": 390
+    },
+    {
+      "epoch": 0.05112147741069717,
+      "grad_norm": 9.935426185831105,
+      "learning_rate": 4e-06,
+      "logits/chosen": -1.5703125,
+      "logits/rejected": -1.25,
+      "logps/chosen": -292.0,
+      "logps/rejected": -318.0,
+      "loss": 0.2399,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 3.875,
+      "rewards/rejected": -11.25,
+      "step": 400
+    },
+    {
+      "epoch": 0.0523995143459646,
+      "grad_norm": 9.749315970491567,
+      "learning_rate": 3.950918386598359e-06,
+      "logits/chosen": -1.6171875,
+      "logits/rejected": -1.109375,
+      "logps/chosen": -288.0,
+      "logps/rejected": -318.0,
+      "loss": 0.2549,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 4.625,
+      "rewards/rejected": -12.0,
+      "step": 410
+    },
+    {
+      "epoch": 0.05367755128123203,
+      "grad_norm": 10.586605552743055,
+      "learning_rate": 3.903600291794132e-06,
+      "logits/chosen": -1.5703125,
+      "logits/rejected": -1.15625,
+      "logps/chosen": -298.0,
+      "logps/rejected": -316.0,
+      "loss": 0.2523,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -8.125,
+      "rewards/margins": 4.21875,
+      "rewards/rejected": -12.3125,
+      "step": 420
+    },
+    {
+      "epoch": 0.05495558821649946,
+      "grad_norm": 7.524974222673779,
+      "learning_rate": 3.857942577363297e-06,
+      "logits/chosen": -1.5,
+      "logits/rejected": -1.1796875,
+      "logps/chosen": -306.0,
+      "logps/rejected": -304.0,
+      "loss": 0.2463,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -9.4375,
+      "rewards/margins": 3.734375,
+      "rewards/rejected": -13.1875,
+      "step": 430
+    },
+    {
+      "epoch": 0.056233625151766886,
+      "grad_norm": 9.772151654208214,
+      "learning_rate": 3.813850356982369e-06,
+      "logits/chosen": -1.4453125,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -310.0,
+      "logps/rejected": -340.0,
+      "loss": 0.3097,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -9.3125,
+      "rewards/margins": 4.6875,
+      "rewards/rejected": -14.0625,
+      "step": 440
+    },
+    {
+      "epoch": 0.057511662087034315,
+      "grad_norm": 13.125270379675268,
+      "learning_rate": 3.7712361663282537e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.1015625,
+      "logps/chosen": -310.0,
+      "logps/rejected": -330.0,
+      "loss": 0.2499,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -9.8125,
+      "rewards/margins": 4.4375,
+      "rewards/rejected": -14.25,
+      "step": 450
+    },
+    {
+      "epoch": 0.058789699022301745,
+      "grad_norm": 9.844594606348434,
+      "learning_rate": 3.730019232961255e-06,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -284.0,
+      "logps/rejected": -320.0,
+      "loss": 0.2518,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -9.3125,
+      "rewards/margins": 4.46875,
+      "rewards/rejected": -13.75,
+      "step": 460
+    },
+    {
+      "epoch": 0.060067735957569174,
+      "grad_norm": 8.211376613863207,
+      "learning_rate": 3.6901248321155403e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.0625,
+      "logps/chosen": -300.0,
+      "logps/rejected": -332.0,
+      "loss": 0.2576,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -9.8125,
+      "rewards/margins": 4.4375,
+      "rewards/rejected": -14.25,
+      "step": 470
+    },
+    {
+      "epoch": 0.0613457728928366,
+      "grad_norm": 14.41595118225166,
+      "learning_rate": 3.6514837167011073e-06,
+      "logits/chosen": -1.4921875,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -328.0,
+      "logps/rejected": -342.0,
+      "loss": 0.2803,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -10.0,
+      "rewards/margins": 4.4375,
+      "rewards/rejected": -14.4375,
+      "step": 480
+    },
+    {
+      "epoch": 0.06262380982810403,
+      "grad_norm": 9.07705933106003,
+      "learning_rate": 3.6140316116210052e-06,
+      "logits/chosen": -1.53125,
+      "logits/rejected": -1.0625,
+      "logps/chosen": -302.0,
+      "logps/rejected": -328.0,
+      "loss": 0.2398,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -9.1875,
+      "rewards/margins": 5.125,
+      "rewards/rejected": -14.3125,
+      "step": 490
+    },
+    {
+      "epoch": 0.06390184676337146,
+      "grad_norm": 6.714733416576985,
+      "learning_rate": 3.5777087639996634e-06,
+      "logits/chosen": -1.625,
+      "logits/rejected": -1.0625,
+      "logps/chosen": -292.0,
+      "logps/rejected": -320.0,
+      "loss": 0.2766,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -8.9375,
+      "rewards/margins": 4.1875,
+      "rewards/rejected": -13.125,
+      "step": 500
+    },
+    {
+      "epoch": 0.06517988369863889,
+      "grad_norm": 8.89044855305341,
+      "learning_rate": 3.5424595421603814e-06,
+      "logits/chosen": -1.484375,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -314.0,
+      "logps/rejected": -324.0,
+      "loss": 0.282,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -10.25,
+      "rewards/margins": 4.5,
+      "rewards/rejected": -14.75,
+      "step": 510
+    },
+    {
+      "epoch": 0.06645792063390632,
+      "grad_norm": 10.626320689969987,
+      "learning_rate": 3.5082320772281165e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -292.0,
+      "logps/rejected": -330.0,
+      "loss": 0.2571,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -9.9375,
+      "rewards/margins": 3.84375,
+      "rewards/rejected": -13.75,
+      "step": 520
+    },
+    {
+      "epoch": 0.06773595756917375,
+      "grad_norm": 5.611409167764897,
+      "learning_rate": 3.474977942104555e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -310.0,
+      "logps/rejected": -330.0,
+      "loss": 0.297,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -9.0625,
+      "rewards/margins": 4.4375,
+      "rewards/rejected": -13.5,
+      "step": 530
+    },
+    {
+      "epoch": 0.06901399450444118,
+      "grad_norm": 11.718803397610097,
+      "learning_rate": 3.442651863295481e-06,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -314.0,
+      "logps/rejected": -342.0,
+      "loss": 0.2308,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -9.25,
+      "rewards/margins": 4.78125,
+      "rewards/rejected": -14.0625,
+      "step": 540
+    },
+    {
+      "epoch": 0.07029203143970861,
+      "grad_norm": 9.598321067974835,
+      "learning_rate": 3.4112114616897665e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.984375,
+      "logps/chosen": -318.0,
+      "logps/rejected": -352.0,
+      "loss": 0.2508,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -11.25,
+      "rewards/margins": 4.625,
+      "rewards/rejected": -15.875,
+      "step": 550
+    },
+    {
+      "epoch": 0.07157006837497604,
+      "grad_norm": 8.923293571083503,
+      "learning_rate": 3.3806170189140663e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.125,
+      "logps/chosen": -356.0,
+      "logps/rejected": -370.0,
+      "loss": 0.2677,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -12.9375,
+      "rewards/margins": 4.1875,
+      "rewards/rejected": -17.125,
+      "step": 560
+    },
+    {
+      "epoch": 0.07284810531024347,
+      "grad_norm": 10.09010877916146,
+      "learning_rate": 3.350831266333564e-06,
+      "logits/chosen": -1.515625,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -324.0,
+      "logps/rejected": -346.0,
+      "loss": 0.2545,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -12.25,
+      "rewards/margins": 4.65625,
+      "rewards/rejected": -16.875,
+      "step": 570
+    },
+    {
+      "epoch": 0.0741261422455109,
+      "grad_norm": 8.196459385766977,
+      "learning_rate": 3.3218191941495984e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -1.0625,
+      "logps/chosen": -316.0,
+      "logps/rejected": -328.0,
+      "loss": 0.2478,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -10.1875,
+      "rewards/margins": 4.46875,
+      "rewards/rejected": -14.6875,
+      "step": 580
+    },
+    {
+      "epoch": 0.07540417918077832,
+      "grad_norm": 7.440878025263232,
+      "learning_rate": 3.293547878370473e-06,
+      "logits/chosen": -1.4765625,
+      "logits/rejected": -0.9921875,
+      "logps/chosen": -332.0,
+      "logps/rejected": -354.0,
+      "loss": 0.2884,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -10.6875,
+      "rewards/margins": 4.21875,
+      "rewards/rejected": -14.9375,
+      "step": 590
+    },
+    {
+      "epoch": 0.07668221611604575,
+      "grad_norm": 6.298577168427532,
+      "learning_rate": 3.2659863237109044e-06,
+      "logits/chosen": -1.5546875,
+      "logits/rejected": -1.125,
+      "logps/chosen": -340.0,
+      "logps/rejected": -368.0,
+      "loss": 0.2469,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -11.0,
+      "rewards/margins": 4.6875,
+      "rewards/rejected": -15.6875,
+      "step": 600
+    },
+    {
+      "epoch": 0.07796025305131318,
+      "grad_norm": 9.184394745556292,
+      "learning_rate": 3.239105320715664e-06,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -1.1328125,
+      "logps/chosen": -332.0,
+      "logps/rejected": -368.0,
+      "loss": 0.25,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -11.125,
+      "rewards/margins": 5.09375,
+      "rewards/rejected": -16.25,
+      "step": 610
+    },
+    {
+      "epoch": 0.07923828998658061,
+      "grad_norm": 10.289964962594814,
+      "learning_rate": 3.2128773156099956e-06,
+      "logits/chosen": -1.5078125,
+      "logits/rejected": -1.1796875,
+      "logps/chosen": -330.0,
+      "logps/rejected": -342.0,
+      "loss": 0.2156,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -10.3125,
+      "rewards/margins": 4.75,
+      "rewards/rejected": -15.125,
+      "step": 620
+    },
+    {
+      "epoch": 0.08051632692184804,
+      "grad_norm": 8.917815290356897,
+      "learning_rate": 3.187276291558383e-06,
+      "logits/chosen": -1.46875,
+      "logits/rejected": -1.140625,
+      "logps/chosen": -308.0,
+      "logps/rejected": -322.0,
+      "loss": 0.2506,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -9.8125,
+      "rewards/margins": 4.78125,
+      "rewards/rejected": -14.5625,
+      "step": 630
+    },
+    {
+      "epoch": 0.08179436385711547,
+      "grad_norm": 9.42167521617446,
+      "learning_rate": 3.1622776601683788e-06,
+      "logits/chosen": -1.5703125,
+      "logits/rejected": -1.1640625,
+      "logps/chosen": -326.0,
+      "logps/rejected": -354.0,
+      "loss": 0.2283,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -10.0625,
+      "rewards/margins": 4.71875,
+      "rewards/rejected": -14.8125,
+      "step": 640
+    },
+    {
+      "epoch": 0.0830724007923829,
+      "grad_norm": 7.6782778935093425,
+      "learning_rate": 3.1378581622109444e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -298.0,
+      "logps/rejected": -350.0,
+      "loss": 0.2182,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -9.8125,
+      "rewards/margins": 4.75,
+      "rewards/rejected": -14.5,
+      "step": 650
+    },
+    {
+      "epoch": 0.08435043772765033,
+      "grad_norm": 8.190035952573828,
+      "learning_rate": 3.113995776646092e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -310.0,
+      "logps/rejected": -332.0,
+      "loss": 0.251,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -10.25,
+      "rewards/margins": 5.0,
+      "rewards/rejected": -15.25,
+      "step": 660
+    },
+    {
+      "epoch": 0.08562847466291776,
+      "grad_norm": 8.356158447219443,
+      "learning_rate": 3.090669637145023e-06,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -316.0,
+      "logps/rejected": -350.0,
+      "loss": 0.2763,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -10.4375,
+      "rewards/margins": 4.375,
+      "rewards/rejected": -14.75,
+      "step": 670
+    },
+    {
+      "epoch": 0.08690651159818519,
+      "grad_norm": 9.236679629675967,
+      "learning_rate": 3.0678599553894814e-06,
+      "logits/chosen": -1.5234375,
+      "logits/rejected": -1.0,
+      "logps/chosen": -328.0,
+      "logps/rejected": -352.0,
+      "loss": 0.2411,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -9.4375,
+      "rewards/margins": 5.03125,
+      "rewards/rejected": -14.5,
+      "step": 680
+    },
+    {
+      "epoch": 0.08818454853345262,
+      "grad_norm": 8.779144745799327,
+      "learning_rate": 3.0455479505075235e-06,
+      "logits/chosen": -1.4453125,
+      "logits/rejected": -1.1171875,
+      "logps/chosen": -300.0,
+      "logps/rejected": -326.0,
+      "loss": 0.2273,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -9.0,
+      "rewards/margins": 4.71875,
+      "rewards/rejected": -13.6875,
+      "step": 690
+    },
+    {
+      "epoch": 0.08946258546872005,
+      "grad_norm": 9.797840736629269,
+      "learning_rate": 3.0237157840738173e-06,
+      "logits/chosen": -1.546875,
+      "logits/rejected": -1.1875,
+      "logps/chosen": -302.0,
+      "logps/rejected": -332.0,
+      "loss": 0.2288,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -9.5,
+      "rewards/margins": 4.78125,
+      "rewards/rejected": -14.3125,
+      "step": 700
+    },
+    {
+      "epoch": 0.09074062240398748,
+      "grad_norm": 7.72003280518675,
+      "learning_rate": 3.002346500163206e-06,
+      "logits/chosen": -1.5078125,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -302.0,
+      "logps/rejected": -326.0,
+      "loss": 0.231,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -9.3125,
+      "rewards/margins": 4.40625,
+      "rewards/rejected": -13.75,
+      "step": 710
+    },
+    {
+      "epoch": 0.0920186593392549,
+      "grad_norm": 11.77552708977719,
+      "learning_rate": 2.9814239699997195e-06,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -1.0,
+      "logps/chosen": -318.0,
+      "logps/rejected": -338.0,
+      "loss": 0.2238,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -9.5625,
+      "rewards/margins": 5.0,
+      "rewards/rejected": -14.5625,
+      "step": 720
+    },
+    {
+      "epoch": 0.09329669627452233,
+      "grad_norm": 11.737044260771562,
+      "learning_rate": 2.9609328407904207e-06,
+      "logits/chosen": -1.5078125,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -308.0,
+      "logps/rejected": -328.0,
+      "loss": 0.215,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -10.625,
+      "rewards/margins": 5.625,
+      "rewards/rejected": -16.25,
+      "step": 730
+    },
+    {
+      "epoch": 0.09457473320978976,
+      "grad_norm": 9.737406822663896,
+      "learning_rate": 2.940858488375231e-06,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -1.1015625,
+      "logps/chosen": -342.0,
+      "logps/rejected": -358.0,
+      "loss": 0.2648,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -10.9375,
+      "rewards/margins": 5.15625,
+      "rewards/rejected": -16.125,
+      "step": 740
+    },
+    {
+      "epoch": 0.09585277014505719,
+      "grad_norm": 7.6918767583684415,
+      "learning_rate": 2.9211869733608857e-06,
+      "logits/chosen": -1.46875,
+      "logits/rejected": -1.1015625,
+      "logps/chosen": -304.0,
+      "logps/rejected": -330.0,
+      "loss": 0.2191,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -10.5,
+      "rewards/margins": 4.78125,
+      "rewards/rejected": -15.3125,
+      "step": 750
+    },
+    {
+      "epoch": 0.09713080708032462,
+      "grad_norm": 8.363246802190268,
+      "learning_rate": 2.901905000440047e-06,
+      "logits/chosen": -1.4765625,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -322.0,
+      "logps/rejected": -360.0,
+      "loss": 0.2206,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -10.0,
+      "rewards/margins": 4.5625,
+      "rewards/rejected": -14.5625,
+      "step": 760
+    },
+    {
+      "epoch": 0.09840884401559205,
+      "grad_norm": 5.743825399712502,
+      "learning_rate": 2.8829998806257885e-06,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -314.0,
+      "logps/rejected": -348.0,
+      "loss": 0.2258,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -10.125,
+      "rewards/margins": 4.6875,
+      "rewards/rejected": -14.8125,
+      "step": 770
+    },
+    {
+      "epoch": 0.09968688095085948,
+      "grad_norm": 9.066149383603953,
+      "learning_rate": 2.8644594961577314e-06,
+      "logits/chosen": -1.53125,
+      "logits/rejected": -1.140625,
+      "logps/chosen": -324.0,
+      "logps/rejected": -358.0,
+      "loss": 0.2248,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -10.5625,
+      "rewards/margins": 5.75,
+      "rewards/rejected": -16.375,
+      "step": 780
+    },
+    {
+      "epoch": 0.10096491788612691,
+      "grad_norm": 5.575414464465586,
+      "learning_rate": 2.84627226785928e-06,
+      "logits/chosen": -1.5,
+      "logits/rejected": -1.1953125,
+      "logps/chosen": -340.0,
+      "logps/rejected": -358.0,
+      "loss": 0.2363,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -12.3125,
+      "rewards/margins": 4.53125,
+      "rewards/rejected": -16.875,
+      "step": 790
+    },
+    {
+      "epoch": 0.10224295482139434,
+      "grad_norm": 9.256168758201992,
+      "learning_rate": 2.82842712474619e-06,
+      "logits/chosen": -1.5625,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -334.0,
+      "logps/rejected": -370.0,
+      "loss": 0.2433,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -12.3125,
+      "rewards/margins": 5.1875,
+      "rewards/rejected": -17.5,
+      "step": 800
+    },
+    {
+      "epoch": 0.10352099175666177,
+      "grad_norm": 8.78929103759398,
+      "learning_rate": 2.810913475705226e-06,
+      "logits/chosen": -1.5234375,
+      "logits/rejected": -1.171875,
+      "logps/chosen": -332.0,
+      "logps/rejected": -356.0,
+      "loss": 0.2018,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -12.1875,
+      "rewards/margins": 5.46875,
+      "rewards/rejected": -17.625,
+      "step": 810
+    },
+    {
+      "epoch": 0.1047990286919292,
+      "grad_norm": 7.667534314581795,
+      "learning_rate": 2.7937211830783128e-06,
+      "logits/chosen": -1.46875,
+      "logits/rejected": -1.1796875,
+      "logps/chosen": -334.0,
+      "logps/rejected": -424.0,
+      "loss": 0.233,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -12.8125,
+      "rewards/margins": 10.0,
+      "rewards/rejected": -22.75,
+      "step": 820
+    },
+    {
+      "epoch": 0.10607706562719663,
+      "grad_norm": 8.969814192089954,
+      "learning_rate": 2.776840538002493e-06,
+      "logits/chosen": -1.5390625,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -352.0,
+      "logps/rejected": -380.0,
+      "loss": 0.2609,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -13.9375,
+      "rewards/margins": 5.8125,
+      "rewards/rejected": -19.75,
+      "step": 830
+    },
+    {
+      "epoch": 0.10735510256246406,
+      "grad_norm": 11.19149837054736,
+      "learning_rate": 2.7602622373694163e-06,
+      "logits/chosen": -1.5625,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -348.0,
+      "logps/rejected": -384.0,
+      "loss": 0.2232,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -12.5,
+      "rewards/margins": 5.4375,
+      "rewards/rejected": -17.875,
+      "step": 840
+    },
+    {
+      "epoch": 0.10863313949773148,
+      "grad_norm": 8.246361536077268,
+      "learning_rate": 2.743977362280141e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.0625,
+      "logps/chosen": -340.0,
+      "logps/rejected": -396.0,
+      "loss": 0.247,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -12.1875,
+      "rewards/margins": 5.40625,
+      "rewards/rejected": -17.625,
+      "step": 850
+    },
+    {
+      "epoch": 0.10991117643299891,
+      "grad_norm": 13.423952379573198,
+      "learning_rate": 2.7279773578818937e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -332.0,
+      "logps/rejected": -378.0,
+      "loss": 0.2054,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -12.3125,
+      "rewards/margins": 5.46875,
+      "rewards/rejected": -17.75,
+      "step": 860
+    },
+    {
+      "epoch": 0.11118921336826634,
+      "grad_norm": 6.688734485973752,
+      "learning_rate": 2.7122540144832417e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -344.0,
+      "logps/rejected": -374.0,
+      "loss": 0.2349,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -14.0,
+      "rewards/margins": 5.03125,
+      "rewards/rejected": -19.0,
+      "step": 870
+    },
+    {
+      "epoch": 0.11246725030353377,
+      "grad_norm": 7.125313747170276,
+      "learning_rate": 2.696799449852968e-06,
+      "logits/chosen": -1.484375,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -348.0,
+      "logps/rejected": -392.0,
+      "loss": 0.227,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -13.25,
+      "rewards/margins": 5.15625,
+      "rewards/rejected": -18.375,
+      "step": 880
+    },
+    {
+      "epoch": 0.1137452872388012,
+      "grad_norm": 8.572837055808646,
+      "learning_rate": 2.6816060926159636e-06,
+      "logits/chosen": -1.46875,
+      "logits/rejected": -0.98046875,
+      "logps/chosen": -380.0,
+      "logps/rejected": -404.0,
+      "loss": 0.2591,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -13.4375,
+      "rewards/margins": 5.5625,
+      "rewards/rejected": -19.0,
+      "step": 890
+    },
+    {
+      "epoch": 0.11502332417406863,
+      "grad_norm": 6.712110400131555,
+      "learning_rate": 2.6666666666666664e-06,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.0,
+      "logps/chosen": -338.0,
+      "logps/rejected": -380.0,
+      "loss": 0.266,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -12.875,
+      "rewards/margins": 5.40625,
+      "rewards/rejected": -18.25,
+      "step": 900
+    },
+    {
+      "epoch": 0.11630136110933606,
+      "grad_norm": 7.859706724590066,
+      "learning_rate": 2.6519741765271837e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.96484375,
+      "logps/chosen": -350.0,
+      "logps/rejected": -376.0,
+      "loss": 0.2127,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -12.1875,
+      "rewards/margins": 5.0,
+      "rewards/rejected": -17.25,
+      "step": 910
+    },
+    {
+      "epoch": 0.11757939804460349,
+      "grad_norm": 9.145529170423432,
+      "learning_rate": 2.637521893583148e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.87890625,
+      "logps/chosen": -330.0,
+      "logps/rejected": -348.0,
+      "loss": 0.2639,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -12.375,
+      "rewards/margins": 5.59375,
+      "rewards/rejected": -18.0,
+      "step": 920
+    },
+    {
+      "epoch": 0.11885743497987092,
+      "grad_norm": 6.9555394920170786,
+      "learning_rate": 2.6233033431358115e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -342.0,
+      "logps/rejected": -384.0,
+      "loss": 0.22,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -13.0625,
+      "rewards/margins": 4.875,
+      "rewards/rejected": -17.875,
+      "step": 930
+    },
+    {
+      "epoch": 0.12013547191513835,
+      "grad_norm": 7.915892716124389,
+      "learning_rate": 2.6093122922137685e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -330.0,
+      "logps/rejected": -366.0,
+      "loss": 0.2068,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -11.625,
+      "rewards/margins": 5.3125,
+      "rewards/rejected": -16.875,
+      "step": 940
+    },
+    {
+      "epoch": 0.12141350885040578,
+      "grad_norm": 10.34790820394252,
+      "learning_rate": 2.5955427380922006e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -338.0,
+      "logps/rejected": -394.0,
+      "loss": 0.2115,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -12.3125,
+      "rewards/margins": 5.8125,
+      "rewards/rejected": -18.125,
+      "step": 950
+    },
+    {
+      "epoch": 0.1226915457856732,
+      "grad_norm": 8.610634131157397,
+      "learning_rate": 2.5819888974716113e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -338.0,
+      "logps/rejected": -370.0,
+      "loss": 0.2749,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -13.6875,
+      "rewards/margins": 5.125,
+      "rewards/rejected": -18.75,
+      "step": 960
+    },
+    {
+      "epoch": 0.12396958272094064,
+      "grad_norm": 7.176106071984723,
+      "learning_rate": 2.5686451962717425e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -338.0,
+      "logps/rejected": -380.0,
+      "loss": 0.193,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -13.0,
+      "rewards/margins": 5.46875,
+      "rewards/rejected": -18.5,
+      "step": 970
+    },
+    {
+      "epoch": 0.12524761965620806,
+      "grad_norm": 11.180244146036813,
+      "learning_rate": 2.5555062599997596e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -342.0,
+      "logps/rejected": -364.0,
+      "loss": 0.2131,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -11.75,
+      "rewards/margins": 5.53125,
+      "rewards/rejected": -17.25,
+      "step": 980
+    },
+    {
+      "epoch": 0.1265256565914755,
+      "grad_norm": 7.3922386008639,
+      "learning_rate": 2.5425669046549126e-06,
+      "logits/chosen": -1.171875,
+      "logits/rejected": -0.79296875,
+      "logps/chosen": -298.0,
+      "logps/rejected": -344.0,
+      "loss": 0.2065,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -10.875,
+      "rewards/margins": 5.21875,
+      "rewards/rejected": -16.125,
+      "step": 990
+    },
+    {
+      "epoch": 0.12780369352674292,
+      "grad_norm": 14.46357223595503,
+      "learning_rate": 2.5298221281347034e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -326.0,
+      "logps/rejected": -358.0,
+      "loss": 0.2083,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -11.125,
+      "rewards/margins": 5.03125,
+      "rewards/rejected": -16.125,
+      "step": 1000
+    },
+    {
+      "epoch": 0.12908173046201035,
+      "grad_norm": 10.499118194489526,
+      "learning_rate": 2.5172671021102103e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -326.0,
+      "logps/rejected": -342.0,
+      "loss": 0.2179,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -11.4375,
+      "rewards/margins": 5.40625,
+      "rewards/rejected": -16.875,
+      "step": 1010
+    },
+    {
+      "epoch": 0.13035976739727778,
+      "grad_norm": 8.741860267158804,
+      "learning_rate": 2.504897164340598e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.95703125,
+      "logps/chosen": -308.0,
+      "logps/rejected": -350.0,
+      "loss": 0.2451,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -11.0,
+      "rewards/margins": 5.1875,
+      "rewards/rejected": -16.25,
+      "step": 1020
+    },
+    {
+      "epoch": 0.1316378043325452,
+      "grad_norm": 9.139436819787727,
+      "learning_rate": 2.492707811399023e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -320.0,
+      "logps/rejected": -358.0,
+      "loss": 0.2151,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -11.875,
+      "rewards/margins": 4.8125,
+      "rewards/rejected": -16.625,
+      "step": 1030
+    },
+    {
+      "epoch": 0.13291584126781264,
+      "grad_norm": 7.9889620240709,
+      "learning_rate": 2.480694691784169e-06,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -328.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1921,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -12.75,
+      "rewards/margins": 5.1875,
+      "rewards/rejected": -18.0,
+      "step": 1040
+    },
+    {
+      "epoch": 0.13419387820308007,
+      "grad_norm": 8.615879823382464,
+      "learning_rate": 2.4688535993934706e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -364.0,
+      "logps/rejected": -378.0,
+      "loss": 0.2219,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -12.8125,
+      "rewards/margins": 5.4375,
+      "rewards/rejected": -18.25,
+      "step": 1050
+    },
+    {
+      "epoch": 0.1354719151383475,
+      "grad_norm": 9.543267757261967,
+      "learning_rate": 2.457180467335805e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.98046875,
+      "logps/chosen": -348.0,
+      "logps/rejected": -378.0,
+      "loss": 0.1964,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -12.625,
+      "rewards/margins": 6.1875,
+      "rewards/rejected": -18.875,
+      "step": 1060
+    },
+    {
+      "epoch": 0.13674995207361493,
+      "grad_norm": 5.1220908372735625,
+      "learning_rate": 2.4456713620629725e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -358.0,
+      "logps/rejected": -378.0,
+      "loss": 0.1875,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -14.0625,
+      "rewards/margins": 4.84375,
+      "rewards/rejected": -18.875,
+      "step": 1070
+    },
+    {
+      "epoch": 0.13802798900888236,
+      "grad_norm": 12.54028050101492,
+      "learning_rate": 2.4343224778007378e-06,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -324.0,
+      "logps/rejected": -376.0,
+      "loss": 0.1964,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -12.5625,
+      "rewards/margins": 5.8125,
+      "rewards/rejected": -18.375,
+      "step": 1080
+    },
+    {
+      "epoch": 0.13930602594414979,
+      "grad_norm": 6.797708534112584,
+      "learning_rate": 2.4231301312615306e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.125,
+      "logps/chosen": -330.0,
+      "logps/rejected": -364.0,
+      "loss": 0.1974,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -11.375,
+      "rewards/margins": 5.90625,
+      "rewards/rejected": -17.25,
+      "step": 1090
+    },
+    {
+      "epoch": 0.14058406287941722,
+      "grad_norm": 10.05882899004973,
+      "learning_rate": 2.412090756622109e-06,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -342.0,
+      "logps/rejected": -370.0,
+      "loss": 0.186,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -12.625,
+      "rewards/margins": 5.75,
+      "rewards/rejected": -18.375,
+      "step": 1100
+    },
+    {
+      "epoch": 0.14186209981468464,
+      "grad_norm": 9.925083177538808,
+      "learning_rate": 2.401200900750657e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -328.0,
+      "logps/rejected": -366.0,
+      "loss": 0.2227,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -13.1875,
+      "rewards/margins": 6.125,
+      "rewards/rejected": -19.25,
+      "step": 1110
+    },
+    {
+      "epoch": 0.14314013674995207,
+      "grad_norm": 8.290643706405318,
+      "learning_rate": 2.390457218668787e-06,
+      "logits/chosen": -1.1640625,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -358.0,
+      "logps/rejected": -376.0,
+      "loss": 0.2143,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -14.5,
+      "rewards/margins": 4.625,
+      "rewards/rejected": -19.125,
+      "step": 1120
+    },
+    {
+      "epoch": 0.1444181736852195,
+      "grad_norm": 7.629768489214631,
+      "learning_rate": 2.379856469234918e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.76171875,
+      "logps/chosen": -374.0,
+      "logps/rejected": -382.0,
+      "loss": 0.2238,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -15.125,
+      "rewards/margins": 5.0625,
+      "rewards/rejected": -20.125,
+      "step": 1130
+    },
+    {
+      "epoch": 0.14569621062048693,
+      "grad_norm": 7.890750923112455,
+      "learning_rate": 2.369395511036369e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -358.0,
+      "logps/rejected": -382.0,
+      "loss": 0.2271,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -14.3125,
+      "rewards/margins": 5.25,
+      "rewards/rejected": -19.5,
+      "step": 1140
+    },
+    {
+      "epoch": 0.14697424755575436,
+      "grad_norm": 8.16419808489084,
+      "learning_rate": 2.359071298478354e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.97265625,
+      "logps/chosen": -354.0,
+      "logps/rejected": -384.0,
+      "loss": 0.2091,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -13.5625,
+      "rewards/margins": 5.3125,
+      "rewards/rejected": -18.875,
+      "step": 1150
+    },
+    {
+      "epoch": 0.1482522844910218,
+      "grad_norm": 13.290309491409749,
+      "learning_rate": 2.3488808780588137e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.87890625,
+      "logps/chosen": -340.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1964,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -12.875,
+      "rewards/margins": 5.6875,
+      "rewards/rejected": -18.625,
+      "step": 1160
+    },
+    {
+      "epoch": 0.14953032142628922,
+      "grad_norm": 5.760311649402745,
+      "learning_rate": 2.3388213848187446e-06,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -326.0,
+      "logps/rejected": -362.0,
+      "loss": 0.1683,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -12.4375,
+      "rewards/margins": 5.375,
+      "rewards/rejected": -17.875,
+      "step": 1170
+    },
+    {
+      "epoch": 0.15080835836155665,
+      "grad_norm": 8.979012978878,
+      "learning_rate": 2.328890038958328e-06,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -340.0,
+      "logps/rejected": -376.0,
+      "loss": 0.1939,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -12.4375,
+      "rewards/margins": 5.84375,
+      "rewards/rejected": -18.25,
+      "step": 1180
+    },
+    {
+      "epoch": 0.15208639529682408,
+      "grad_norm": 8.33811700877816,
+      "learning_rate": 2.3190841426097937e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.21875,
+      "logps/chosen": -332.0,
+      "logps/rejected": -378.0,
+      "loss": 0.2095,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -13.4375,
+      "rewards/margins": 5.53125,
+      "rewards/rejected": -19.0,
+      "step": 1190
+    },
+    {
+      "epoch": 0.1533644322320915,
+      "grad_norm": 9.813716252877187,
+      "learning_rate": 2.309401076758503e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.875,
+      "logps/chosen": -344.0,
+      "logps/rejected": -362.0,
+      "loss": 0.2324,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -12.3125,
+      "rewards/margins": 4.9375,
+      "rewards/rejected": -17.25,
+      "step": 1200
+    },
+    {
+      "epoch": 0.15464246916735894,
+      "grad_norm": 9.345516040121323,
+      "learning_rate": 2.299838298304276e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.9921875,
+      "logps/chosen": -318.0,
+      "logps/rejected": -366.0,
+      "loss": 0.1888,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -11.375,
+      "rewards/margins": 5.5625,
+      "rewards/rejected": -17.0,
+      "step": 1210
+    },
+    {
+      "epoch": 0.15592050610262637,
+      "grad_norm": 7.615968191407808,
+      "learning_rate": 2.2903933372554728e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -328.0,
+      "logps/rejected": -378.0,
+      "loss": 0.2138,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -12.75,
+      "rewards/margins": 5.0,
+      "rewards/rejected": -17.75,
+      "step": 1220
+    },
+    {
+      "epoch": 0.1571985430378938,
+      "grad_norm": 7.438310396591626,
+      "learning_rate": 2.281063794048804e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.98046875,
+      "logps/chosen": -354.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1963,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -13.8125,
+      "rewards/margins": 5.0,
+      "rewards/rejected": -18.875,
+      "step": 1230
+    },
+    {
+      "epoch": 0.15847657997316122,
+      "grad_norm": 7.455544929175229,
+      "learning_rate": 2.271847336988259e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -350.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1773,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -13.875,
+      "rewards/margins": 6.0625,
+      "rewards/rejected": -19.875,
+      "step": 1240
+    },
+    {
+      "epoch": 0.15975461690842865,
+      "grad_norm": 9.576403981081084,
+      "learning_rate": 2.262741699796952e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -346.0,
+      "logps/rejected": -386.0,
+      "loss": 0.1973,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -12.5625,
+      "rewards/margins": 6.53125,
+      "rewards/rejected": -19.125,
+      "step": 1250
+    },
+    {
+      "epoch": 0.16103265384369608,
+      "grad_norm": 8.400242035542771,
+      "learning_rate": 2.253744679276044e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.96484375,
+      "logps/chosen": -334.0,
+      "logps/rejected": -370.0,
+      "loss": 0.2471,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -12.4375,
+      "rewards/margins": 5.5625,
+      "rewards/rejected": -18.0,
+      "step": 1260
+    },
+    {
+      "epoch": 0.1623106907789635,
+      "grad_norm": 8.482485461834482,
+      "learning_rate": 2.244854133065255e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -358.0,
+      "logps/rejected": -384.0,
+      "loss": 0.21,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -12.75,
+      "rewards/margins": 4.78125,
+      "rewards/rejected": -17.5,
+      "step": 1270
+    },
+    {
+      "epoch": 0.16358872771423094,
+      "grad_norm": 8.519423706921335,
+      "learning_rate": 2.2360679774997895e-06,
+      "logits/chosen": -1.1171875,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -348.0,
+      "logps/rejected": -364.0,
+      "loss": 0.2091,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -13.75,
+      "rewards/margins": 4.71875,
+      "rewards/rejected": -18.5,
+      "step": 1280
+    },
+    {
+      "epoch": 0.16486676464949837,
+      "grad_norm": 5.681746699183822,
+      "learning_rate": 2.2273841855588183e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -346.0,
+      "logps/rejected": -372.0,
+      "loss": 0.2002,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -13.0,
+      "rewards/margins": 5.59375,
+      "rewards/rejected": -18.625,
+      "step": 1290
+    },
+    {
+      "epoch": 0.1661448015847658,
+      "grad_norm": 8.98952098687221,
+      "learning_rate": 2.2188007849009167e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.84765625,
+      "logps/chosen": -336.0,
+      "logps/rejected": -380.0,
+      "loss": 0.2003,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -12.3125,
+      "rewards/margins": 5.8125,
+      "rewards/rejected": -18.125,
+      "step": 1300
+    },
+    {
+      "epoch": 0.16742283852003323,
+      "grad_norm": 9.488126975341013,
+      "learning_rate": 2.2103158559821502e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.98046875,
+      "logps/chosen": -344.0,
+      "logps/rejected": -378.0,
+      "loss": 0.1822,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -12.6875,
+      "rewards/margins": 5.4375,
+      "rewards/rejected": -18.125,
+      "step": 1310
+    },
+    {
+      "epoch": 0.16870087545530066,
+      "grad_norm": 8.78053013879103,
+      "learning_rate": 2.2019275302527213e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -326.0,
+      "logps/rejected": -370.0,
+      "loss": 0.1937,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -13.25,
+      "rewards/margins": 5.0625,
+      "rewards/rejected": -18.375,
+      "step": 1320
+    },
+    {
+      "epoch": 0.1699789123905681,
+      "grad_norm": 7.100764582269635,
+      "learning_rate": 2.193633988428327e-06,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -0.98046875,
+      "logps/chosen": -340.0,
+      "logps/rejected": -378.0,
+      "loss": 0.2116,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -13.25,
+      "rewards/margins": 6.9375,
+      "rewards/rejected": -20.125,
+      "step": 1330
+    },
+    {
+      "epoch": 0.17125694932583552,
+      "grad_norm": 7.801226467351775,
+      "learning_rate": 2.185433458832612e-06,
+      "logits/chosen": -1.5078125,
+      "logits/rejected": -1.0859375,
+      "logps/chosen": -350.0,
+      "logps/rejected": -386.0,
+      "loss": 0.181,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -12.9375,
+      "rewards/margins": 5.75,
+      "rewards/rejected": -18.625,
+      "step": 1340
+    },
+    {
+      "epoch": 0.17253498626110295,
+      "grad_norm": 6.5068106613396015,
+      "learning_rate": 2.177324215807269e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -360.0,
+      "logps/rejected": -380.0,
+      "loss": 0.2216,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -13.125,
+      "rewards/margins": 4.75,
+      "rewards/rejected": -17.875,
+      "step": 1350
+    },
+    {
+      "epoch": 0.17381302319637038,
+      "grad_norm": 8.524526129373655,
+      "learning_rate": 2.1693045781865616e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -366.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1969,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -12.625,
+      "rewards/margins": 6.625,
+      "rewards/rejected": -19.25,
+      "step": 1360
+    },
+    {
+      "epoch": 0.1750910601316378,
+      "grad_norm": 6.431180199385379,
+      "learning_rate": 2.1613729078331965e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.828125,
+      "logps/chosen": -334.0,
+      "logps/rejected": -366.0,
+      "loss": 0.1825,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -13.125,
+      "rewards/margins": 5.1875,
+      "rewards/rejected": -18.25,
+      "step": 1370
+    },
+    {
+      "epoch": 0.17636909706690523,
+      "grad_norm": 5.438643555919341,
+      "learning_rate": 2.1535276082326617e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -336.0,
+      "logps/rejected": -370.0,
+      "loss": 0.1955,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -12.625,
+      "rewards/margins": 5.3125,
+      "rewards/rejected": -17.875,
+      "step": 1380
+    },
+    {
+      "epoch": 0.17764713400217266,
+      "grad_norm": 8.369475522462334,
+      "learning_rate": 2.14576712314328e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.77734375,
+      "logps/chosen": -336.0,
+      "logps/rejected": -368.0,
+      "loss": 0.1862,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -12.625,
+      "rewards/margins": 5.59375,
+      "rewards/rejected": -18.25,
+      "step": 1390
+    },
+    {
+      "epoch": 0.1789251709374401,
+      "grad_norm": 11.334497995986077,
+      "learning_rate": 2.138089935299395e-06,
+      "logits/chosen": -1.4609375,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -366.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1923,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -13.1875,
+      "rewards/margins": 6.5,
+      "rewards/rejected": -19.625,
+      "step": 1400
+    },
+    {
+      "epoch": 0.18020320787270752,
+      "grad_norm": 7.761688419386625,
+      "learning_rate": 2.1304945651652297e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -360.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1683,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -13.8125,
+      "rewards/margins": 6.21875,
+      "rewards/rejected": -20.0,
+      "step": 1410
+    },
+    {
+      "epoch": 0.18148124480797495,
+      "grad_norm": 6.874611759250077,
+      "learning_rate": 2.122979569737101e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -360.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1791,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.5,
+      "rewards/margins": 6.21875,
+      "rewards/rejected": -20.75,
+      "step": 1420
+    },
+    {
+      "epoch": 0.18275928174324238,
+      "grad_norm": 6.5753719742073775,
+      "learning_rate": 2.11554354139178e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.9296875,
+      "logps/chosen": -344.0,
+      "logps/rejected": -390.0,
+      "loss": 0.2192,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 6.3125,
+      "rewards/rejected": -20.75,
+      "step": 1430
+    },
+    {
+      "epoch": 0.1840373186785098,
+      "grad_norm": 9.634987061446262,
+      "learning_rate": 2.1081851067789196e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.0625,
+      "logps/chosen": -378.0,
+      "logps/rejected": -418.0,
+      "loss": 0.2083,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -15.25,
+      "rewards/margins": 6.21875,
+      "rewards/rejected": -21.5,
+      "step": 1440
+    },
+    {
+      "epoch": 0.18531535561377724,
+      "grad_norm": 9.581571009188673,
+      "learning_rate": 2.1009029257555606e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -350.0,
+      "logps/rejected": -390.0,
+      "loss": 0.2171,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -13.5,
+      "rewards/margins": 5.1875,
+      "rewards/rejected": -18.625,
+      "step": 1450
+    },
+    {
+      "epoch": 0.18659339254904467,
+      "grad_norm": 9.166977960938443,
+      "learning_rate": 2.0936956903608545e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -332.0,
+      "logps/rejected": -362.0,
+      "loss": 0.2153,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -12.625,
+      "rewards/margins": 5.40625,
+      "rewards/rejected": -18.0,
+      "step": 1460
+    },
+    {
+      "epoch": 0.1878714294843121,
+      "grad_norm": 12.547408328770173,
+      "learning_rate": 2.0865621238292046e-06,
+      "logits/chosen": -1.4609375,
+      "logits/rejected": -1.09375,
+      "logps/chosen": -360.0,
+      "logps/rejected": -390.0,
+      "loss": 0.227,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -12.6875,
+      "rewards/margins": 5.625,
+      "rewards/rejected": -18.25,
+      "step": 1470
+    },
+    {
+      "epoch": 0.18914946641957953,
+      "grad_norm": 9.813482149979079,
+      "learning_rate": 2.079500979640145e-06,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -360.0,
+      "logps/rejected": -388.0,
+      "loss": 0.1905,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -13.375,
+      "rewards/margins": 5.5,
+      "rewards/rejected": -18.875,
+      "step": 1480
+    },
+    {
+      "epoch": 0.19042750335484696,
+      "grad_norm": 8.077327714247682,
+      "learning_rate": 2.072511040603359e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -352.0,
+      "logps/rejected": -378.0,
+      "loss": 0.2043,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -12.8125,
+      "rewards/margins": 5.875,
+      "rewards/rejected": -18.75,
+      "step": 1490
+    },
+    {
+      "epoch": 0.19170554029011438,
+      "grad_norm": 7.689497269305014,
+      "learning_rate": 2.065591117977289e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -336.0,
+      "logps/rejected": -368.0,
+      "loss": 0.19,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -12.625,
+      "rewards/margins": 6.0625,
+      "rewards/rejected": -18.75,
+      "step": 1500
+    },
+    {
+      "epoch": 0.1929835772253818,
+      "grad_norm": 9.017445928823717,
+      "learning_rate": 2.058740050619915e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.77734375,
+      "logps/chosen": -340.0,
+      "logps/rejected": -354.0,
+      "loss": 0.1738,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -12.6875,
+      "rewards/margins": 5.8125,
+      "rewards/rejected": -18.5,
+      "step": 1510
+    },
+    {
+      "epoch": 0.19426161416064924,
+      "grad_norm": 5.638158665371567,
+      "learning_rate": 2.0519567041703083e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -350.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1842,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -13.25,
+      "rewards/margins": 5.625,
+      "rewards/rejected": -18.875,
+      "step": 1520
+    },
+    {
+      "epoch": 0.19553965109591667,
+      "grad_norm": 9.055842696660081,
+      "learning_rate": 2.0452399702596544e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -348.0,
+      "logps/rejected": -386.0,
+      "loss": 0.1892,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -14.3125,
+      "rewards/margins": 5.1875,
+      "rewards/rejected": -19.5,
+      "step": 1530
+    },
+    {
+      "epoch": 0.1968176880311841,
+      "grad_norm": 7.060434319873597,
+      "learning_rate": 2.0385887657505017e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -370.0,
+      "logps/rejected": -392.0,
+      "loss": 0.1988,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -14.25,
+      "rewards/margins": 5.75,
+      "rewards/rejected": -20.0,
+      "step": 1540
+    },
+    {
+      "epoch": 0.19809572496645153,
+      "grad_norm": 8.49361200807995,
+      "learning_rate": 2.032002032003048e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -370.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1573,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.1875,
+      "rewards/margins": 5.8125,
+      "rewards/rejected": -20.0,
+      "step": 1550
+    },
+    {
+      "epoch": 0.19937376190171896,
+      "grad_norm": 7.65231755533092,
+      "learning_rate": 2.025478734167333e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.82421875,
+      "logps/chosen": -358.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1968,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -13.6875,
+      "rewards/margins": 6.40625,
+      "rewards/rejected": -20.125,
+      "step": 1560
+    },
+    {
+      "epoch": 0.2006517988369864,
+      "grad_norm": 8.893296741061858,
+      "learning_rate": 2.0190178605002747e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -346.0,
+      "logps/rejected": -388.0,
+      "loss": 0.23,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -13.5,
+      "rewards/margins": 6.03125,
+      "rewards/rejected": -19.5,
+      "step": 1570
+    },
+    {
+      "epoch": 0.20192983577225382,
+      "grad_norm": 8.546061439992066,
+      "learning_rate": 2.0126184217065104e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.75390625,
+      "logps/chosen": -350.0,
+      "logps/rejected": -378.0,
+      "loss": 0.2178,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 5.03125,
+      "rewards/rejected": -20.0,
+      "step": 1580
+    },
+    {
+      "epoch": 0.20320787270752125,
+      "grad_norm": 9.299351997571943,
+      "learning_rate": 2.0062794503020765e-06,
+      "logits/chosen": -1.1328125,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -364.0,
+      "logps/rejected": -388.0,
+      "loss": 0.1993,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 5.0,
+      "rewards/rejected": -19.5,
+      "step": 1590
+    },
+    {
+      "epoch": 0.20448590964278868,
+      "grad_norm": 5.8965653265489655,
+      "learning_rate": 2e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -356.0,
+      "logps/rejected": -394.0,
+      "loss": 0.173,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.125,
+      "rewards/margins": 5.84375,
+      "rewards/rejected": -21.0,
+      "step": 1600
+    },
+    {
+      "epoch": 0.2057639465780561,
+      "grad_norm": 13.98091982604496,
+      "learning_rate": 1.993779145116907e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.796875,
+      "logps/chosen": -358.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1815,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 6.5,
+      "rewards/rejected": -21.625,
+      "step": 1610
+    },
+    {
+      "epoch": 0.20704198351332354,
+      "grad_norm": 8.207943759274128,
+      "learning_rate": 1.987615979999813e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.79296875,
+      "logps/chosen": -370.0,
+      "logps/rejected": -448.0,
+      "loss": 0.206,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -15.75,
+      "rewards/margins": 6.21875,
+      "rewards/rejected": -21.875,
+      "step": 1620
+    },
+    {
+      "epoch": 0.20832002044859096,
+      "grad_norm": 8.980127423940925,
+      "learning_rate": 1.9815096184722797e-06,
+      "logits/chosen": -1.140625,
+      "logits/rejected": -0.78125,
+      "logps/chosen": -378.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1889,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 5.25,
+      "rewards/rejected": -21.625,
+      "step": 1630
+    },
+    {
+      "epoch": 0.2095980573838584,
+      "grad_norm": 4.98986224171766,
+      "learning_rate": 1.9754591932991793e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.82421875,
+      "logps/chosen": -364.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1581,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -14.3125,
+      "rewards/margins": 6.3125,
+      "rewards/rejected": -20.625,
+      "step": 1640
+    },
+    {
+      "epoch": 0.21087609431912582,
+      "grad_norm": 9.805599747899638,
+      "learning_rate": 1.9694638556693235e-06,
+      "logits/chosen": -1.1953125,
+      "logits/rejected": -0.82421875,
+      "logps/chosen": -356.0,
+      "logps/rejected": -390.0,
+      "loss": 0.1659,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -14.5625,
+      "rewards/margins": 6.9375,
+      "rewards/rejected": -21.5,
+      "step": 1650
+    },
+    {
+      "epoch": 0.21215413125439325,
+      "grad_norm": 7.913621471404465,
+      "learning_rate": 1.963522774695264e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -370.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1889,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -14.8125,
+      "rewards/margins": 6.09375,
+      "rewards/rejected": -20.875,
+      "step": 1660
+    },
+    {
+      "epoch": 0.21343216818966068,
+      "grad_norm": 7.642444479167847,
+      "learning_rate": 1.9576351369295853e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -380.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1683,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 5.4375,
+      "rewards/rejected": -20.375,
+      "step": 1670
+    },
+    {
+      "epoch": 0.2147102051249281,
+      "grad_norm": 9.208357956843475,
+      "learning_rate": 1.951800145897066e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.796875,
+      "logps/chosen": -340.0,
+      "logps/rejected": -360.0,
+      "loss": 0.2029,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -14.5625,
+      "rewards/margins": 5.15625,
+      "rewards/rejected": -19.75,
+      "step": 1680
+    },
+    {
+      "epoch": 0.21598824206019554,
+      "grad_norm": 6.28782280328706,
+      "learning_rate": 1.9460170216420797e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -368.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1773,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -14.5625,
+      "rewards/margins": 5.3125,
+      "rewards/rejected": -19.875,
+      "step": 1690
+    },
+    {
+      "epoch": 0.21726627899546297,
+      "grad_norm": 9.824672538792008,
+      "learning_rate": 1.9402850002906637e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.84765625,
+      "logps/chosen": -354.0,
+      "logps/rejected": -386.0,
+      "loss": 0.195,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 5.6875,
+      "rewards/rejected": -20.125,
+      "step": 1700
+    },
+    {
+      "epoch": 0.2185443159307304,
+      "grad_norm": 3.2075694144226934,
+      "learning_rate": 1.9346033336266974e-06,
+      "logits/chosen": -1.15625,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -360.0,
+      "logps/rejected": -392.0,
+      "loss": 0.1722,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.3125,
+      "rewards/margins": 5.21875,
+      "rewards/rejected": -19.625,
+      "step": 1710
+    },
+    {
+      "epoch": 0.21982235286599783,
+      "grad_norm": 6.817474257539379,
+      "learning_rate": 1.9289712886816486e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.8359375,
+      "logps/chosen": -348.0,
+      "logps/rejected": -388.0,
+      "loss": 0.1695,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -13.375,
+      "rewards/margins": 6.1875,
+      "rewards/rejected": -19.625,
+      "step": 1720
+    },
+    {
+      "epoch": 0.22110038980126526,
+      "grad_norm": 8.514437581470213,
+      "learning_rate": 1.92338814733738e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.859375,
+      "logps/chosen": -336.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1799,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -13.0625,
+      "rewards/margins": 6.3125,
+      "rewards/rejected": -19.375,
+      "step": 1730
+    },
+    {
+      "epoch": 0.22237842673653269,
+      "grad_norm": 11.337796113909468,
+      "learning_rate": 1.9178532059415367e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -362.0,
+      "logps/rejected": -400.0,
+      "loss": 0.1706,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -14.125,
+      "rewards/margins": 6.1875,
+      "rewards/rejected": -20.25,
+      "step": 1740
+    },
+    {
+      "epoch": 0.22365646367180012,
+      "grad_norm": 9.162881205585903,
+      "learning_rate": 1.9123657749350298e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -346.0,
+      "logps/rejected": -370.0,
+      "loss": 0.1973,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -13.625,
+      "rewards/margins": 5.90625,
+      "rewards/rejected": -19.5,
+      "step": 1750
+    },
+    {
+      "epoch": 0.22493450060706754,
+      "grad_norm": 8.313992060216734,
+      "learning_rate": 1.9069251784911844e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -374.0,
+      "logps/rejected": -394.0,
+      "loss": 0.1986,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -14.8125,
+      "rewards/margins": 5.21875,
+      "rewards/rejected": -20.125,
+      "step": 1760
+    },
+    {
+      "epoch": 0.22621253754233497,
+      "grad_norm": 6.7074957554433485,
+      "learning_rate": 1.9015307541661132e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -378.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1645,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -15.0,
+      "rewards/margins": 5.4375,
+      "rewards/rejected": -20.5,
+      "step": 1770
+    },
+    {
+      "epoch": 0.2274905744776024,
+      "grad_norm": 5.791857088794324,
+      "learning_rate": 1.8961818525599089e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -358.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1892,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -15.0,
+      "rewards/margins": 5.9375,
+      "rewards/rejected": -20.875,
+      "step": 1780
+    },
+    {
+      "epoch": 0.22876861141286983,
+      "grad_norm": 7.416476699785206,
+      "learning_rate": 1.890877836988262e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -348.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1681,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -14.25,
+      "rewards/margins": 7.125,
+      "rewards/rejected": -21.375,
+      "step": 1790
+    },
+    {
+      "epoch": 0.23004664834813726,
+      "grad_norm": 7.204444366824245,
+      "learning_rate": 1.8856180831641269e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -368.0,
+      "logps/rejected": -388.0,
+      "loss": 0.1616,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -13.875,
+      "rewards/margins": 5.65625,
+      "rewards/rejected": -19.5,
+      "step": 1800
+    },
+    {
+      "epoch": 0.2313246852834047,
+      "grad_norm": 4.548408274146384,
+      "learning_rate": 1.8804019788890737e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.828125,
+      "logps/chosen": -354.0,
+      "logps/rejected": -382.0,
+      "loss": 0.1728,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -13.8125,
+      "rewards/margins": 5.9375,
+      "rewards/rejected": -19.75,
+      "step": 1810
+    },
+    {
+      "epoch": 0.23260272221867212,
+      "grad_norm": 6.917109498479077,
+      "learning_rate": 1.8752289237539816e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -348.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1523,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.375,
+      "rewards/margins": 6.125,
+      "rewards/rejected": -20.5,
+      "step": 1820
+    },
+    {
+      "epoch": 0.23388075915393955,
+      "grad_norm": 11.612402569318638,
+      "learning_rate": 1.8700983288487376e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.8046875,
+      "logps/chosen": -364.0,
+      "logps/rejected": -418.0,
+      "loss": 0.205,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.125,
+      "rewards/margins": 6.40625,
+      "rewards/rejected": -21.5,
+      "step": 1830
+    },
+    {
+      "epoch": 0.23515879608920698,
+      "grad_norm": 9.517954583923483,
+      "learning_rate": 1.8650096164806275e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.78515625,
+      "logps/chosen": -356.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1774,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -14.5,
+      "rewards/margins": 6.59375,
+      "rewards/rejected": -21.0,
+      "step": 1840
+    },
+    {
+      "epoch": 0.2364368330244744,
+      "grad_norm": 7.327531904031042,
+      "learning_rate": 1.8599622199011084e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.765625,
+      "logps/chosen": -368.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1763,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -14.875,
+      "rewards/margins": 6.125,
+      "rewards/rejected": -21.0,
+      "step": 1850
+    },
+    {
+      "epoch": 0.23771486995974184,
+      "grad_norm": 9.118167589956133,
+      "learning_rate": 1.854955583040673e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -378.0,
+      "logps/rejected": -388.0,
+      "loss": 0.1884,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -14.875,
+      "rewards/margins": 5.53125,
+      "rewards/rejected": -20.375,
+      "step": 1860
+    },
+    {
+      "epoch": 0.23899290689500927,
+      "grad_norm": 9.046606314866981,
+      "learning_rate": 1.849989160251521e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.7578125,
+      "logps/chosen": -358.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1725,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -14.625,
+      "rewards/margins": 5.6875,
+      "rewards/rejected": -20.375,
+      "step": 1870
+    },
+    {
+      "epoch": 0.2402709438302767,
+      "grad_norm": 5.0268115120828485,
+      "learning_rate": 1.8450624160577701e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -372.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1705,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.125,
+      "rewards/margins": 6.9375,
+      "rewards/rejected": -22.125,
+      "step": 1880
+    },
+    {
+      "epoch": 0.24154898076554412,
+      "grad_norm": 6.278148896233312,
+      "learning_rate": 1.8401748249129445e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.69140625,
+      "logps/chosen": -348.0,
+      "logps/rejected": -382.0,
+      "loss": 0.1702,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -14.1875,
+      "rewards/margins": 6.8125,
+      "rewards/rejected": -21.0,
+      "step": 1890
+    },
+    {
+      "epoch": 0.24282701770081155,
+      "grad_norm": 9.419133956656545,
+      "learning_rate": 1.8353258709644938e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -352.0,
+      "logps/rejected": -378.0,
+      "loss": 0.2116,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.3125,
+      "rewards/margins": 6.15625,
+      "rewards/rejected": -20.5,
+      "step": 1900
+    },
+    {
+      "epoch": 0.24410505463607898,
+      "grad_norm": 8.00030950363539,
+      "learning_rate": 1.830515047825102e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.7109375,
+      "logps/chosen": -366.0,
+      "logps/rejected": -398.0,
+      "loss": 0.2012,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.5,
+      "rewards/margins": 5.71875,
+      "rewards/rejected": -20.25,
+      "step": 1910
+    },
+    {
+      "epoch": 0.2453830915713464,
+      "grad_norm": 8.967256104881239,
+      "learning_rate": 1.8257418583505536e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.74609375,
+      "logps/chosen": -374.0,
+      "logps/rejected": -386.0,
+      "loss": 0.1978,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -14.1875,
+      "rewards/margins": 5.0,
+      "rewards/rejected": -19.125,
+      "step": 1920
+    },
+    {
+      "epoch": 0.24666112850661384,
+      "grad_norm": 9.789439997590655,
+      "learning_rate": 1.8210058144239416e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.80078125,
+      "logps/chosen": -368.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1787,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -14.75,
+      "rewards/margins": 5.15625,
+      "rewards/rejected": -19.875,
+      "step": 1930
+    },
+    {
+      "epoch": 0.24793916544188127,
+      "grad_norm": 5.675470563883412,
+      "learning_rate": 1.816306436745999e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -354.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1983,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -14.5,
+      "rewards/margins": 5.6875,
+      "rewards/rejected": -20.125,
+      "step": 1940
+    },
+    {
+      "epoch": 0.2492172023771487,
+      "grad_norm": 5.588145319622337,
+      "learning_rate": 1.8116432546313529e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -356.0,
+      "logps/rejected": -400.0,
+      "loss": 0.1738,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -14.0,
+      "rewards/margins": 5.96875,
+      "rewards/rejected": -20.0,
+      "step": 1950
+    },
+    {
+      "epoch": 0.25049523931241613,
+      "grad_norm": 5.813751766410226,
+      "learning_rate": 1.8070158058105026e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -360.0,
+      "logps/rejected": -382.0,
+      "loss": 0.1974,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -15.25,
+      "rewards/margins": 4.84375,
+      "rewards/rejected": -20.125,
+      "step": 1960
+    },
+    {
+      "epoch": 0.25177327624768353,
+      "grad_norm": 7.300217456102993,
+      "learning_rate": 1.8024236362373315e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.765625,
+      "logps/chosen": -356.0,
+      "logps/rejected": -374.0,
+      "loss": 0.1789,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -14.8125,
+      "rewards/margins": 5.3125,
+      "rewards/rejected": -20.125,
+      "step": 1970
+    },
+    {
+      "epoch": 0.253051313182951,
+      "grad_norm": 6.836916573663873,
+      "learning_rate": 1.7978662999019787e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -352.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1679,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.3125,
+      "rewards/margins": 5.8125,
+      "rewards/rejected": -21.125,
+      "step": 1980
+    },
+    {
+      "epoch": 0.2543293501182184,
+      "grad_norm": 11.42053182790777,
+      "learning_rate": 1.793343358648881e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.82421875,
+      "logps/chosen": -360.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1921,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -15.3125,
+      "rewards/margins": 5.84375,
+      "rewards/rejected": -21.125,
+      "step": 1990
+    },
+    {
+      "epoch": 0.25560738705348585,
+      "grad_norm": 6.653131675731774,
+      "learning_rate": 1.7888543819998317e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.77734375,
+      "logps/chosen": -364.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1741,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -15.125,
+      "rewards/margins": 6.34375,
+      "rewards/rejected": -21.5,
+      "step": 2000
+    },
+    {
+      "epoch": 0.25688542398875325,
+      "grad_norm": 9.767131495442785,
+      "learning_rate": 1.7843989469818819e-06,
+      "logits/chosen": -1.1953125,
+      "logits/rejected": -0.875,
+      "logps/chosen": -364.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1871,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.125,
+      "rewards/margins": 6.625,
+      "rewards/rejected": -20.75,
+      "step": 2010
+    },
+    {
+      "epoch": 0.2581634609240207,
+      "grad_norm": 7.986438654340379,
+      "learning_rate": 1.779976637959939e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -356.0,
+      "logps/rejected": -390.0,
+      "loss": 0.1537,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -13.6875,
+      "rewards/margins": 5.65625,
+      "rewards/rejected": -19.375,
+      "step": 2020
+    },
+    {
+      "epoch": 0.2594414978592881,
+      "grad_norm": 8.010249765654981,
+      "learning_rate": 1.7755870464739012e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.80859375,
+      "logps/chosen": -342.0,
+      "logps/rejected": -394.0,
+      "loss": 0.1841,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -13.875,
+      "rewards/margins": 6.40625,
+      "rewards/rejected": -20.25,
+      "step": 2030
+    },
+    {
+      "epoch": 0.26071953479455556,
+      "grad_norm": 7.2891750745622215,
+      "learning_rate": 1.7712297710801907e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.75,
+      "logps/chosen": -348.0,
+      "logps/rejected": -392.0,
+      "loss": 0.1972,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 5.5,
+      "rewards/rejected": -20.0,
+      "step": 2040
+    },
+    {
+      "epoch": 0.26199757172982296,
+      "grad_norm": 7.718473023541538,
+      "learning_rate": 1.7669044171975444e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -380.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1868,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -15.0625,
+      "rewards/margins": 5.5,
+      "rewards/rejected": -20.625,
+      "step": 2050
+    },
+    {
+      "epoch": 0.2632756086650904,
+      "grad_norm": 6.029258331965988,
+      "learning_rate": 1.7626105969569268e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -374.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1941,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 5.96875,
+      "rewards/rejected": -20.375,
+      "step": 2060
+    },
+    {
+      "epoch": 0.2645536456003578,
+      "grad_norm": 5.763126494010597,
+      "learning_rate": 1.758347929055432e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -346.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1802,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -13.25,
+      "rewards/margins": 6.125,
+      "rewards/rejected": -19.375,
+      "step": 2070
+    },
+    {
+      "epoch": 0.2658316825356253,
+      "grad_norm": 5.329824202776029,
+      "learning_rate": 1.7541160386140582e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.796875,
+      "logps/chosen": -366.0,
+      "logps/rejected": -368.0,
+      "loss": 0.1869,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -13.875,
+      "rewards/margins": 5.46875,
+      "rewards/rejected": -19.375,
+      "step": 2080
+    },
+    {
+      "epoch": 0.2671097194708927,
+      "grad_norm": 8.917341335647293,
+      "learning_rate": 1.7499145570392284e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.79296875,
+      "logps/chosen": -354.0,
+      "logps/rejected": -378.0,
+      "loss": 0.1735,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.125,
+      "rewards/margins": 5.9375,
+      "rewards/rejected": -20.0,
+      "step": 2090
+    },
+    {
+      "epoch": 0.26838775640616014,
+      "grad_norm": 6.7262816484589365,
+      "learning_rate": 1.745743121887939e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.75,
+      "logps/chosen": -352.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1659,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -13.6875,
+      "rewards/margins": 6.1875,
+      "rewards/rejected": -19.875,
+      "step": 2100
+    },
+    {
+      "epoch": 0.26966579334142754,
+      "grad_norm": 7.701328393803493,
+      "learning_rate": 1.7416013767364324e-06,
+      "logits/chosen": -1.109375,
+      "logits/rejected": -0.7734375,
+      "logps/chosen": -364.0,
+      "logps/rejected": -394.0,
+      "loss": 0.1753,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -14.625,
+      "rewards/margins": 6.375,
+      "rewards/rejected": -21.0,
+      "step": 2110
+    },
+    {
+      "epoch": 0.270943830276695,
+      "grad_norm": 8.824184144487734,
+      "learning_rate": 1.7374889710522776e-06,
+      "logits/chosen": -1.0859375,
+      "logits/rejected": -0.60546875,
+      "logps/chosen": -348.0,
+      "logps/rejected": -416.0,
+      "loss": 0.2096,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.0,
+      "rewards/margins": 7.0625,
+      "rewards/rejected": -21.125,
+      "step": 2120
+    },
+    {
+      "epoch": 0.2722218672119624,
+      "grad_norm": 6.119144594016658,
+      "learning_rate": 1.7334055600697579e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.79296875,
+      "logps/chosen": -384.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1643,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -15.8125,
+      "rewards/margins": 6.03125,
+      "rewards/rejected": -21.875,
+      "step": 2130
+    },
+    {
+      "epoch": 0.27349990414722986,
+      "grad_norm": 6.791220888158369,
+      "learning_rate": 1.7293508046684678e-06,
+      "logits/chosen": -1.125,
+      "logits/rejected": -0.6640625,
+      "logps/chosen": -356.0,
+      "logps/rejected": -406.0,
+      "loss": 0.147,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 5.59375,
+      "rewards/rejected": -21.625,
+      "step": 2140
+    },
+    {
+      "epoch": 0.27477794108249726,
+      "grad_norm": 8.819458020439479,
+      "learning_rate": 1.7253243712550145e-06,
+      "logits/chosen": -1.15625,
+      "logits/rejected": -0.75,
+      "logps/chosen": -372.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1722,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.75,
+      "rewards/margins": 6.625,
+      "rewards/rejected": -22.375,
+      "step": 2150
+    },
+    {
+      "epoch": 0.2760559780177647,
+      "grad_norm": 12.513659646562013,
+      "learning_rate": 1.7213259316477406e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.82421875,
+      "logps/chosen": -378.0,
+      "logps/rejected": -432.0,
+      "loss": 0.166,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.6875,
+      "rewards/margins": 6.6875,
+      "rewards/rejected": -22.375,
+      "step": 2160
+    },
+    {
+      "epoch": 0.2773340149530321,
+      "grad_norm": 8.150110629562679,
+      "learning_rate": 1.7173551629643674e-06,
+      "logits/chosen": -1.140625,
+      "logits/rejected": -0.7265625,
+      "logps/chosen": -360.0,
+      "logps/rejected": -390.0,
+      "loss": 0.1853,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.375,
+      "rewards/margins": 5.59375,
+      "rewards/rejected": -21.0,
+      "step": 2170
+    },
+    {
+      "epoch": 0.27861205188829957,
+      "grad_norm": 8.168768217134746,
+      "learning_rate": 1.713411747512477e-06,
+      "logits/chosen": -1.109375,
+      "logits/rejected": -0.7265625,
+      "logps/chosen": -378.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1839,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.5625,
+      "rewards/margins": 6.0,
+      "rewards/rejected": -21.625,
+      "step": 2180
+    },
+    {
+      "epoch": 0.279890088823567,
+      "grad_norm": 8.657316394343697,
+      "learning_rate": 1.709495372682753e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.82421875,
+      "logps/chosen": -376.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1746,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 6.0625,
+      "rewards/rejected": -22.125,
+      "step": 2190
+    },
+    {
+      "epoch": 0.28116812575883443,
+      "grad_norm": 7.275908451907876,
+      "learning_rate": 1.7056057308448832e-06,
+      "logits/chosen": -1.1953125,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -358.0,
+      "logps/rejected": -396.0,
+      "loss": 0.1455,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -14.1875,
+      "rewards/margins": 6.1875,
+      "rewards/rejected": -20.375,
+      "step": 2200
+    },
+    {
+      "epoch": 0.28244616269410183,
+      "grad_norm": 3.515561929130765,
+      "learning_rate": 1.701742519246068e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -372.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1331,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.125,
+      "rewards/margins": 7.0625,
+      "rewards/rejected": -22.25,
+      "step": 2210
+    },
+    {
+      "epoch": 0.2837241996293693,
+      "grad_norm": 5.110742215437251,
+      "learning_rate": 1.6979054399120355e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -370.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1632,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -15.4375,
+      "rewards/margins": 6.8125,
+      "rewards/rejected": -22.25,
+      "step": 2220
+    },
+    {
+      "epoch": 0.2850022365646367,
+      "grad_norm": 7.535991418610705,
+      "learning_rate": 1.6940941995505069e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.828125,
+      "logps/chosen": -376.0,
+      "logps/rejected": -406.0,
+      "loss": 0.175,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.9375,
+      "rewards/margins": 6.0,
+      "rewards/rejected": -22.0,
+      "step": 2230
+    },
+    {
+      "epoch": 0.28628027349990415,
+      "grad_norm": 6.70419983810628,
+      "learning_rate": 1.6903085094570331e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -378.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1826,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -15.9375,
+      "rewards/margins": 5.9375,
+      "rewards/rejected": -21.875,
+      "step": 2240
+    },
+    {
+      "epoch": 0.28755831043517155,
+      "grad_norm": 9.41676767860322,
+      "learning_rate": 1.6865480854231356e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.7734375,
+      "logps/chosen": -366.0,
+      "logps/rejected": -400.0,
+      "loss": 0.1544,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.875,
+      "rewards/margins": 6.375,
+      "rewards/rejected": -21.25,
+      "step": 2250
+    },
+    {
+      "epoch": 0.288836347370439,
+      "grad_norm": 6.683883025870184,
+      "learning_rate": 1.682812647646685e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -348.0,
+      "logps/rejected": -392.0,
+      "loss": 0.1455,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.75,
+      "rewards/margins": 6.09375,
+      "rewards/rejected": -20.875,
+      "step": 2260
+    },
+    {
+      "epoch": 0.2901143843057064,
+      "grad_norm": 6.827239453821361,
+      "learning_rate": 1.6791019206444541e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -368.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1474,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 6.375,
+      "rewards/rejected": -20.875,
+      "step": 2270
+    },
+    {
+      "epoch": 0.29139242124097386,
+      "grad_norm": 6.278738918572633,
+      "learning_rate": 1.675415633166782e-06,
+      "logits/chosen": -1.171875,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -358.0,
+      "logps/rejected": -396.0,
+      "loss": 0.172,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -14.5625,
+      "rewards/margins": 6.5625,
+      "rewards/rejected": -21.125,
+      "step": 2280
+    },
+    {
+      "epoch": 0.29267045817624127,
+      "grad_norm": 8.445675698994542,
+      "learning_rate": 1.6717535181142914e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -378.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1785,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -14.5,
+      "rewards/margins": 6.5625,
+      "rewards/rejected": -21.0,
+      "step": 2290
+    },
+    {
+      "epoch": 0.2939484951115087,
+      "grad_norm": 5.857718153352363,
+      "learning_rate": 1.668115312456598e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -372.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1313,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -14.875,
+      "rewards/margins": 6.96875,
+      "rewards/rejected": -21.875,
+      "step": 2300
+    },
+    {
+      "epoch": 0.2952265320467761,
+      "grad_norm": 9.352933666769587,
+      "learning_rate": 1.6645007571529578e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.80859375,
+      "logps/chosen": -348.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1711,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -14.75,
+      "rewards/margins": 6.3125,
+      "rewards/rejected": -21.125,
+      "step": 2310
+    },
+    {
+      "epoch": 0.2965045689820436,
+      "grad_norm": 7.965618240636307,
+      "learning_rate": 1.6609095970747992e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.82421875,
+      "logps/chosen": -370.0,
+      "logps/rejected": -392.0,
+      "loss": 0.2154,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 6.09375,
+      "rewards/rejected": -21.0,
+      "step": 2320
+    },
+    {
+      "epoch": 0.297782605917311,
+      "grad_norm": 4.954672834782572,
+      "learning_rate": 1.6573415809300833e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.7265625,
+      "logps/chosen": -346.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1525,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.3125,
+      "rewards/margins": 5.8125,
+      "rewards/rejected": -20.125,
+      "step": 2330
+    },
+    {
+      "epoch": 0.29906064285257844,
+      "grad_norm": 5.993685664678074,
+      "learning_rate": 1.6537964611894462e-06,
+      "logits/chosen": -1.15625,
+      "logits/rejected": -0.76953125,
+      "logps/chosen": -356.0,
+      "logps/rejected": -382.0,
+      "loss": 0.1585,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -13.625,
+      "rewards/margins": 5.625,
+      "rewards/rejected": -19.25,
+      "step": 2340
+    },
+    {
+      "epoch": 0.30033867978784584,
+      "grad_norm": 6.300315998453001,
+      "learning_rate": 1.6502739940140692e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -370.0,
+      "logps/rejected": -412.0,
+      "loss": 0.173,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.375,
+      "rewards/margins": 6.5625,
+      "rewards/rejected": -21.0,
+      "step": 2350
+    },
+    {
+      "epoch": 0.3016167167231133,
+      "grad_norm": 9.548190293133839,
+      "learning_rate": 1.6467739391852364e-06,
+      "logits/chosen": -1.1484375,
+      "logits/rejected": -0.76953125,
+      "logps/chosen": -362.0,
+      "logps/rejected": -396.0,
+      "loss": 0.2018,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 6.46875,
+      "rewards/rejected": -20.875,
+      "step": 2360
+    },
+    {
+      "epoch": 0.3028947536583807,
+      "grad_norm": 6.346384840023915,
+      "learning_rate": 1.6432960600355221e-06,
+      "logits/chosen": -1.109375,
+      "logits/rejected": -0.7265625,
+      "logps/chosen": -366.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1867,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.0625,
+      "rewards/margins": 6.1875,
+      "rewards/rejected": -20.25,
+      "step": 2370
+    },
+    {
+      "epoch": 0.30417279059364816,
+      "grad_norm": 6.633303401925544,
+      "learning_rate": 1.6398401233815756e-06,
+      "logits/chosen": -1.109375,
+      "logits/rejected": -0.7734375,
+      "logps/chosen": -352.0,
+      "logps/rejected": -378.0,
+      "loss": 0.1826,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.1875,
+      "rewards/margins": 6.21875,
+      "rewards/rejected": -20.375,
+      "step": 2380
+    },
+    {
+      "epoch": 0.30545082752891556,
+      "grad_norm": 5.69334109659055,
+      "learning_rate": 1.6364058994584524e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -360.0,
+      "logps/rejected": -392.0,
+      "loss": 0.1837,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -14.625,
+      "rewards/margins": 6.1875,
+      "rewards/rejected": -20.875,
+      "step": 2390
+    },
+    {
+      "epoch": 0.306728864464183,
+      "grad_norm": 5.375234477773909,
+      "learning_rate": 1.6329931618554522e-06,
+      "logits/chosen": -1.171875,
+      "logits/rejected": -0.78125,
+      "logps/chosen": -344.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1541,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -14.25,
+      "rewards/margins": 6.125,
+      "rewards/rejected": -20.375,
+      "step": 2400
+    },
+    {
+      "epoch": 0.3080069013994504,
+      "grad_norm": 5.312140258608359,
+      "learning_rate": 1.6296016874534209e-06,
+      "logits/chosen": -1.1171875,
+      "logits/rejected": -0.78125,
+      "logps/chosen": -372.0,
+      "logps/rejected": -374.0,
+      "loss": 0.1744,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -14.0,
+      "rewards/margins": 6.15625,
+      "rewards/rejected": -20.125,
+      "step": 2410
+    },
+    {
+      "epoch": 0.3092849383347179,
+      "grad_norm": 8.054045343175655,
+      "learning_rate": 1.6262312563634835e-06,
+      "logits/chosen": -1.1328125,
+      "logits/rejected": -0.8046875,
+      "logps/chosen": -334.0,
+      "logps/rejected": -376.0,
+      "loss": 0.1634,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -12.3125,
+      "rewards/margins": 7.03125,
+      "rewards/rejected": -19.375,
+      "step": 2420
+    },
+    {
+      "epoch": 0.3105629752699853,
+      "grad_norm": 7.5134988122239665,
+      "learning_rate": 1.6228816518671587e-06,
+      "logits/chosen": -1.140625,
+      "logits/rejected": -0.71484375,
+      "logps/chosen": -350.0,
+      "logps/rejected": -376.0,
+      "loss": 0.1415,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -13.3125,
+      "rewards/margins": 6.09375,
+      "rewards/rejected": -19.375,
+      "step": 2430
+    },
+    {
+      "epoch": 0.31184101220525273,
+      "grad_norm": 8.143621721450158,
+      "learning_rate": 1.619552660357832e-06,
+      "logits/chosen": -1.140625,
+      "logits/rejected": -0.8359375,
+      "logps/chosen": -370.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1775,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -14.125,
+      "rewards/margins": 5.75,
+      "rewards/rejected": -19.875,
+      "step": 2440
+    },
+    {
+      "epoch": 0.31311904914052013,
+      "grad_norm": 7.9099347771838895,
+      "learning_rate": 1.616244071283537e-06,
+      "logits/chosen": -1.09375,
+      "logits/rejected": -0.76171875,
+      "logps/chosen": -342.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1741,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -14.3125,
+      "rewards/margins": 5.59375,
+      "rewards/rejected": -19.875,
+      "step": 2450
+    },
+    {
+      "epoch": 0.3143970860757876,
+      "grad_norm": 4.9707781112467835,
+      "learning_rate": 1.6129556770910235e-06,
+      "logits/chosen": -1.171875,
+      "logits/rejected": -0.75,
+      "logps/chosen": -338.0,
+      "logps/rejected": -384.0,
+      "loss": 0.1468,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.5625,
+      "rewards/margins": 6.09375,
+      "rewards/rejected": -20.625,
+      "step": 2460
+    },
+    {
+      "epoch": 0.315675123011055,
+      "grad_norm": 16.12049587644324,
+      "learning_rate": 1.6096872731710673e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -382.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1693,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.625,
+      "rewards/margins": 7.03125,
+      "rewards/rejected": -22.625,
+      "step": 2470
+    },
+    {
+      "epoch": 0.31695315994632245,
+      "grad_norm": 4.43995252331447,
+      "learning_rate": 1.6064386578049978e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -378.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1722,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -14.75,
+      "rewards/margins": 6.40625,
+      "rewards/rejected": -21.125,
+      "step": 2480
+    },
+    {
+      "epoch": 0.31823119688158985,
+      "grad_norm": 9.732189050626204,
+      "learning_rate": 1.6032096321124046e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -352.0,
+      "logps/rejected": -378.0,
+      "loss": 0.1396,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -13.5,
+      "rewards/margins": 6.4375,
+      "rewards/rejected": -19.875,
+      "step": 2490
+    },
+    {
+      "epoch": 0.3195092338168573,
+      "grad_norm": 7.374499922487743,
+      "learning_rate": 1.6e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -368.0,
+      "logps/rejected": -418.0,
+      "loss": 0.212,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -14.375,
+      "rewards/margins": 6.5,
+      "rewards/rejected": -20.875,
+      "step": 2500
+    },
+    {
+      "epoch": 0.3207872707521247,
+      "grad_norm": 7.4183580840459395,
+      "learning_rate": 1.5968095681115984e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -360.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1645,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -14.6875,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -21.5,
+      "step": 2510
+    },
+    {
+      "epoch": 0.32206530768739217,
+      "grad_norm": 6.8737421826244365,
+      "learning_rate": 1.5936381457791914e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.9296875,
+      "logps/chosen": -364.0,
+      "logps/rejected": -380.0,
+      "loss": 0.1555,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -13.625,
+      "rewards/margins": 6.4375,
+      "rewards/rejected": -20.125,
+      "step": 2520
+    },
+    {
+      "epoch": 0.32334334462265957,
+      "grad_norm": 5.66510197104367,
+      "learning_rate": 1.590485544975088e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.99609375,
+      "logps/chosen": -348.0,
+      "logps/rejected": -386.0,
+      "loss": 0.1621,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -13.4375,
+      "rewards/margins": 6.28125,
+      "rewards/rejected": -19.75,
+      "step": 2530
+    },
+    {
+      "epoch": 0.324621381557927,
+      "grad_norm": 7.4181333828293745,
+      "learning_rate": 1.5873515802650901e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -352.0,
+      "logps/rejected": -376.0,
+      "loss": 0.1586,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -13.0,
+      "rewards/margins": 6.34375,
+      "rewards/rejected": -19.375,
+      "step": 2540
+    },
+    {
+      "epoch": 0.3258994184931944,
+      "grad_norm": 7.431962604591743,
+      "learning_rate": 1.584236068762679e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -358.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1415,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -14.6875,
+      "rewards/margins": 6.53125,
+      "rewards/rejected": -21.25,
+      "step": 2550
+    },
+    {
+      "epoch": 0.3271774554284619,
+      "grad_norm": 7.1603258631602005,
+      "learning_rate": 1.5811388300841894e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -394.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1676,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.3125,
+      "rewards/margins": 6.625,
+      "rewards/rejected": -21.875,
+      "step": 2560
+    },
+    {
+      "epoch": 0.3284554923637293,
+      "grad_norm": 5.449377730320968,
+      "learning_rate": 1.5780596863049431e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -362.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1409,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 6.25,
+      "rewards/rejected": -21.5,
+      "step": 2570
+    },
+    {
+      "epoch": 0.32973352929899674,
+      "grad_norm": 8.26863360412724,
+      "learning_rate": 1.5749984619163156e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.99609375,
+      "logps/chosen": -358.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1538,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.5,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -22.0,
+      "step": 2580
+    },
+    {
+      "epoch": 0.33101156623426414,
+      "grad_norm": 11.386121523987761,
+      "learning_rate": 1.5719549837837187e-06,
+      "logits/chosen": -1.171875,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -378.0,
+      "logps/rejected": -400.0,
+      "loss": 0.156,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.5625,
+      "rewards/margins": 6.6875,
+      "rewards/rejected": -21.25,
+      "step": 2590
+    },
+    {
+      "epoch": 0.3322896031695316,
+      "grad_norm": 7.313627175926704,
+      "learning_rate": 1.5689290811054722e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -370.0,
+      "logps/rejected": -404.0,
+      "loss": 0.174,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 6.25,
+      "rewards/rejected": -21.375,
+      "step": 2600
+    },
+    {
+      "epoch": 0.333567640104799,
+      "grad_norm": 6.9277782474621725,
+      "learning_rate": 1.5659205853725426e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -366.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1825,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -15.0,
+      "rewards/margins": 5.84375,
+      "rewards/rejected": -20.875,
+      "step": 2610
+    },
+    {
+      "epoch": 0.33484567704006646,
+      "grad_norm": 2.940732392940206,
+      "learning_rate": 1.562929330329127e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.859375,
+      "logps/chosen": -342.0,
+      "logps/rejected": -390.0,
+      "loss": 0.1549,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -14.875,
+      "rewards/margins": 5.625,
+      "rewards/rejected": -20.5,
+      "step": 2620
+    },
+    {
+      "epoch": 0.33612371397533386,
+      "grad_norm": 11.101657578283342,
+      "learning_rate": 1.5599551519340636e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.84765625,
+      "logps/chosen": -352.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1724,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 6.4375,
+      "rewards/rejected": -20.875,
+      "step": 2630
+    },
+    {
+      "epoch": 0.3374017509106013,
+      "grad_norm": 8.670335415068632,
+      "learning_rate": 1.556997888323046e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.9296875,
+      "logps/chosen": -380.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1975,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -15.5625,
+      "rewards/margins": 5.96875,
+      "rewards/rejected": -21.5,
+      "step": 2640
+    },
+    {
+      "epoch": 0.3386797878458687,
+      "grad_norm": 5.752338888030522,
+      "learning_rate": 1.5540573797716226e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -350.0,
+      "logps/rejected": -400.0,
+      "loss": 0.1442,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -14.375,
+      "rewards/margins": 6.3125,
+      "rewards/rejected": -20.75,
+      "step": 2650
+    },
+    {
+      "epoch": 0.3399578247811362,
+      "grad_norm": 8.388520423132393,
+      "learning_rate": 1.5511334686589623e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -380.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1641,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -15.9375,
+      "rewards/margins": 6.3125,
+      "rewards/rejected": -22.25,
+      "step": 2660
+    },
+    {
+      "epoch": 0.3412358617164036,
+      "grad_norm": 8.939976182731543,
+      "learning_rate": 1.548225999432367e-06,
+      "logits/chosen": -1.15625,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -370.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1499,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.5625,
+      "rewards/margins": 5.78125,
+      "rewards/rejected": -21.375,
+      "step": 2670
+    },
+    {
+      "epoch": 0.34251389865167103,
+      "grad_norm": 5.9496176479005785,
+      "learning_rate": 1.5453348185725114e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -366.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1729,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -15.4375,
+      "rewards/margins": 6.5625,
+      "rewards/rejected": -22.0,
+      "step": 2680
+    },
+    {
+      "epoch": 0.34379193558693844,
+      "grad_norm": 13.966256776865988,
+      "learning_rate": 1.542459774559398e-06,
+      "logits/chosen": -1.171875,
+      "logits/rejected": -0.78515625,
+      "logps/chosen": -362.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1621,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.875,
+      "rewards/margins": 7.03125,
+      "rewards/rejected": -22.0,
+      "step": 2690
+    },
+    {
+      "epoch": 0.3450699725222059,
+      "grad_norm": 7.034028534310162,
+      "learning_rate": 1.539600717839002e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -362.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1688,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -15.3125,
+      "rewards/margins": 6.59375,
+      "rewards/rejected": -21.875,
+      "step": 2700
+    },
+    {
+      "epoch": 0.3463480094574733,
+      "grad_norm": 9.005567186950962,
+      "learning_rate": 1.536757500790597e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.77734375,
+      "logps/chosen": -364.0,
+      "logps/rejected": -392.0,
+      "loss": 0.1909,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -15.375,
+      "rewards/margins": 6.25,
+      "rewards/rejected": -21.625,
+      "step": 2710
+    },
+    {
+      "epoch": 0.34762604639274075,
+      "grad_norm": 9.364505905029324,
+      "learning_rate": 1.5339299776947407e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -400.0,
+      "logps/rejected": -424.0,
+      "loss": 0.178,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 6.4375,
+      "rewards/rejected": -22.75,
+      "step": 2720
+    },
+    {
+      "epoch": 0.34890408332800815,
+      "grad_norm": 5.003086199253565,
+      "learning_rate": 1.5311180047019054e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.80859375,
+      "logps/chosen": -360.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1549,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.625,
+      "rewards/margins": 6.46875,
+      "rewards/rejected": -22.125,
+      "step": 2730
+    },
+    {
+      "epoch": 0.3501821202632756,
+      "grad_norm": 9.623306360331048,
+      "learning_rate": 1.5283214398017402e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -348.0,
+      "logps/rejected": -394.0,
+      "loss": 0.1629,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -15.5,
+      "rewards/margins": 6.0625,
+      "rewards/rejected": -21.625,
+      "step": 2740
+    },
+    {
+      "epoch": 0.351460157198543,
+      "grad_norm": 10.01836754137656,
+      "learning_rate": 1.5255401427929477e-06,
+      "logits/chosen": -1.1171875,
+      "logits/rejected": -0.75390625,
+      "logps/chosen": -344.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1731,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.25,
+      "rewards/margins": 7.0625,
+      "rewards/rejected": -21.375,
+      "step": 2750
+    },
+    {
+      "epoch": 0.35273819413381047,
+      "grad_norm": 4.800000068443697,
+      "learning_rate": 1.5227739752537617e-06,
+      "logits/chosen": -1.1328125,
+      "logits/rejected": -0.73046875,
+      "logps/chosen": -370.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1784,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -14.75,
+      "rewards/margins": 6.40625,
+      "rewards/rejected": -21.125,
+      "step": 2760
+    },
+    {
+      "epoch": 0.35401623106907787,
+      "grad_norm": 6.373189708088555,
+      "learning_rate": 1.5200228005130127e-06,
+      "logits/chosen": -1.125,
+      "logits/rejected": -0.80078125,
+      "logps/chosen": -366.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1444,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -14.8125,
+      "rewards/margins": 6.0,
+      "rewards/rejected": -20.75,
+      "step": 2770
+    },
+    {
+      "epoch": 0.3552942680043453,
+      "grad_norm": 7.644695485371685,
+      "learning_rate": 1.5172864836217631e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -374.0,
+      "logps/rejected": -440.0,
+      "loss": 0.137,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 6.4375,
+      "rewards/rejected": -22.5,
+      "step": 2780
+    },
+    {
+      "epoch": 0.3565723049396127,
+      "grad_norm": 8.097310489161947,
+      "learning_rate": 1.514564891325506e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -390.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1555,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 6.3125,
+      "rewards/rejected": -22.875,
+      "step": 2790
+    },
+    {
+      "epoch": 0.3578503418748802,
+      "grad_norm": 8.99040852846663,
+      "learning_rate": 1.5118578920369086e-06,
+      "logits/chosen": -1.171875,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -402.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1426,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.40625,
+      "rewards/rejected": -23.625,
+      "step": 2800
+    },
+    {
+      "epoch": 0.3591283788101476,
+      "grad_norm": 8.535334476511535,
+      "learning_rate": 1.5091653558090898e-06,
+      "logits/chosen": -1.171875,
+      "logits/rejected": -0.76953125,
+      "logps/chosen": -362.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1965,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 6.5625,
+      "rewards/rejected": -22.875,
+      "step": 2810
+    },
+    {
+      "epoch": 0.36040641574541504,
+      "grad_norm": 9.293771809635514,
+      "learning_rate": 1.506487154309419e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.71484375,
+      "logps/chosen": -410.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1594,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 7.15625,
+      "rewards/rejected": -24.25,
+      "step": 2820
+    },
+    {
+      "epoch": 0.36168445268068244,
+      "grad_norm": 8.743440778589854,
+      "learning_rate": 1.5038231607938247e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.74609375,
+      "logps/chosen": -392.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1476,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 7.0,
+      "rewards/rejected": -24.375,
+      "step": 2830
+    },
+    {
+      "epoch": 0.3629624896159499,
+      "grad_norm": 9.690078036330567,
+      "learning_rate": 1.501173250081603e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.98046875,
+      "logps/chosen": -404.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1938,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 7.125,
+      "rewards/rejected": -25.0,
+      "step": 2840
+    },
+    {
+      "epoch": 0.3642405265512173,
+      "grad_norm": 4.2621163641566975,
+      "learning_rate": 1.4985372985307103e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -404.0,
+      "logps/rejected": -458.0,
+      "loss": 0.1472,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.0,
+      "rewards/rejected": -23.75,
+      "step": 2850
+    },
+    {
+      "epoch": 0.36551856348648476,
+      "grad_norm": 4.828756121555101,
+      "learning_rate": 1.4959151840135313e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -392.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1249,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -24.25,
+      "step": 2860
+    },
+    {
+      "epoch": 0.36679660042175216,
+      "grad_norm": 5.944331190146127,
+      "learning_rate": 1.4933067858931148e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.765625,
+      "logps/chosen": -344.0,
+      "logps/rejected": -430.0,
+      "loss": 0.134,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.125,
+      "rewards/rejected": -23.375,
+      "step": 2870
+    },
+    {
+      "epoch": 0.3680746373570196,
+      "grad_norm": 9.008419520073058,
+      "learning_rate": 1.4907119849998597e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.84765625,
+      "logps/chosen": -364.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1347,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -15.8125,
+      "rewards/margins": 6.84375,
+      "rewards/rejected": -22.625,
+      "step": 2880
+    },
+    {
+      "epoch": 0.369352674292287,
+      "grad_norm": 9.42319291093154,
+      "learning_rate": 1.488130663608649e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -354.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1123,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.3125,
+      "rewards/margins": 7.125,
+      "rewards/rejected": -22.375,
+      "step": 2890
+    },
+    {
+      "epoch": 0.3706307112275545,
+      "grad_norm": 4.157268907195367,
+      "learning_rate": 1.4855627054164149e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.76953125,
+      "logps/chosen": -372.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1363,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 6.625,
+      "rewards/rejected": -23.625,
+      "step": 2900
+    },
+    {
+      "epoch": 0.3719087481628219,
+      "grad_norm": 8.135218575835253,
+      "learning_rate": 1.4830079955201294e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.73046875,
+      "logps/chosen": -388.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1517,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 8.1875,
+      "rewards/rejected": -24.125,
+      "step": 2910
+    },
+    {
+      "epoch": 0.37318678509808934,
+      "grad_norm": 12.42181364328349,
+      "learning_rate": 1.4804664203952103e-06,
+      "logits/chosen": -1.1328125,
+      "logits/rejected": -0.625,
+      "logps/chosen": -366.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1575,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.34375,
+      "rewards/rejected": -24.0,
+      "step": 2920
+    },
+    {
+      "epoch": 0.37446482203335674,
+      "grad_norm": 7.042876777904147,
+      "learning_rate": 1.4779378678743327e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -396.0,
+      "logps/rejected": -436.0,
+      "loss": 0.139,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 6.9375,
+      "rewards/rejected": -25.0,
+      "step": 2930
+    },
+    {
+      "epoch": 0.3757428589686242,
+      "grad_norm": 11.999055540560665,
+      "learning_rate": 1.4754222271266348e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -404.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1495,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -24.25,
+      "step": 2940
+    },
+    {
+      "epoch": 0.3770208959038916,
+      "grad_norm": 4.279088372892711,
+      "learning_rate": 1.4729193886373175e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -388.0,
+      "logps/rejected": -442.0,
+      "loss": 0.1205,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 6.65625,
+      "rewards/rejected": -24.0,
+      "step": 2950
+    },
+    {
+      "epoch": 0.37829893283915905,
+      "grad_norm": 8.463208205684165,
+      "learning_rate": 1.4704292441876156e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -406.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1605,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 7.96875,
+      "rewards/rejected": -25.125,
+      "step": 2960
+    },
+    {
+      "epoch": 0.37957696977442645,
+      "grad_norm": 8.555129356496396,
+      "learning_rate": 1.4679516868351474e-06,
+      "logits/chosen": -1.15625,
+      "logits/rejected": -0.78515625,
+      "logps/chosen": -378.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1793,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 6.40625,
+      "rewards/rejected": -24.5,
+      "step": 2970
+    },
+    {
+      "epoch": 0.3808550067096939,
+      "grad_norm": 7.52494832454018,
+      "learning_rate": 1.4654866108946234e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.72265625,
+      "logps/chosen": -390.0,
+      "logps/rejected": -442.0,
+      "loss": 0.1575,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 6.9375,
+      "rewards/rejected": -24.75,
+      "step": 2980
+    },
+    {
+      "epoch": 0.3821330436449613,
+      "grad_norm": 6.54524466297766,
+      "learning_rate": 1.4630339119189101e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -388.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1352,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 5.8125,
+      "rewards/rejected": -23.5,
+      "step": 2990
+    },
+    {
+      "epoch": 0.38341108058022877,
+      "grad_norm": 5.486439248931473,
+      "learning_rate": 1.4605934866804429e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -380.0,
+      "logps/rejected": -438.0,
+      "loss": 0.139,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 5.8125,
+      "rewards/rejected": -24.0,
+      "step": 3000
+    },
+    {
+      "epoch": 0.38468911751549617,
+      "grad_norm": 7.415487137618921,
+      "learning_rate": 1.4581652331529784e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -390.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1555,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.09375,
+      "rewards/rejected": -24.0,
+      "step": 3010
+    },
+    {
+      "epoch": 0.3859671544507636,
+      "grad_norm": 6.598090895788844,
+      "learning_rate": 1.4557490504936778e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1609,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 6.96875,
+      "rewards/rejected": -23.75,
+      "step": 3020
+    },
+    {
+      "epoch": 0.38724519138603103,
+      "grad_norm": 10.385367206717596,
+      "learning_rate": 1.453344839025519e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.82421875,
+      "logps/chosen": -406.0,
+      "logps/rejected": -458.0,
+      "loss": 0.1603,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 6.625,
+      "rewards/rejected": -24.375,
+      "step": 3030
+    },
+    {
+      "epoch": 0.3885232283212985,
+      "grad_norm": 5.0455645746713325,
+      "learning_rate": 1.4509525002200234e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -400.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1597,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 6.09375,
+      "rewards/rejected": -23.25,
+      "step": 3040
+    },
+    {
+      "epoch": 0.3898012652565659,
+      "grad_norm": 6.136449991772514,
+      "learning_rate": 1.4485719366802965e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -392.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1406,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 6.875,
+      "rewards/rejected": -23.625,
+      "step": 3050
+    },
+    {
+      "epoch": 0.39107930219183334,
+      "grad_norm": 6.329761045193293,
+      "learning_rate": 1.4462030521243742e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -398.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1124,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 7.25,
+      "rewards/rejected": -23.5,
+      "step": 3060
+    },
+    {
+      "epoch": 0.39235733912710075,
+      "grad_norm": 8.590404260167736,
+      "learning_rate": 1.443845751368867e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -378.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1636,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -15.75,
+      "rewards/margins": 7.09375,
+      "rewards/rejected": -22.875,
+      "step": 3070
+    },
+    {
+      "epoch": 0.3936353760623682,
+      "grad_norm": 5.0185114606029675,
+      "learning_rate": 1.4414999403128943e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -360.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1615,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 7.03125,
+      "rewards/rejected": -22.25,
+      "step": 3080
+    },
+    {
+      "epoch": 0.3949134129976356,
+      "grad_norm": 6.567002171118385,
+      "learning_rate": 1.439165525922309e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -376.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1409,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 6.375,
+      "rewards/rejected": -22.625,
+      "step": 3090
+    },
+    {
+      "epoch": 0.39619144993290306,
+      "grad_norm": 4.546303949601467,
+      "learning_rate": 1.4368424162141992e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.9921875,
+      "logps/chosen": -370.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1792,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -15.625,
+      "rewards/margins": 6.46875,
+      "rewards/rejected": -22.125,
+      "step": 3100
+    },
+    {
+      "epoch": 0.39746948686817046,
+      "grad_norm": 5.28102598897647,
+      "learning_rate": 1.434530520241665e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -358.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1331,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.375,
+      "rewards/margins": 6.40625,
+      "rewards/rejected": -21.75,
+      "step": 3110
+    },
+    {
+      "epoch": 0.3987475238034379,
+      "grad_norm": 9.951396214687046,
+      "learning_rate": 1.4322297480788657e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -352.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1494,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.0625,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -20.875,
+      "step": 3120
+    },
+    {
+      "epoch": 0.4000255607387053,
+      "grad_norm": 8.010911701910125,
+      "learning_rate": 1.4299400108063247e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -374.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1459,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.375,
+      "rewards/margins": 6.71875,
+      "rewards/rejected": -22.125,
+      "step": 3130
+    },
+    {
+      "epoch": 0.4013035976739728,
+      "grad_norm": 9.211069861794954,
+      "learning_rate": 1.4276612204964992e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -366.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1134,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.25,
+      "rewards/margins": 6.9375,
+      "rewards/rejected": -22.25,
+      "step": 3140
+    },
+    {
+      "epoch": 0.4025816346092402,
+      "grad_norm": 7.078441411817842,
+      "learning_rate": 1.4253932901995967e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -384.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1641,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.375,
+      "rewards/margins": 7.4375,
+      "rewards/rejected": -22.875,
+      "step": 3150
+    },
+    {
+      "epoch": 0.40385967154450764,
+      "grad_norm": 8.418417366520787,
+      "learning_rate": 1.42313613392964e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.97265625,
+      "logps/chosen": -382.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1451,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 6.5,
+      "rewards/rejected": -23.0,
+      "step": 3160
+    },
+    {
+      "epoch": 0.40513770847977504,
+      "grad_norm": 5.618206024968514,
+      "learning_rate": 1.4208896666507756e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -374.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1325,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -15.6875,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -23.125,
+      "step": 3170
+    },
+    {
+      "epoch": 0.4064157454150425,
+      "grad_norm": 4.77507154812098,
+      "learning_rate": 1.4186538042638173e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -384.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1262,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 6.5625,
+      "rewards/rejected": -22.875,
+      "step": 3180
+    },
+    {
+      "epoch": 0.4076937823503099,
+      "grad_norm": 10.916775322429848,
+      "learning_rate": 1.416428463593022e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -364.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1404,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 6.84375,
+      "rewards/rejected": -23.0,
+      "step": 3190
+    },
+    {
+      "epoch": 0.40897181928557735,
+      "grad_norm": 6.999668889227452,
+      "learning_rate": 1.414213562373095e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -372.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1705,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.0,
+      "rewards/rejected": -23.625,
+      "step": 3200
+    },
+    {
+      "epoch": 0.41024985622084476,
+      "grad_norm": 8.395416969574491,
+      "learning_rate": 1.4120090192364154e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -364.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1292,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -15.6875,
+      "rewards/margins": 7.21875,
+      "rewards/rejected": -22.875,
+      "step": 3210
+    },
+    {
+      "epoch": 0.4115278931561122,
+      "grad_norm": 8.0067966090509,
+      "learning_rate": 1.4098147537004828e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.828125,
+      "logps/chosen": -374.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1661,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -15.75,
+      "rewards/margins": 6.875,
+      "rewards/rejected": -22.625,
+      "step": 3220
+    },
+    {
+      "epoch": 0.4128059300913796,
+      "grad_norm": 7.6385206218980235,
+      "learning_rate": 1.4076306861555735e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -1.0,
+      "logps/chosen": -378.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1334,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 6.59375,
+      "rewards/rejected": -22.875,
+      "step": 3230
+    },
+    {
+      "epoch": 0.41408396702664707,
+      "grad_norm": 8.210301082655517,
+      "learning_rate": 1.405456737852613e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -378.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1466,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 7.03125,
+      "rewards/rejected": -23.25,
+      "step": 3240
+    },
+    {
+      "epoch": 0.41536200396191447,
+      "grad_norm": 4.867613780092944,
+      "learning_rate": 1.4032928308912468e-06,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -0.99609375,
+      "logps/chosen": -398.0,
+      "logps/rejected": -448.0,
+      "loss": 0.14,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 6.53125,
+      "rewards/rejected": -23.25,
+      "step": 3250
+    },
+    {
+      "epoch": 0.41664004089718193,
+      "grad_norm": 4.811035199842136,
+      "learning_rate": 1.4011388882081175e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -362.0,
+      "logps/rejected": -400.0,
+      "loss": 0.1588,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 6.28125,
+      "rewards/rejected": -22.625,
+      "step": 3260
+    },
+    {
+      "epoch": 0.41791807783244933,
+      "grad_norm": 4.059663541242835,
+      "learning_rate": 1.3989948335653378e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -384.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1265,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 6.5625,
+      "rewards/rejected": -23.0,
+      "step": 3270
+    },
+    {
+      "epoch": 0.4191961147677168,
+      "grad_norm": 8.701692804817053,
+      "learning_rate": 1.3968605915391564e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -378.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1361,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.0625,
+      "rewards/rejected": -24.125,
+      "step": 3280
+    },
+    {
+      "epoch": 0.4204741517029842,
+      "grad_norm": 7.318323367714644,
+      "learning_rate": 1.3947360875088132e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.984375,
+      "logps/chosen": -386.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1491,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 6.90625,
+      "rewards/rejected": -23.875,
+      "step": 3290
+    },
+    {
+      "epoch": 0.42175218863825165,
+      "grad_norm": 7.433705955493434,
+      "learning_rate": 1.3926212476455828e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -378.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1456,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 6.21875,
+      "rewards/rejected": -24.125,
+      "step": 3300
+    },
+    {
+      "epoch": 0.42303022557351905,
+      "grad_norm": 7.109405172748184,
+      "learning_rate": 1.3905159989019964e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.78515625,
+      "logps/chosen": -394.0,
+      "logps/rejected": -468.0,
+      "loss": 0.151,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 6.8125,
+      "rewards/rejected": -24.0,
+      "step": 3310
+    },
+    {
+      "epoch": 0.4243082625087865,
+      "grad_norm": 10.502226152582534,
+      "learning_rate": 1.3884202690012465e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -396.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1594,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 6.78125,
+      "rewards/rejected": -24.125,
+      "step": 3320
+    },
+    {
+      "epoch": 0.4255862994440539,
+      "grad_norm": 6.602924904771743,
+      "learning_rate": 1.3863339864267636e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -368.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1484,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.25,
+      "rewards/rejected": -23.875,
+      "step": 3330
+    },
+    {
+      "epoch": 0.42686433637932136,
+      "grad_norm": 8.183594026985892,
+      "learning_rate": 1.3842570804119655e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.95703125,
+      "logps/chosen": -378.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1428,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 7.75,
+      "rewards/rejected": -24.875,
+      "step": 3340
+    },
+    {
+      "epoch": 0.42814237331458876,
+      "grad_norm": 7.0083478277008355,
+      "learning_rate": 1.3821894809301763e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -400.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1251,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -24.625,
+      "step": 3350
+    },
+    {
+      "epoch": 0.4294204102498562,
+      "grad_norm": 8.524203508703888,
+      "learning_rate": 1.3801311186847081e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.796875,
+      "logps/chosen": -386.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1317,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.5625,
+      "rewards/rejected": -24.25,
+      "step": 3360
+    },
+    {
+      "epoch": 0.4306984471851236,
+      "grad_norm": 7.785040103569925,
+      "learning_rate": 1.378081925099109e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.8046875,
+      "logps/chosen": -368.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1345,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.15625,
+      "rewards/rejected": -23.25,
+      "step": 3370
+    },
+    {
+      "epoch": 0.4319764841203911,
+      "grad_norm": 7.47296139788113,
+      "learning_rate": 1.376041832307563e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.75390625,
+      "logps/chosen": -364.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1503,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.40625,
+      "rewards/rejected": -23.75,
+      "step": 3380
+    },
+    {
+      "epoch": 0.4332545210556585,
+      "grad_norm": 6.012540650209215,
+      "learning_rate": 1.3740107731454524e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -372.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1339,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 6.59375,
+      "rewards/rejected": -23.25,
+      "step": 3390
+    },
+    {
+      "epoch": 0.43453255799092594,
+      "grad_norm": 7.050229938636495,
+      "learning_rate": 1.3719886811400705e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -380.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1477,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.9375,
+      "rewards/margins": 6.90625,
+      "rewards/rejected": -22.875,
+      "step": 3400
+    },
+    {
+      "epoch": 0.43581059492619334,
+      "grad_norm": 5.797064618099703,
+      "learning_rate": 1.3699754905014834e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -360.0,
+      "logps/rejected": -402.0,
+      "loss": 0.118,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 6.8125,
+      "rewards/rejected": -22.75,
+      "step": 3410
+    },
+    {
+      "epoch": 0.4370886318614608,
+      "grad_norm": 7.162764965684899,
+      "learning_rate": 1.3679711361135388e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.796875,
+      "logps/chosen": -360.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1411,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 7.34375,
+      "rewards/rejected": -23.25,
+      "step": 3420
+    },
+    {
+      "epoch": 0.4383666687967282,
+      "grad_norm": 8.279660889538418,
+      "learning_rate": 1.3659755535250212e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -364.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1438,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.3125,
+      "rewards/margins": 7.25,
+      "rewards/rejected": -22.625,
+      "step": 3430
+    },
+    {
+      "epoch": 0.43964470573199566,
+      "grad_norm": 5.130505672394273,
+      "learning_rate": 1.3639886789409469e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -360.0,
+      "logps/rejected": -396.0,
+      "loss": 0.1241,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 5.875,
+      "rewards/rejected": -21.875,
+      "step": 3440
+    },
+    {
+      "epoch": 0.44092274266726306,
+      "grad_norm": 6.648716979490025,
+      "learning_rate": 1.3620104492139977e-06,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -358.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1334,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -23.625,
+      "step": 3450
+    },
+    {
+      "epoch": 0.4422007796025305,
+      "grad_norm": 6.675946919519406,
+      "learning_rate": 1.3600408018360918e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -366.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1621,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 6.59375,
+      "rewards/rejected": -23.0,
+      "step": 3460
+    },
+    {
+      "epoch": 0.4434788165377979,
+      "grad_norm": 8.299955451919594,
+      "learning_rate": 1.3580796749300878e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -412.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1263,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.0,
+      "rewards/rejected": -23.625,
+      "step": 3470
+    },
+    {
+      "epoch": 0.44475685347306537,
+      "grad_norm": 6.9150783658610395,
+      "learning_rate": 1.3561270072416209e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -402.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1509,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 6.78125,
+      "rewards/rejected": -24.125,
+      "step": 3480
+    },
+    {
+      "epoch": 0.4460348904083328,
+      "grad_norm": 5.570607362596177,
+      "learning_rate": 1.3541827381310652e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -366.0,
+      "logps/rejected": -424.0,
+      "loss": 0.124,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.25,
+      "rewards/rejected": -23.375,
+      "step": 3490
+    },
+    {
+      "epoch": 0.44731292734360023,
+      "grad_norm": 5.940882296922743,
+      "learning_rate": 1.3522468075656264e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -350.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1516,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 7.25,
+      "rewards/rejected": -23.25,
+      "step": 3500
+    },
+    {
+      "epoch": 0.44859096427886763,
+      "grad_norm": 9.961070966676642,
+      "learning_rate": 1.3503191561115553e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -372.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1304,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.28125,
+      "rewards/rejected": -23.875,
+      "step": 3510
+    },
+    {
+      "epoch": 0.4498690012141351,
+      "grad_norm": 6.5528005519508525,
+      "learning_rate": 1.348399724926484e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -396.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1405,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 7.8125,
+      "rewards/rejected": -24.375,
+      "step": 3520
+    },
+    {
+      "epoch": 0.4511470381494025,
+      "grad_norm": 6.578623396344712,
+      "learning_rate": 1.346488455751882e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.796875,
+      "logps/chosen": -384.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1819,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 6.96875,
+      "rewards/rejected": -24.625,
+      "step": 3530
+    },
+    {
+      "epoch": 0.45242507508466995,
+      "grad_norm": 8.785093804488785,
+      "learning_rate": 1.3445852909056286e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -388.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1652,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 6.65625,
+      "rewards/rejected": -24.125,
+      "step": 3540
+    },
+    {
+      "epoch": 0.45370311201993735,
+      "grad_norm": 6.659970917194002,
+      "learning_rate": 1.3426901732747024e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -408.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1442,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 6.625,
+      "rewards/rejected": -24.125,
+      "step": 3550
+    },
+    {
+      "epoch": 0.4549811489552048,
+      "grad_norm": 8.705929632452246,
+      "learning_rate": 1.3408030463079818e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.8046875,
+      "logps/chosen": -378.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1656,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 6.84375,
+      "rewards/rejected": -24.25,
+      "step": 3560
+    },
+    {
+      "epoch": 0.4562591858904722,
+      "grad_norm": 7.865053838646732,
+      "learning_rate": 1.3389238540091568e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.7734375,
+      "logps/chosen": -376.0,
+      "logps/rejected": -428.0,
+      "loss": 0.123,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -23.875,
+      "step": 3570
+    },
+    {
+      "epoch": 0.45753722282573966,
+      "grad_norm": 9.84530105170181,
+      "learning_rate": 1.337052540929751e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -368.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1261,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.65625,
+      "rewards/rejected": -24.125,
+      "step": 3580
+    },
+    {
+      "epoch": 0.45881525976100707,
+      "grad_norm": 11.876771603542881,
+      "learning_rate": 1.33518905216225e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.96484375,
+      "logps/chosen": -382.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1477,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 7.09375,
+      "rewards/rejected": -23.375,
+      "step": 3590
+    },
+    {
+      "epoch": 0.4600932966962745,
+      "grad_norm": 5.902063754155465,
+      "learning_rate": 1.3333333333333332e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -374.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1339,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -23.5,
+      "step": 3600
+    },
+    {
+      "epoch": 0.4613713336315419,
+      "grad_norm": 4.309507216845633,
+      "learning_rate": 1.3314853305972122e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -384.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1481,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.9375,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -23.375,
+      "step": 3610
+    },
+    {
+      "epoch": 0.4626493705668094,
+      "grad_norm": 5.5609111347573625,
+      "learning_rate": 1.3296449906290671e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.9296875,
+      "logps/chosen": -366.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1167,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 6.5,
+      "rewards/rejected": -21.625,
+      "step": 3620
+    },
+    {
+      "epoch": 0.4639274075020768,
+      "grad_norm": 6.634986614020427,
+      "learning_rate": 1.3278122606185844e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -354.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1248,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -15.25,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -22.5,
+      "step": 3630
+    },
+    {
+      "epoch": 0.46520544443734424,
+      "grad_norm": 6.326554708004468,
+      "learning_rate": 1.3259870882635918e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1526,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 6.34375,
+      "rewards/rejected": -22.375,
+      "step": 3640
+    },
+    {
+      "epoch": 0.46648348137261164,
+      "grad_norm": 5.094202837967204,
+      "learning_rate": 1.3241694217637886e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.97265625,
+      "logps/chosen": -360.0,
+      "logps/rejected": -394.0,
+      "loss": 0.1396,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 6.9375,
+      "rewards/rejected": -22.75,
+      "step": 3650
+    },
+    {
+      "epoch": 0.4677615183078791,
+      "grad_norm": 8.427385570807399,
+      "learning_rate": 1.3223592098145723e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -388.0,
+      "logps/rejected": -456.0,
+      "loss": 0.1286,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 8.375,
+      "rewards/rejected": -23.5,
+      "step": 3660
+    },
+    {
+      "epoch": 0.4690395552431465,
+      "grad_norm": 7.211951371799652,
+      "learning_rate": 1.3205564016009555e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -372.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1165,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.21875,
+      "rewards/rejected": -23.375,
+      "step": 3670
+    },
+    {
+      "epoch": 0.47031759217841396,
+      "grad_norm": 6.809528260138286,
+      "learning_rate": 1.318760946791574e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -352.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1237,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 7.6875,
+      "rewards/rejected": -23.875,
+      "step": 3680
+    },
+    {
+      "epoch": 0.47159562911368136,
+      "grad_norm": 8.846543297014522,
+      "learning_rate": 1.316972795532786e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -390.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1506,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -23.5,
+      "step": 3690
+    },
+    {
+      "epoch": 0.4728736660489488,
+      "grad_norm": 3.2133593797776285,
+      "learning_rate": 1.3151918984428582e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -364.0,
+      "logps/rejected": -418.0,
+      "loss": 0.0956,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.25,
+      "rewards/margins": 7.6875,
+      "rewards/rejected": -23.0,
+      "step": 3700
+    },
+    {
+      "epoch": 0.4741517029842162,
+      "grad_norm": 5.784453643785327,
+      "learning_rate": 1.313418206606237e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -368.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1197,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 6.53125,
+      "rewards/rejected": -22.5,
+      "step": 3710
+    },
+    {
+      "epoch": 0.4754297399194837,
+      "grad_norm": 6.781922564364168,
+      "learning_rate": 1.3116516715679057e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.9296875,
+      "logps/chosen": -364.0,
+      "logps/rejected": -398.0,
+      "loss": 0.154,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.125,
+      "rewards/margins": 6.875,
+      "rewards/rejected": -22.0,
+      "step": 3720
+    },
+    {
+      "epoch": 0.4767077768547511,
+      "grad_norm": 6.579401642637866,
+      "learning_rate": 1.3098922453278258e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -386.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1346,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.28125,
+      "rewards/rejected": -23.625,
+      "step": 3730
+    },
+    {
+      "epoch": 0.47798581379001853,
+      "grad_norm": 6.015763061900998,
+      "learning_rate": 1.3081398803354573e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.98828125,
+      "logps/chosen": -392.0,
+      "logps/rejected": -442.0,
+      "loss": 0.1295,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.75,
+      "rewards/rejected": -23.875,
+      "step": 3740
+    },
+    {
+      "epoch": 0.47926385072528593,
+      "grad_norm": 6.669262621364384,
+      "learning_rate": 1.3063945294843617e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -372.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1294,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -23.5,
+      "step": 3750
+    },
+    {
+      "epoch": 0.4805418876605534,
+      "grad_norm": 4.0120252515287955,
+      "learning_rate": 1.3046561461068843e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -378.0,
+      "logps/rejected": -436.0,
+      "loss": 0.12,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 6.84375,
+      "rewards/rejected": -23.0,
+      "step": 3760
+    },
+    {
+      "epoch": 0.4818199245958208,
+      "grad_norm": 7.621500757066371,
+      "learning_rate": 1.3029246839689124e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.98046875,
+      "logps/chosen": -392.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1411,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -24.0,
+      "step": 3770
+    },
+    {
+      "epoch": 0.48309796153108825,
+      "grad_norm": 7.352830789591676,
+      "learning_rate": 1.3012000972647109e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -380.0,
+      "logps/rejected": -418.0,
+      "loss": 0.158,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -15.625,
+      "rewards/margins": 6.59375,
+      "rewards/rejected": -22.25,
+      "step": 3780
+    },
+    {
+      "epoch": 0.48437599846635565,
+      "grad_norm": 5.607864180489799,
+      "learning_rate": 1.299482340611832e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.77734375,
+      "logps/chosen": -356.0,
+      "logps/rejected": -426.0,
+      "loss": 0.172,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.125,
+      "rewards/margins": 6.46875,
+      "rewards/rejected": -21.625,
+      "step": 3790
+    },
+    {
+      "epoch": 0.4856540354016231,
+      "grad_norm": 8.337199650164589,
+      "learning_rate": 1.2977713690461003e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -360.0,
+      "logps/rejected": -410.0,
+      "loss": 0.143,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 6.5,
+      "rewards/rejected": -22.5,
+      "step": 3800
+    },
+    {
+      "epoch": 0.4869320723368905,
+      "grad_norm": 7.803981114105088,
+      "learning_rate": 1.296067138016669e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -380.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1407,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -23.0,
+      "step": 3810
+    },
+    {
+      "epoch": 0.48821010927215797,
+      "grad_norm": 8.038860278583183,
+      "learning_rate": 1.294369603381147e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -386.0,
+      "logps/rejected": -418.0,
+      "loss": 0.144,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.0,
+      "rewards/rejected": -23.375,
+      "step": 3820
+    },
+    {
+      "epoch": 0.48948814620742537,
+      "grad_norm": 6.43591814741643,
+      "learning_rate": 1.2926787214007981e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.875,
+      "logps/chosen": -398.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1177,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.15625,
+      "rewards/rejected": -23.75,
+      "step": 3830
+    },
+    {
+      "epoch": 0.4907661831426928,
+      "grad_norm": 3.5902859695524527,
+      "learning_rate": 1.2909944487358056e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -366.0,
+      "logps/rejected": -400.0,
+      "loss": 0.1301,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.25,
+      "rewards/margins": 7.78125,
+      "rewards/rejected": -23.0,
+      "step": 3840
+    },
+    {
+      "epoch": 0.4920442200779602,
+      "grad_norm": 7.283578333151166,
+      "learning_rate": 1.2893167424406084e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -386.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1177,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 6.625,
+      "rewards/rejected": -23.625,
+      "step": 3850
+    },
+    {
+      "epoch": 0.4933222570132277,
+      "grad_norm": 4.745729088913617,
+      "learning_rate": 1.2876455599593008e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -390.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1236,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 7.65625,
+      "rewards/rejected": -24.875,
+      "step": 3860
+    },
+    {
+      "epoch": 0.4946002939484951,
+      "grad_norm": 6.486734448244225,
+      "learning_rate": 1.285980859121099e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -376.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1458,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.4375,
+      "rewards/rejected": -24.25,
+      "step": 3870
+    },
+    {
+      "epoch": 0.49587833088376254,
+      "grad_norm": 6.750032073045271,
+      "learning_rate": 1.2843225981358712e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1463,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.875,
+      "rewards/rejected": -24.75,
+      "step": 3880
+    },
+    {
+      "epoch": 0.49715636781902994,
+      "grad_norm": 5.366617500901339,
+      "learning_rate": 1.2826707355897317e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -388.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1171,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.96875,
+      "rewards/rejected": -24.625,
+      "step": 3890
+    },
+    {
+      "epoch": 0.4984344047542974,
+      "grad_norm": 5.6016430982311505,
+      "learning_rate": 1.281025230440697e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -388.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1458,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 7.03125,
+      "rewards/rejected": -24.125,
+      "step": 3900
+    },
+    {
+      "epoch": 0.4997124416895648,
+      "grad_norm": 7.300524831924248,
+      "learning_rate": 1.2793860420144025e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.92578125,
+      "logps/chosen": -398.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1305,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.5625,
+      "rewards/rejected": -24.25,
+      "step": 3910
+    },
+    {
+      "epoch": 0.5009904786248323,
+      "grad_norm": 5.725813515946119,
+      "learning_rate": 1.2777531299998798e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -406.0,
+      "logps/rejected": -444.0,
+      "loss": 0.111,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -15.9375,
+      "rewards/margins": 6.9375,
+      "rewards/rejected": -22.875,
+      "step": 3920
+    },
+    {
+      "epoch": 0.5022685155600997,
+      "grad_norm": 5.96376715796183,
+      "learning_rate": 1.2761264544453928e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.828125,
+      "logps/chosen": -378.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1181,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -15.4375,
+      "rewards/margins": 6.0,
+      "rewards/rejected": -21.375,
+      "step": 3930
+    },
+    {
+      "epoch": 0.5035465524953671,
+      "grad_norm": 8.80359119969491,
+      "learning_rate": 1.2745059757543324e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -1.0,
+      "logps/chosen": -384.0,
+      "logps/rejected": -422.0,
+      "loss": 0.134,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 6.59375,
+      "rewards/rejected": -23.25,
+      "step": 3940
+    },
+    {
+      "epoch": 0.5048245894306346,
+      "grad_norm": 7.6550399984015405,
+      "learning_rate": 1.272891654681168e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -368.0,
+      "logps/rejected": -416.0,
+      "loss": 0.143,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 6.71875,
+      "rewards/rejected": -22.875,
+      "step": 3950
+    },
+    {
+      "epoch": 0.506102626365902,
+      "grad_norm": 5.483471349047118,
+      "learning_rate": 1.2712834523274563e-06,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -368.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1324,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -15.8125,
+      "rewards/margins": 7.34375,
+      "rewards/rejected": -23.125,
+      "step": 3960
+    },
+    {
+      "epoch": 0.5073806633011694,
+      "grad_norm": 10.71304960324114,
+      "learning_rate": 1.2696813301379032e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -372.0,
+      "logps/rejected": -414.0,
+      "loss": 0.137,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -23.625,
+      "step": 3970
+    },
+    {
+      "epoch": 0.5086587002364368,
+      "grad_norm": 8.277701889137077,
+      "learning_rate": 1.2680852498964829e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -362.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1548,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -23.75,
+      "step": 3980
+    },
+    {
+      "epoch": 0.5099367371717043,
+      "grad_norm": 5.769367783646121,
+      "learning_rate": 1.266495173722607e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -354.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1464,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -23.125,
+      "step": 3990
+    },
+    {
+      "epoch": 0.5112147741069717,
+      "grad_norm": 6.272387903015177,
+      "learning_rate": 1.2649110640673517e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -356.0,
+      "logps/rejected": -392.0,
+      "loss": 0.1231,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.4375,
+      "rewards/margins": 6.5625,
+      "rewards/rejected": -22.0,
+      "step": 4000
+    },
+    {
+      "epoch": 0.5124928110422391,
+      "grad_norm": 6.397249871423641,
+      "learning_rate": 1.2633328837097308e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -382.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1148,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.0,
+      "rewards/rejected": -23.375,
+      "step": 4010
+    },
+    {
+      "epoch": 0.5137708479775065,
+      "grad_norm": 7.715633010076507,
+      "learning_rate": 1.2617605957530233e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -380.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1263,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 6.84375,
+      "rewards/rejected": -23.625,
+      "step": 4020
+    },
+    {
+      "epoch": 0.515048884912774,
+      "grad_norm": 7.265100762096714,
+      "learning_rate": 1.2601941636211516e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -394.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1321,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 7.21875,
+      "rewards/rejected": -23.375,
+      "step": 4030
+    },
+    {
+      "epoch": 0.5163269218480414,
+      "grad_norm": 7.086112132708502,
+      "learning_rate": 1.2586335510551052e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.96484375,
+      "logps/chosen": -372.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1387,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -15.5,
+      "rewards/margins": 6.6875,
+      "rewards/rejected": -22.125,
+      "step": 4040
+    },
+    {
+      "epoch": 0.5176049587833088,
+      "grad_norm": 8.125515700540811,
+      "learning_rate": 1.2570787221094177e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -380.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1574,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 7.9375,
+      "rewards/rejected": -22.875,
+      "step": 4050
+    },
+    {
+      "epoch": 0.5188829957185762,
+      "grad_norm": 7.315820021163162,
+      "learning_rate": 1.255529641148689e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -372.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1538,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -15.25,
+      "rewards/margins": 6.6875,
+      "rewards/rejected": -22.0,
+      "step": 4060
+    },
+    {
+      "epoch": 0.5201610326538437,
+      "grad_norm": 22.39114015996305,
+      "learning_rate": 1.2539862728441536e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -364.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1639,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.6875,
+      "rewards/margins": 6.625,
+      "rewards/rejected": -22.25,
+      "step": 4070
+    },
+    {
+      "epoch": 0.5214390695891111,
+      "grad_norm": 8.686670239736506,
+      "learning_rate": 1.252448582170299e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -370.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1083,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.375,
+      "rewards/margins": 6.1875,
+      "rewards/rejected": -21.5,
+      "step": 4080
+    },
+    {
+      "epoch": 0.5227171065243785,
+      "grad_norm": 8.143737644318952,
+      "learning_rate": 1.2509165344015243e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -384.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1209,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -22.75,
+      "step": 4090
+    },
+    {
+      "epoch": 0.5239951434596459,
+      "grad_norm": 7.705985852088029,
+      "learning_rate": 1.2493900951088486e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -374.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1371,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.375,
+      "rewards/margins": 7.40625,
+      "rewards/rejected": -22.875,
+      "step": 4100
+    },
+    {
+      "epoch": 0.5252731803949134,
+      "grad_norm": 4.480208286223466,
+      "learning_rate": 1.2478692301566601e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -378.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1414,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -15.75,
+      "rewards/margins": 7.5625,
+      "rewards/rejected": -23.375,
+      "step": 4110
+    },
+    {
+      "epoch": 0.5265512173301808,
+      "grad_norm": 2.4333876673784225,
+      "learning_rate": 1.2463539056995116e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.80078125,
+      "logps/chosen": -362.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1238,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.1875,
+      "rewards/rejected": -23.625,
+      "step": 4120
+    },
+    {
+      "epoch": 0.5278292542654482,
+      "grad_norm": 5.35055137682526,
+      "learning_rate": 1.2448440881789541e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.953125,
+      "logps/chosen": -394.0,
+      "logps/rejected": -452.0,
+      "loss": 0.1305,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 8.4375,
+      "rewards/rejected": -24.25,
+      "step": 4130
+    },
+    {
+      "epoch": 0.5291072912007156,
+      "grad_norm": 5.93172402383006,
+      "learning_rate": 1.2433397443204184e-06,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -388.0,
+      "logps/rejected": -452.0,
+      "loss": 0.1503,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -24.375,
+      "step": 4140
+    },
+    {
+      "epoch": 0.5303853281359832,
+      "grad_norm": 11.147043198835181,
+      "learning_rate": 1.2418408411301324e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -370.0,
+      "logps/rejected": -426.0,
+      "loss": 0.15,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 6.21875,
+      "rewards/rejected": -23.625,
+      "step": 4150
+    },
+    {
+      "epoch": 0.5316633650712506,
+      "grad_norm": 6.23147011628777,
+      "learning_rate": 1.2403473458920844e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.828125,
+      "logps/chosen": -356.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1429,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -15.0625,
+      "rewards/margins": 7.53125,
+      "rewards/rejected": -22.625,
+      "step": 4160
+    },
+    {
+      "epoch": 0.532941402006518,
+      "grad_norm": 4.446541921821213,
+      "learning_rate": 1.2388592261650217e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -362.0,
+      "logps/rejected": -396.0,
+      "loss": 0.1123,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.8125,
+      "rewards/margins": 6.875,
+      "rewards/rejected": -22.625,
+      "step": 4170
+    },
+    {
+      "epoch": 0.5342194389417854,
+      "grad_norm": 7.657536701484431,
+      "learning_rate": 1.2373764497794918e-06,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -390.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1476,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 6.71875,
+      "rewards/rejected": -23.75,
+      "step": 4180
+    },
+    {
+      "epoch": 0.5354974758770529,
+      "grad_norm": 5.478370016381274,
+      "learning_rate": 1.2358989848349217e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -364.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1193,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 6.6875,
+      "rewards/rejected": -23.125,
+      "step": 4190
+    },
+    {
+      "epoch": 0.5367755128123203,
+      "grad_norm": 5.7301600946823825,
+      "learning_rate": 1.2344267996967353e-06,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -376.0,
+      "logps/rejected": -442.0,
+      "loss": 0.1326,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -24.375,
+      "step": 4200
+    },
+    {
+      "epoch": 0.5380535497475877,
+      "grad_norm": 4.774275280916184,
+      "learning_rate": 1.2329598629935076e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -378.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1263,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 7.1875,
+      "rewards/rejected": -23.125,
+      "step": 4210
+    },
+    {
+      "epoch": 0.5393315866828551,
+      "grad_norm": 7.774838256510226,
+      "learning_rate": 1.2314981436141583e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.92578125,
+      "logps/chosen": -358.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1137,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.4375,
+      "rewards/margins": 7.5625,
+      "rewards/rejected": -23.0,
+      "step": 4220
+    },
+    {
+      "epoch": 0.5406096236181226,
+      "grad_norm": 8.433635024915732,
+      "learning_rate": 1.2300416107051802e-06,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -356.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1393,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -15.25,
+      "rewards/margins": 7.0,
+      "rewards/rejected": -22.25,
+      "step": 4230
+    },
+    {
+      "epoch": 0.54188766055339,
+      "grad_norm": 10.875227564567004,
+      "learning_rate": 1.2285902336679024e-06,
+      "logits/chosen": -1.4609375,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -362.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1097,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.8125,
+      "rewards/margins": 7.0625,
+      "rewards/rejected": -22.875,
+      "step": 4240
+    },
+    {
+      "epoch": 0.5431656974886574,
+      "grad_norm": 8.921357330227176,
+      "learning_rate": 1.2271439821557926e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -366.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1154,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.9375,
+      "rewards/margins": 7.40625,
+      "rewards/rejected": -23.25,
+      "step": 4250
+    },
+    {
+      "epoch": 0.5444437344239248,
+      "grad_norm": 8.252792716610507,
+      "learning_rate": 1.225702826071791e-06,
+      "logits/chosen": -1.5078125,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -376.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1443,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.34375,
+      "rewards/rejected": -23.5,
+      "step": 4260
+    },
+    {
+      "epoch": 0.5457217713591923,
+      "grad_norm": 7.473202204971072,
+      "learning_rate": 1.2242667355656797e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -380.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1364,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.375,
+      "rewards/margins": 7.5625,
+      "rewards/rejected": -23.0,
+      "step": 4270
+    },
+    {
+      "epoch": 0.5469998082944597,
+      "grad_norm": 7.7363886995317355,
+      "learning_rate": 1.2228356810314862e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.984375,
+      "logps/chosen": -364.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1502,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -15.625,
+      "rewards/margins": 6.53125,
+      "rewards/rejected": -22.125,
+      "step": 4280
+    },
+    {
+      "epoch": 0.5482778452297271,
+      "grad_norm": 8.818626772338112,
+      "learning_rate": 1.2214096331049186e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -358.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1258,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -14.0625,
+      "rewards/margins": 7.53125,
+      "rewards/rejected": -21.625,
+      "step": 4290
+    },
+    {
+      "epoch": 0.5495558821649945,
+      "grad_norm": 7.5505625686209825,
+      "learning_rate": 1.2199885626608373e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -354.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1376,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.25,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -22.625,
+      "step": 4300
+    },
+    {
+      "epoch": 0.550833919100262,
+      "grad_norm": 6.678882624518472,
+      "learning_rate": 1.2185724408107546e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -364.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1399,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -15.0625,
+      "rewards/margins": 6.96875,
+      "rewards/rejected": -22.0,
+      "step": 4310
+    },
+    {
+      "epoch": 0.5521119560355294,
+      "grad_norm": 7.245285765739401,
+      "learning_rate": 1.2171612389003689e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -352.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1356,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -15.0625,
+      "rewards/margins": 7.09375,
+      "rewards/rejected": -22.125,
+      "step": 4320
+    },
+    {
+      "epoch": 0.5533899929707968,
+      "grad_norm": 8.21866407833178,
+      "learning_rate": 1.2157549285071297e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -372.0,
+      "logps/rejected": -410.0,
+      "loss": 0.106,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 6.5625,
+      "rewards/rejected": -22.375,
+      "step": 4330
+    },
+    {
+      "epoch": 0.5546680299060642,
+      "grad_norm": 8.661730450075092,
+      "learning_rate": 1.2143534814378327e-06,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -378.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1242,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -15.625,
+      "rewards/margins": 6.59375,
+      "rewards/rejected": -22.25,
+      "step": 4340
+    },
+    {
+      "epoch": 0.5559460668413317,
+      "grad_norm": 5.5083363733839805,
+      "learning_rate": 1.2129568697262454e-06,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -0.95703125,
+      "logps/chosen": -368.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1263,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -22.75,
+      "step": 4350
+    },
+    {
+      "epoch": 0.5572241037765991,
+      "grad_norm": 5.512307398794429,
+      "learning_rate": 1.2115650656307653e-06,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -380.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1244,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.625,
+      "rewards/margins": 7.25,
+      "rewards/rejected": -22.875,
+      "step": 4360
+    },
+    {
+      "epoch": 0.5585021407118665,
+      "grad_norm": 7.5971401253884165,
+      "learning_rate": 1.210178041632103e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -370.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1305,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.40625,
+      "rewards/rejected": -23.75,
+      "step": 4370
+    },
+    {
+      "epoch": 0.559780177647134,
+      "grad_norm": 6.630350438067686,
+      "learning_rate": 1.2087957704309988e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -384.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1244,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.0625,
+      "rewards/rejected": -23.5,
+      "step": 4380
+    },
+    {
+      "epoch": 0.5610582145824015,
+      "grad_norm": 10.488261818934928,
+      "learning_rate": 1.2074182249459642e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -376.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1551,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 7.1875,
+      "rewards/rejected": -23.125,
+      "step": 4390
+    },
+    {
+      "epoch": 0.5623362515176689,
+      "grad_norm": 6.230269184402891,
+      "learning_rate": 1.2060453783110545e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -374.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1294,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -23.375,
+      "step": 4400
+    },
+    {
+      "epoch": 0.5636142884529363,
+      "grad_norm": 6.040356565097965,
+      "learning_rate": 1.2046772038736682e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -366.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1105,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 7.03125,
+      "rewards/rejected": -23.0,
+      "step": 4410
+    },
+    {
+      "epoch": 0.5648923253882037,
+      "grad_norm": 4.666732622676692,
+      "learning_rate": 1.2033136751923736e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -386.0,
+      "logps/rejected": -450.0,
+      "loss": 0.1379,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.28125,
+      "rewards/rejected": -23.625,
+      "step": 4420
+    },
+    {
+      "epoch": 0.5661703623234712,
+      "grad_norm": 5.96478737879899,
+      "learning_rate": 1.201954766034762e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.9296875,
+      "logps/chosen": -378.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1438,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -23.375,
+      "step": 4430
+    },
+    {
+      "epoch": 0.5674483992587386,
+      "grad_norm": 7.332223375221347,
+      "learning_rate": 1.2006004503753285e-06,
+      "logits/chosen": -1.4453125,
+      "logits/rejected": -0.99609375,
+      "logps/chosen": -362.0,
+      "logps/rejected": -450.0,
+      "loss": 0.1319,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -24.125,
+      "step": 4440
+    },
+    {
+      "epoch": 0.568726436194006,
+      "grad_norm": 8.698913609191186,
+      "learning_rate": 1.1992507023933782e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -408.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1348,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 6.8125,
+      "rewards/rejected": -24.0,
+      "step": 4450
+    },
+    {
+      "epoch": 0.5700044731292734,
+      "grad_norm": 5.776887310182114,
+      "learning_rate": 1.1979054964709597e-06,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -396.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1294,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.78125,
+      "rewards/rejected": -24.375,
+      "step": 4460
+    },
+    {
+      "epoch": 0.5712825100645409,
+      "grad_norm": 5.951138908035143,
+      "learning_rate": 1.1965648071908207e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -378.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1051,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 6.65625,
+      "rewards/rejected": -24.125,
+      "step": 4470
+    },
+    {
+      "epoch": 0.5725605469998083,
+      "grad_norm": 6.097785333676172,
+      "learning_rate": 1.1952286093343935e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -380.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1212,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.75,
+      "rewards/rejected": -24.375,
+      "step": 4480
+    },
+    {
+      "epoch": 0.5738385839350757,
+      "grad_norm": 9.143941912660718,
+      "learning_rate": 1.1938968778798005e-06,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -394.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1349,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 7.28125,
+      "rewards/rejected": -24.125,
+      "step": 4490
+    },
+    {
+      "epoch": 0.5751166208703431,
+      "grad_norm": 6.440276465141123,
+      "learning_rate": 1.1925695879998878e-06,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -398.0,
+      "logps/rejected": -456.0,
+      "loss": 0.125,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 7.875,
+      "rewards/rejected": -25.25,
+      "step": 4500
+    },
+    {
+      "epoch": 0.5763946578056106,
+      "grad_norm": 8.716998336076474,
+      "learning_rate": 1.1912467150602794e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -366.0,
+      "logps/rejected": -452.0,
+      "loss": 0.1199,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 8.8125,
+      "rewards/rejected": -25.375,
+      "step": 4510
+    },
+    {
+      "epoch": 0.577672694740878,
+      "grad_norm": 6.651575355256573,
+      "learning_rate": 1.189928234617459e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.859375,
+      "logps/chosen": -374.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1191,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -24.125,
+      "step": 4520
+    },
+    {
+      "epoch": 0.5789507316761454,
+      "grad_norm": 6.247748910842715,
+      "learning_rate": 1.1886141224168716e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -390.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1212,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -24.625,
+      "step": 4530
+    },
+    {
+      "epoch": 0.5802287686114128,
+      "grad_norm": 6.9258690089374255,
+      "learning_rate": 1.1873043543910495e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -380.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1167,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 7.28125,
+      "rewards/rejected": -23.75,
+      "step": 4540
+    },
+    {
+      "epoch": 0.5815068055466803,
+      "grad_norm": 8.000132895834788,
+      "learning_rate": 1.1859989066577617e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -370.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1314,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 7.71875,
+      "rewards/rejected": -22.875,
+      "step": 4550
+    },
+    {
+      "epoch": 0.5827848424819477,
+      "grad_norm": 5.966222259143792,
+      "learning_rate": 1.1846977555181846e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -376.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1325,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 7.25,
+      "rewards/rejected": -23.25,
+      "step": 4560
+    },
+    {
+      "epoch": 0.5840628794172151,
+      "grad_norm": 6.147076902370972,
+      "learning_rate": 1.1834008774550946e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -372.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1173,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.4375,
+      "rewards/margins": 6.84375,
+      "rewards/rejected": -22.25,
+      "step": 4570
+    },
+    {
+      "epoch": 0.5853409163524825,
+      "grad_norm": 8.010391100544743,
+      "learning_rate": 1.1821082491310835e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -364.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1168,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -15.5625,
+      "rewards/margins": 7.78125,
+      "rewards/rejected": -23.25,
+      "step": 4580
+    },
+    {
+      "epoch": 0.58661895328775,
+      "grad_norm": 6.068474078239673,
+      "learning_rate": 1.1808198473867937e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -376.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1246,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.8125,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -23.375,
+      "step": 4590
+    },
+    {
+      "epoch": 0.5878969902230174,
+      "grad_norm": 6.406617177416852,
+      "learning_rate": 1.179535649239177e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -364.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1096,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -14.8125,
+      "rewards/margins": 8.25,
+      "rewards/rejected": -23.125,
+      "step": 4600
+    },
+    {
+      "epoch": 0.5891750271582848,
+      "grad_norm": 8.290927899216316,
+      "learning_rate": 1.178255631879771e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -362.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1148,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.25,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -22.75,
+      "step": 4610
+    },
+    {
+      "epoch": 0.5904530640935522,
+      "grad_norm": 5.9605715783820274,
+      "learning_rate": 1.1769797726729992e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -366.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1086,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -14.5,
+      "rewards/margins": 7.1875,
+      "rewards/rejected": -21.625,
+      "step": 4620
+    },
+    {
+      "epoch": 0.5917311010288198,
+      "grad_norm": 6.725237663405956,
+      "learning_rate": 1.1757080491544881e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -364.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1042,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.3125,
+      "rewards/margins": 7.875,
+      "rewards/rejected": -22.25,
+      "step": 4630
+    },
+    {
+      "epoch": 0.5930091379640872,
+      "grad_norm": 8.002073696839526,
+      "learning_rate": 1.1744404390294068e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.9921875,
+      "logps/chosen": -364.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1137,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -14.3125,
+      "rewards/margins": 6.3125,
+      "rewards/rejected": -20.625,
+      "step": 4640
+    },
+    {
+      "epoch": 0.5942871748993546,
+      "grad_norm": 6.054743738816849,
+      "learning_rate": 1.1731769201708264e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -346.0,
+      "logps/rejected": -398.0,
+      "loss": 0.1371,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 7.1875,
+      "rewards/rejected": -21.625,
+      "step": 4650
+    },
+    {
+      "epoch": 0.595565211834622,
+      "grad_norm": 7.417536537941938,
+      "learning_rate": 1.1719174706180952e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -376.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1373,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.125,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -22.625,
+      "step": 4660
+    },
+    {
+      "epoch": 0.5968432487698895,
+      "grad_norm": 5.408735428493242,
+      "learning_rate": 1.1706620685752386e-06,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -0.99609375,
+      "logps/chosen": -356.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1264,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.5625,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -22.125,
+      "step": 4670
+    },
+    {
+      "epoch": 0.5981212857051569,
+      "grad_norm": 5.538882792551052,
+      "learning_rate": 1.1694106924093723e-06,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -352.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1069,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -14.8125,
+      "rewards/margins": 8.3125,
+      "rewards/rejected": -23.125,
+      "step": 4680
+    },
+    {
+      "epoch": 0.5993993226404243,
+      "grad_norm": 10.875730375001794,
+      "learning_rate": 1.1681633206491381e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.98828125,
+      "logps/chosen": -370.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1227,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.6875,
+      "rewards/margins": 8.1875,
+      "rewards/rejected": -23.875,
+      "step": 4690
+    },
+    {
+      "epoch": 0.6006773595756917,
+      "grad_norm": 5.554613321442294,
+      "learning_rate": 1.1669199319831564e-06,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -372.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1381,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.6875,
+      "rewards/margins": 7.5625,
+      "rewards/rejected": -23.25,
+      "step": 4700
+    },
+    {
+      "epoch": 0.6019553965109592,
+      "grad_norm": 7.948360914985641,
+      "learning_rate": 1.1656805052584958e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.82421875,
+      "logps/chosen": -370.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1109,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -22.75,
+      "step": 4710
+    },
+    {
+      "epoch": 0.6032334334462266,
+      "grad_norm": 4.633069393467309,
+      "learning_rate": 1.164445019479164e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -350.0,
+      "logps/rejected": -396.0,
+      "loss": 0.0897,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -13.5625,
+      "rewards/margins": 7.9375,
+      "rewards/rejected": -21.5,
+      "step": 4720
+    },
+    {
+      "epoch": 0.604511470381494,
+      "grad_norm": 4.436902856769011,
+      "learning_rate": 1.1632134538046105e-06,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -328.0,
+      "logps/rejected": -396.0,
+      "loss": 0.1261,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -13.625,
+      "rewards/margins": 6.78125,
+      "rewards/rejected": -20.375,
+      "step": 4730
+    },
+    {
+      "epoch": 0.6057895073167614,
+      "grad_norm": 10.94179383389387,
+      "learning_rate": 1.1619857875482536e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -338.0,
+      "logps/rejected": -380.0,
+      "loss": 0.0886,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -13.625,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -21.125,
+      "step": 4740
+    },
+    {
+      "epoch": 0.6070675442520289,
+      "grad_norm": 6.8102231649488845,
+      "learning_rate": 1.1607620001760185e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -346.0,
+      "logps/rejected": -388.0,
+      "loss": 0.133,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 7.125,
+      "rewards/rejected": -22.0,
+      "step": 4750
+    },
+    {
+      "epoch": 0.6083455811872963,
+      "grad_norm": 6.295002527370573,
+      "learning_rate": 1.1595420713048968e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -370.0,
+      "logps/rejected": -400.0,
+      "loss": 0.1375,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -14.75,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -22.375,
+      "step": 4760
+    },
+    {
+      "epoch": 0.6096236181225637,
+      "grad_norm": 3.8682784456164043,
+      "learning_rate": 1.1583259807015182e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -362.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1076,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -22.75,
+      "step": 4770
+    },
+    {
+      "epoch": 0.6109016550578311,
+      "grad_norm": 6.259411555819717,
+      "learning_rate": 1.1571137082807434e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -1.0546875,
+      "logps/chosen": -372.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1198,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.25,
+      "rewards/margins": 7.09375,
+      "rewards/rejected": -22.375,
+      "step": 4780
+    },
+    {
+      "epoch": 0.6121796919930986,
+      "grad_norm": 4.7119463547404985,
+      "learning_rate": 1.155905234104269e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.79296875,
+      "logps/chosen": -360.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1502,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -15.0625,
+      "rewards/margins": 7.78125,
+      "rewards/rejected": -22.875,
+      "step": 4790
+    },
+    {
+      "epoch": 0.613457728928366,
+      "grad_norm": 8.436949916773173,
+      "learning_rate": 1.1547005383792516e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.87890625,
+      "logps/chosen": -358.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1072,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.5,
+      "rewards/margins": 7.34375,
+      "rewards/rejected": -21.875,
+      "step": 4800
+    },
+    {
+      "epoch": 0.6147357658636334,
+      "grad_norm": 9.05724145481619,
+      "learning_rate": 1.1534996014569446e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -348.0,
+      "logps/rejected": -400.0,
+      "loss": 0.1371,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -14.3125,
+      "rewards/margins": 7.25,
+      "rewards/rejected": -21.625,
+      "step": 4810
+    },
+    {
+      "epoch": 0.6160138027989008,
+      "grad_norm": 6.680898558403465,
+      "learning_rate": 1.1523024038313547e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -350.0,
+      "logps/rejected": -404.0,
+      "loss": 0.0981,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -13.8125,
+      "rewards/margins": 8.4375,
+      "rewards/rejected": -22.25,
+      "step": 4820
+    },
+    {
+      "epoch": 0.6172918397341683,
+      "grad_norm": 5.196546719881567,
+      "learning_rate": 1.1511089261379083e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.96484375,
+      "logps/chosen": -364.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1348,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -14.125,
+      "rewards/margins": 8.3125,
+      "rewards/rejected": -22.375,
+      "step": 4830
+    },
+    {
+      "epoch": 0.6185698766694357,
+      "grad_norm": 6.819986977247515,
+      "learning_rate": 1.149919149152138e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -364.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1416,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -14.0625,
+      "rewards/margins": 7.28125,
+      "rewards/rejected": -21.25,
+      "step": 4840
+    },
+    {
+      "epoch": 0.6198479136047031,
+      "grad_norm": 4.471205884622944,
+      "learning_rate": 1.148733053788381e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -356.0,
+      "logps/rejected": -402.0,
+      "loss": 0.1315,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -14.4375,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -22.0,
+      "step": 4850
+    },
+    {
+      "epoch": 0.6211259505399706,
+      "grad_norm": 7.629937630238964,
+      "learning_rate": 1.1475506210984938e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.859375,
+      "logps/chosen": -350.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1427,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.3125,
+      "rewards/margins": 7.53125,
+      "rewards/rejected": -21.75,
+      "step": 4860
+    },
+    {
+      "epoch": 0.6224039874752381,
+      "grad_norm": 6.432729934577347,
+      "learning_rate": 1.1463718322705807e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -362.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1067,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -14.25,
+      "rewards/margins": 7.71875,
+      "rewards/rejected": -22.0,
+      "step": 4870
+    },
+    {
+      "epoch": 0.6236820244105055,
+      "grad_norm": 4.387614046775704,
+      "learning_rate": 1.1451966686277364e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -356.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1256,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -14.9375,
+      "rewards/margins": 6.9375,
+      "rewards/rejected": -21.875,
+      "step": 4880
+    },
+    {
+      "epoch": 0.6249600613457729,
+      "grad_norm": 5.4356875039741945,
+      "learning_rate": 1.1440251116268034e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -382.0,
+      "logps/rejected": -434.0,
+      "loss": 0.146,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.4375,
+      "rewards/margins": 7.40625,
+      "rewards/rejected": -22.875,
+      "step": 4890
+    },
+    {
+      "epoch": 0.6262380982810403,
+      "grad_norm": 6.3188172784081535,
+      "learning_rate": 1.1428571428571428e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -374.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1038,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 7.34375,
+      "rewards/rejected": -23.25,
+      "step": 4900
+    },
+    {
+      "epoch": 0.6275161352163078,
+      "grad_norm": 5.818348906644234,
+      "learning_rate": 1.14169274403942e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.84765625,
+      "logps/chosen": -350.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1197,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.0625,
+      "rewards/margins": 7.53125,
+      "rewards/rejected": -22.5,
+      "step": 4910
+    },
+    {
+      "epoch": 0.6287941721515752,
+      "grad_norm": 25.767775517162317,
+      "learning_rate": 1.140531897024402e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -362.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1151,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.0,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -22.375,
+      "step": 4920
+    },
+    {
+      "epoch": 0.6300722090868426,
+      "grad_norm": 4.286045883976597,
+      "learning_rate": 1.13937458379177e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.95703125,
+      "logps/chosen": -356.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1218,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.25,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -21.875,
+      "step": 4930
+    },
+    {
+      "epoch": 0.63135024602211,
+      "grad_norm": 6.91100706405828,
+      "learning_rate": 1.1382207864489444e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -378.0,
+      "logps/rejected": -404.0,
+      "loss": 0.1236,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -14.875,
+      "rewards/margins": 6.8125,
+      "rewards/rejected": -21.75,
+      "step": 4940
+    },
+    {
+      "epoch": 0.6326282829573775,
+      "grad_norm": 10.542327371620994,
+      "learning_rate": 1.1370704872299223e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -364.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1075,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -14.6875,
+      "rewards/margins": 6.75,
+      "rewards/rejected": -21.5,
+      "step": 4950
+    },
+    {
+      "epoch": 0.6339063198926449,
+      "grad_norm": 5.923278643383796,
+      "learning_rate": 1.1359236684941295e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -372.0,
+      "logps/rejected": -416.0,
+      "loss": 0.1345,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -15.4375,
+      "rewards/margins": 7.28125,
+      "rewards/rejected": -22.75,
+      "step": 4960
+    },
+    {
+      "epoch": 0.6351843568279123,
+      "grad_norm": 7.9590466989774225,
+      "learning_rate": 1.1347803127252839e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -374.0,
+      "logps/rejected": -422.0,
+      "loss": 0.0909,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -15.0,
+      "rewards/margins": 7.25,
+      "rewards/rejected": -22.25,
+      "step": 4970
+    },
+    {
+      "epoch": 0.6364623937631797,
+      "grad_norm": 8.84619599705904,
+      "learning_rate": 1.1336404025302715e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.875,
+      "logps/chosen": -368.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1417,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.5625,
+      "rewards/margins": 7.5625,
+      "rewards/rejected": -23.125,
+      "step": 4980
+    },
+    {
+      "epoch": 0.6377404306984472,
+      "grad_norm": 10.580305156452308,
+      "learning_rate": 1.1325039206380352e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.92578125,
+      "logps/chosen": -374.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1636,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 7.0,
+      "rewards/rejected": -22.875,
+      "step": 4990
+    },
+    {
+      "epoch": 0.6390184676337146,
+      "grad_norm": 4.708608372250035,
+      "learning_rate": 1.131370849898476e-06,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -392.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1208,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -24.25,
+      "step": 5000
+    },
+    {
+      "epoch": 0.6390184676337146,
+      "eval_logits/chosen": -1.3203125,
+      "eval_logits/rejected": -0.9140625,
+      "eval_logps/chosen": -376.0,
+      "eval_logps/rejected": -418.0,
+      "eval_loss": 0.23097696900367737,
+      "eval_rewards/accuracies": 0.90625,
+      "eval_rewards/chosen": -16.625,
+      "eval_rewards/margins": 5.9375,
+      "eval_rewards/rejected": -22.625,
+      "eval_runtime": 1571.0995,
+      "eval_samples_per_second": 35.329,
+      "eval_steps_per_second": 0.552,
+      "step": 5000
+    },
+    {
+      "epoch": 0.640296504568982,
+      "grad_norm": 11.295415796723384,
+      "learning_rate": 1.130241173281366e-06,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -374.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1151,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 6.875,
+      "rewards/rejected": -23.375,
+      "step": 5010
+    },
+    {
+      "epoch": 0.6415745415042494,
+      "grad_norm": 5.58698306592888,
+      "learning_rate": 1.1291148738752732e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.8359375,
+      "logps/chosen": -368.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1098,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 6.59375,
+      "rewards/rejected": -23.0,
+      "step": 5020
+    },
+    {
+      "epoch": 0.6428525784395169,
+      "grad_norm": 6.985826330429579,
+      "learning_rate": 1.1279919348864981e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -358.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1261,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -15.125,
+      "rewards/margins": 7.96875,
+      "rewards/rejected": -23.125,
+      "step": 5030
+    },
+    {
+      "epoch": 0.6441306153747843,
+      "grad_norm": 7.998915517709166,
+      "learning_rate": 1.126872339638022e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -366.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1172,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.75,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -23.125,
+      "step": 5040
+    },
+    {
+      "epoch": 0.6454086523100517,
+      "grad_norm": 7.23061507423466,
+      "learning_rate": 1.1257560715684668e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -362.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1076,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 7.1875,
+      "rewards/rejected": -23.375,
+      "step": 5050
+    },
+    {
+      "epoch": 0.6466866892453191,
+      "grad_norm": 15.020738613717494,
+      "learning_rate": 1.1246431142310665e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -378.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1417,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -15.8125,
+      "rewards/margins": 7.28125,
+      "rewards/rejected": -23.125,
+      "step": 5060
+    },
+    {
+      "epoch": 0.6479647261805866,
+      "grad_norm": 11.049831059182878,
+      "learning_rate": 1.1235334512926484e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -372.0,
+      "logps/rejected": -396.0,
+      "loss": 0.1146,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.3125,
+      "rewards/margins": 7.125,
+      "rewards/rejected": -22.375,
+      "step": 5070
+    },
+    {
+      "epoch": 0.649242763115854,
+      "grad_norm": 6.191616654960539,
+      "learning_rate": 1.1224270665326274e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -384.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1188,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.9375,
+      "rewards/margins": 7.6875,
+      "rewards/rejected": -23.625,
+      "step": 5080
+    },
+    {
+      "epoch": 0.6505208000511215,
+      "grad_norm": 8.619554833284234,
+      "learning_rate": 1.12132394384201e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.984375,
+      "logps/chosen": -386.0,
+      "logps/rejected": -422.0,
+      "loss": 0.1279,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.9375,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -23.5,
+      "step": 5090
+    },
+    {
+      "epoch": 0.6517988369863889,
+      "grad_norm": 6.601200973759248,
+      "learning_rate": 1.1202240672224076e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -378.0,
+      "logps/rejected": -444.0,
+      "loss": 0.0882,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -23.875,
+      "step": 5100
+    },
+    {
+      "epoch": 0.6530768739216564,
+      "grad_norm": 8.04096961653137,
+      "learning_rate": 1.1191274207850654e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -372.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1176,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 7.09375,
+      "rewards/rejected": -23.0,
+      "step": 5110
+    },
+    {
+      "epoch": 0.6543549108569238,
+      "grad_norm": 6.547883240166143,
+      "learning_rate": 1.1180339887498948e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.9296875,
+      "logps/chosen": -380.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1268,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.1875,
+      "rewards/rejected": -24.25,
+      "step": 5120
+    },
+    {
+      "epoch": 0.6556329477921912,
+      "grad_norm": 7.279893527286292,
+      "learning_rate": 1.1169437554445213e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -376.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1333,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 7.40625,
+      "rewards/rejected": -23.375,
+      "step": 5130
+    },
+    {
+      "epoch": 0.6569109847274586,
+      "grad_norm": 6.563085921516533,
+      "learning_rate": 1.1158567053033413e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -382.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1204,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 6.90625,
+      "rewards/rejected": -23.375,
+      "step": 5140
+    },
+    {
+      "epoch": 0.6581890216627261,
+      "grad_norm": 6.859710731776834,
+      "learning_rate": 1.1147728228665882e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -374.0,
+      "logps/rejected": -410.0,
+      "loss": 0.1196,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -23.875,
+      "step": 5150
+    },
+    {
+      "epoch": 0.6594670585979935,
+      "grad_norm": 5.7147355620170766,
+      "learning_rate": 1.1136920927794092e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -354.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1184,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -15.8125,
+      "rewards/margins": 7.75,
+      "rewards/rejected": -23.5,
+      "step": 5160
+    },
+    {
+      "epoch": 0.6607450955332609,
+      "grad_norm": 6.925617894721734,
+      "learning_rate": 1.1126144997909508e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -376.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1374,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.625,
+      "rewards/margins": 7.09375,
+      "rewards/rejected": -22.75,
+      "step": 5170
+    },
+    {
+      "epoch": 0.6620231324685283,
+      "grad_norm": 6.21872200788245,
+      "learning_rate": 1.1115400287534568e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -364.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1279,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -23.5,
+      "step": 5180
+    },
+    {
+      "epoch": 0.6633011694037958,
+      "grad_norm": 9.813222338138841,
+      "learning_rate": 1.110468664621372e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -368.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1377,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 7.78125,
+      "rewards/rejected": -23.875,
+      "step": 5190
+    },
+    {
+      "epoch": 0.6645792063390632,
+      "grad_norm": 8.519972095383148,
+      "learning_rate": 1.1094003924504583e-06,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -0.98828125,
+      "logps/chosen": -362.0,
+      "logps/rejected": -432.0,
+      "loss": 0.0949,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -15.5,
+      "rewards/margins": 8.3125,
+      "rewards/rejected": -23.75,
+      "step": 5200
+    },
+    {
+      "epoch": 0.6658572432743306,
+      "grad_norm": 10.496434044935995,
+      "learning_rate": 1.1083351973969191e-06,
+      "logits/chosen": -1.453125,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -380.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1542,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -15.75,
+      "rewards/margins": 8.0625,
+      "rewards/rejected": -23.75,
+      "step": 5210
+    },
+    {
+      "epoch": 0.667135280209598,
+      "grad_norm": 6.05816951251384,
+      "learning_rate": 1.107273064716533e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -368.0,
+      "logps/rejected": -418.0,
+      "loss": 0.1214,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 6.8125,
+      "rewards/rejected": -23.375,
+      "step": 5220
+    },
+    {
+      "epoch": 0.6684133171448655,
+      "grad_norm": 3.974125254534399,
+      "learning_rate": 1.1062139797637962e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -380.0,
+      "logps/rejected": -434.0,
+      "loss": 0.0913,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -15.5625,
+      "rewards/margins": 7.875,
+      "rewards/rejected": -23.375,
+      "step": 5230
+    },
+    {
+      "epoch": 0.6696913540801329,
+      "grad_norm": 6.58222741661903,
+      "learning_rate": 1.1051579279910751e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.84765625,
+      "logps/chosen": -378.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1519,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 6.875,
+      "rewards/rejected": -23.875,
+      "step": 5240
+    },
+    {
+      "epoch": 0.6709693910154003,
+      "grad_norm": 9.68820555281317,
+      "learning_rate": 1.1041048949477667e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -378.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1156,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 6.6875,
+      "rewards/rejected": -24.25,
+      "step": 5250
+    },
+    {
+      "epoch": 0.6722474279506677,
+      "grad_norm": 14.316233055868132,
+      "learning_rate": 1.1030548662794673e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.953125,
+      "logps/chosen": -414.0,
+      "logps/rejected": -464.0,
+      "loss": 0.1079,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -25.0,
+      "step": 5260
+    },
+    {
+      "epoch": 0.6735254648859352,
+      "grad_norm": 9.528844999393858,
+      "learning_rate": 1.102007827727152e-06,
+      "logits/chosen": -1.46875,
+      "logits/rejected": -0.953125,
+      "logps/chosen": -376.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1257,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.46875,
+      "rewards/rejected": -24.5,
+      "step": 5270
+    },
+    {
+      "epoch": 0.6748035018212026,
+      "grad_norm": 2.9013715805789775,
+      "learning_rate": 1.1009637651263607e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -392.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1408,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.65625,
+      "rewards/rejected": -24.625,
+      "step": 5280
+    },
+    {
+      "epoch": 0.67608153875647,
+      "grad_norm": 6.578568151417815,
+      "learning_rate": 1.0999226644063927e-06,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1151,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 7.21875,
+      "rewards/rejected": -24.125,
+      "step": 5290
+    },
+    {
+      "epoch": 0.6773595756917374,
+      "grad_norm": 5.631355276407825,
+      "learning_rate": 1.0988845115895123e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -376.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1116,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 6.9375,
+      "rewards/rejected": -24.125,
+      "step": 5300
+    },
+    {
+      "epoch": 0.678637612627005,
+      "grad_norm": 7.096796599512148,
+      "learning_rate": 1.0978492927901574e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.87890625,
+      "logps/chosen": -378.0,
+      "logps/rejected": -428.0,
+      "loss": 0.0822,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.84375,
+      "rewards/rejected": -24.0,
+      "step": 5310
+    },
+    {
+      "epoch": 0.6799156495622724,
+      "grad_norm": 5.907027178745851,
+      "learning_rate": 1.0968169942141634e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.953125,
+      "logps/chosen": -382.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1167,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.71875,
+      "rewards/rejected": -24.375,
+      "step": 5320
+    },
+    {
+      "epoch": 0.6811936864975398,
+      "grad_norm": 8.1842125539458,
+      "learning_rate": 1.0957876021579874e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -382.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1159,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 8.75,
+      "rewards/rejected": -25.25,
+      "step": 5330
+    },
+    {
+      "epoch": 0.6824717234328072,
+      "grad_norm": 8.885534263166706,
+      "learning_rate": 1.0947611030079466e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.96484375,
+      "logps/chosen": -384.0,
+      "logps/rejected": -442.0,
+      "loss": 0.1002,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 8.1875,
+      "rewards/rejected": -25.25,
+      "step": 5340
+    },
+    {
+      "epoch": 0.6837497603680747,
+      "grad_norm": 6.456655882208527,
+      "learning_rate": 1.0937374832394612e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -364.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1052,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 8.125,
+      "rewards/rejected": -24.75,
+      "step": 5350
+    },
+    {
+      "epoch": 0.6850277973033421,
+      "grad_norm": 8.092204386399079,
+      "learning_rate": 1.092716729416306e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.0,
+      "logps/chosen": -412.0,
+      "logps/rejected": -464.0,
+      "loss": 0.1357,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 8.0,
+      "rewards/rejected": -25.5,
+      "step": 5360
+    },
+    {
+      "epoch": 0.6863058342386095,
+      "grad_norm": 7.469176385500727,
+      "learning_rate": 1.0916988281898703e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.95703125,
+      "logps/chosen": -384.0,
+      "logps/rejected": -450.0,
+      "loss": 0.1253,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 7.125,
+      "rewards/rejected": -24.875,
+      "step": 5370
+    },
+    {
+      "epoch": 0.6875838711738769,
+      "grad_norm": 5.371777934139812,
+      "learning_rate": 1.0906837662984237e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -390.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1155,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 7.0625,
+      "rewards/rejected": -25.125,
+      "step": 5380
+    },
+    {
+      "epoch": 0.6888619081091444,
+      "grad_norm": 7.895283266618486,
+      "learning_rate": 1.089671530566391e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.99609375,
+      "logps/chosen": -388.0,
+      "logps/rejected": -452.0,
+      "loss": 0.1223,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 6.78125,
+      "rewards/rejected": -23.5,
+      "step": 5390
+    },
+    {
+      "epoch": 0.6901399450444118,
+      "grad_norm": 5.272539569617998,
+      "learning_rate": 1.0886621079036346e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -396.0,
+      "logps/rejected": -434.0,
+      "loss": 0.0868,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -15.875,
+      "rewards/margins": 8.875,
+      "rewards/rejected": -24.75,
+      "step": 5400
+    },
+    {
+      "epoch": 0.6914179819796792,
+      "grad_norm": 3.304606353811427,
+      "learning_rate": 1.0876554853047417e-06,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -386.0,
+      "logps/rejected": -432.0,
+      "loss": 0.0837,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 8.8125,
+      "rewards/rejected": -25.375,
+      "step": 5410
+    },
+    {
+      "epoch": 0.6926960189149466,
+      "grad_norm": 5.647835224393562,
+      "learning_rate": 1.0866516498483225e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -382.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1249,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -24.5,
+      "step": 5420
+    },
+    {
+      "epoch": 0.6939740558502141,
+      "grad_norm": 7.128974950465777,
+      "learning_rate": 1.0856505886963116e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -378.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1209,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 8.0,
+      "rewards/rejected": -24.5,
+      "step": 5430
+    },
+    {
+      "epoch": 0.6952520927854815,
+      "grad_norm": 7.706901220915576,
+      "learning_rate": 1.0846522890932808e-06,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -390.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1293,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.9375,
+      "rewards/rejected": -24.625,
+      "step": 5440
+    },
+    {
+      "epoch": 0.6965301297207489,
+      "grad_norm": 3.3019002227821557,
+      "learning_rate": 1.0836567383657542e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -370.0,
+      "logps/rejected": -442.0,
+      "loss": 0.0946,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 7.84375,
+      "rewards/rejected": -25.0,
+      "step": 5450
+    },
+    {
+      "epoch": 0.6978081666560163,
+      "grad_norm": 4.5131885166339085,
+      "learning_rate": 1.0826639239215334e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -376.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1058,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 7.9375,
+      "rewards/rejected": -25.0,
+      "step": 5460
+    },
+    {
+      "epoch": 0.6990862035912838,
+      "grad_norm": 6.4653396564489585,
+      "learning_rate": 1.0816738332490292e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -376.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1164,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 8.375,
+      "rewards/rejected": -25.25,
+      "step": 5470
+    },
+    {
+      "epoch": 0.7003642405265512,
+      "grad_norm": 10.433476743828498,
+      "learning_rate": 1.0806864539165982e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.73828125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1322,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 8.375,
+      "rewards/rejected": -25.75,
+      "step": 5480
+    },
+    {
+      "epoch": 0.7016422774618186,
+      "grad_norm": 9.913307099823063,
+      "learning_rate": 1.0797017735718878e-06,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -0.95703125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -436.0,
+      "loss": 0.121,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 7.78125,
+      "rewards/rejected": -25.5,
+      "step": 5490
+    },
+    {
+      "epoch": 0.702920314397086,
+      "grad_norm": 8.841806642146528,
+      "learning_rate": 1.0787197799411874e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.87890625,
+      "logps/chosen": -394.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1283,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -18.375,
+      "rewards/margins": 6.9375,
+      "rewards/rejected": -25.375,
+      "step": 5500
+    },
+    {
+      "epoch": 0.7041983513323535,
+      "grad_norm": 6.105609254510579,
+      "learning_rate": 1.0777404608287846e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.84765625,
+      "logps/chosen": -370.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1083,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -18.25,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -25.875,
+      "step": 5510
+    },
+    {
+      "epoch": 0.7054763882676209,
+      "grad_norm": 7.3548973031669025,
+      "learning_rate": 1.0767638041163309e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -390.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1039,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 8.5625,
+      "rewards/rejected": -25.875,
+      "step": 5520
+    },
+    {
+      "epoch": 0.7067544252028883,
+      "grad_norm": 5.420618941574323,
+      "learning_rate": 1.0757897977622107e-06,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -458.0,
+      "loss": 0.1013,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -24.5,
+      "step": 5530
+    },
+    {
+      "epoch": 0.7080324621381557,
+      "grad_norm": 6.487177821526908,
+      "learning_rate": 1.074818429800918e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -1.0,
+      "logps/chosen": -370.0,
+      "logps/rejected": -438.0,
+      "loss": 0.105,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.84375,
+      "rewards/rejected": -24.5,
+      "step": 5540
+    },
+    {
+      "epoch": 0.7093104990734233,
+      "grad_norm": 7.105523315521007,
+      "learning_rate": 1.073849688342439e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.8359375,
+      "logps/chosen": -370.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1091,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 8.0625,
+      "rewards/rejected": -24.25,
+      "step": 5550
+    },
+    {
+      "epoch": 0.7105885360086907,
+      "grad_norm": 3.6179510705771674,
+      "learning_rate": 1.07288356157164e-06,
+      "logits/chosen": -1.1484375,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -396.0,
+      "logps/rejected": -434.0,
+      "loss": 0.0962,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 7.4375,
+      "rewards/rejected": -24.625,
+      "step": 5560
+    },
+    {
+      "epoch": 0.711866572943958,
+      "grad_norm": 5.666226764292592,
+      "learning_rate": 1.0719200377476648e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -390.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1197,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 8.5,
+      "rewards/rejected": -25.375,
+      "step": 5570
+    },
+    {
+      "epoch": 0.7131446098792255,
+      "grad_norm": 5.299207994641014,
+      "learning_rate": 1.0709591052033317e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -366.0,
+      "logps/rejected": -460.0,
+      "loss": 0.1071,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.65625,
+      "rewards/rejected": -24.75,
+      "step": 5580
+    },
+    {
+      "epoch": 0.714422646814493,
+      "grad_norm": 8.721747641874673,
+      "learning_rate": 1.0700007523445435e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -402.0,
+      "logps/rejected": -438.0,
+      "loss": 0.0945,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 8.125,
+      "rewards/rejected": -25.375,
+      "step": 5590
+    },
+    {
+      "epoch": 0.7157006837497604,
+      "grad_norm": 50.55739435455809,
+      "learning_rate": 1.0690449676496976e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -378.0,
+      "logps/rejected": -436.0,
+      "loss": 0.104,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 8.4375,
+      "rewards/rejected": -24.625,
+      "step": 5600
+    },
+    {
+      "epoch": 0.7169787206850278,
+      "grad_norm": 4.796417257282072,
+      "learning_rate": 1.0680917396691054e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -1.0,
+      "logps/chosen": -396.0,
+      "logps/rejected": -458.0,
+      "loss": 0.1303,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.21875,
+      "rewards/rejected": -24.0,
+      "step": 5610
+    },
+    {
+      "epoch": 0.7182567576202952,
+      "grad_norm": 6.326979825762225,
+      "learning_rate": 1.0671410570244164e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -376.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1288,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 8.25,
+      "rewards/rejected": -24.625,
+      "step": 5620
+    },
+    {
+      "epoch": 0.7195347945555627,
+      "grad_norm": 7.7160357685515715,
+      "learning_rate": 1.0661929084080466e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -396.0,
+      "logps/rejected": -440.0,
+      "loss": 0.113,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 7.8125,
+      "rewards/rejected": -24.625,
+      "step": 5630
+    },
+    {
+      "epoch": 0.7208128314908301,
+      "grad_norm": 6.273850144744979,
+      "learning_rate": 1.0652472825826149e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -370.0,
+      "logps/rejected": -428.0,
+      "loss": 0.0895,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 7.6875,
+      "rewards/rejected": -23.75,
+      "step": 5640
+    },
+    {
+      "epoch": 0.7220908684260975,
+      "grad_norm": 11.690638765986426,
+      "learning_rate": 1.0643041683803828e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -396.0,
+      "logps/rejected": -442.0,
+      "loss": 0.1293,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 8.25,
+      "rewards/rejected": -25.25,
+      "step": 5650
+    },
+    {
+      "epoch": 0.7233689053613649,
+      "grad_norm": 10.561230828113331,
+      "learning_rate": 1.063363554702701e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.97265625,
+      "logps/chosen": -384.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1211,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -24.0,
+      "step": 5660
+    },
+    {
+      "epoch": 0.7246469422966324,
+      "grad_norm": 8.987744855590927,
+      "learning_rate": 1.0624254305194609e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -392.0,
+      "logps/rejected": -454.0,
+      "loss": 0.1102,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.5,
+      "rewards/margins": 8.3125,
+      "rewards/rejected": -23.875,
+      "step": 5670
+    },
+    {
+      "epoch": 0.7259249792318998,
+      "grad_norm": 5.031312237716629,
+      "learning_rate": 1.0614897848685505e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.859375,
+      "logps/chosen": -364.0,
+      "logps/rejected": -408.0,
+      "loss": 0.0859,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -14.625,
+      "rewards/margins": 7.71875,
+      "rewards/rejected": -22.375,
+      "step": 5680
+    },
+    {
+      "epoch": 0.7272030161671672,
+      "grad_norm": 4.723184939683974,
+      "learning_rate": 1.0605566068553173e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1126,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.90625,
+      "rewards/rejected": -24.0,
+      "step": 5690
+    },
+    {
+      "epoch": 0.7284810531024346,
+      "grad_norm": 3.9648053653403474,
+      "learning_rate": 1.0596258856520351e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.95703125,
+      "logps/chosen": -366.0,
+      "logps/rejected": -450.0,
+      "loss": 0.1502,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -15.375,
+      "rewards/margins": 8.4375,
+      "rewards/rejected": -23.75,
+      "step": 5700
+    },
+    {
+      "epoch": 0.7297590900377021,
+      "grad_norm": 7.36901264836233,
+      "learning_rate": 1.0586976104973764e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -386.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1197,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 8.0625,
+      "rewards/rejected": -24.125,
+      "step": 5710
+    },
+    {
+      "epoch": 0.7310371269729695,
+      "grad_norm": 9.86787624923407,
+      "learning_rate": 1.05777177069589e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -380.0,
+      "logps/rejected": -420.0,
+      "loss": 0.1191,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.4375,
+      "rewards/margins": 8.1875,
+      "rewards/rejected": -23.5,
+      "step": 5720
+    },
+    {
+      "epoch": 0.7323151639082369,
+      "grad_norm": 3.389720655674331,
+      "learning_rate": 1.0568483556174834e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -366.0,
+      "logps/rejected": -408.0,
+      "loss": 0.1235,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.8125,
+      "rewards/margins": 7.0625,
+      "rewards/rejected": -22.875,
+      "step": 5730
+    },
+    {
+      "epoch": 0.7335932008435043,
+      "grad_norm": 5.791540135325104,
+      "learning_rate": 1.055927354696909e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -366.0,
+      "logps/rejected": -414.0,
+      "loss": 0.1109,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.0625,
+      "rewards/rejected": -23.125,
+      "step": 5740
+    },
+    {
+      "epoch": 0.7348712377787718,
+      "grad_norm": 9.795803359532263,
+      "learning_rate": 1.0550087574332592e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -370.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1167,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.40625,
+      "rewards/rejected": -24.375,
+      "step": 5750
+    },
+    {
+      "epoch": 0.7361492747140392,
+      "grad_norm": 10.117993795896517,
+      "learning_rate": 1.0540925533894598e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -368.0,
+      "logps/rejected": -424.0,
+      "loss": 0.0988,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -15.4375,
+      "rewards/margins": 8.0625,
+      "rewards/rejected": -23.5,
+      "step": 5760
+    },
+    {
+      "epoch": 0.7374273116493066,
+      "grad_norm": 3.026313941959101,
+      "learning_rate": 1.053178732191775e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -362.0,
+      "logps/rejected": -418.0,
+      "loss": 0.0762,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.125,
+      "rewards/margins": 7.8125,
+      "rewards/rejected": -24.0,
+      "step": 5770
+    },
+    {
+      "epoch": 0.738705348584574,
+      "grad_norm": 6.059520734126738,
+      "learning_rate": 1.0522672835293127e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.80859375,
+      "logps/chosen": -380.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1278,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 7.5625,
+      "rewards/rejected": -24.875,
+      "step": 5780
+    },
+    {
+      "epoch": 0.7399833855198416,
+      "grad_norm": 10.22248227729163,
+      "learning_rate": 1.0513581971535365e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -376.0,
+      "logps/rejected": -442.0,
+      "loss": 0.1136,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -24.625,
+      "step": 5790
+    },
+    {
+      "epoch": 0.741261422455109,
+      "grad_norm": 8.312734489380222,
+      "learning_rate": 1.0504514628777803e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -380.0,
+      "logps/rejected": -450.0,
+      "loss": 0.1385,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.65625,
+      "rewards/rejected": -24.625,
+      "step": 5800
+    },
+    {
+      "epoch": 0.7425394593903764,
+      "grad_norm": 7.083933263675265,
+      "learning_rate": 1.0495470705767713e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -394.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1079,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.65625,
+      "rewards/rejected": -24.25,
+      "step": 5810
+    },
+    {
+      "epoch": 0.7438174963256438,
+      "grad_norm": 9.51895098035189,
+      "learning_rate": 1.0486450101861527e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -386.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1215,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 7.25,
+      "rewards/rejected": -24.5,
+      "step": 5820
+    },
+    {
+      "epoch": 0.7450955332609113,
+      "grad_norm": 9.400990809143735,
+      "learning_rate": 1.0477452717020143e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.828125,
+      "logps/chosen": -378.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1271,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.21875,
+      "rewards/rejected": -23.875,
+      "step": 5830
+    },
+    {
+      "epoch": 0.7463735701961787,
+      "grad_norm": 8.24359325012189,
+      "learning_rate": 1.0468478451804272e-06,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.859375,
+      "logps/chosen": -388.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1347,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 8.0,
+      "rewards/rejected": -24.75,
+      "step": 5840
+    },
+    {
+      "epoch": 0.7476516071314461,
+      "grad_norm": 6.820197458880494,
+      "learning_rate": 1.0459527207369814e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.8359375,
+      "logps/chosen": -386.0,
+      "logps/rejected": -454.0,
+      "loss": 0.1138,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 8.5,
+      "rewards/rejected": -24.875,
+      "step": 5850
+    },
+    {
+      "epoch": 0.7489296440667135,
+      "grad_norm": 6.561309358334411,
+      "learning_rate": 1.0450598885463281e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.78515625,
+      "logps/chosen": -362.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1095,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 8.0,
+      "rewards/rejected": -24.5,
+      "step": 5860
+    },
+    {
+      "epoch": 0.750207681001981,
+      "grad_norm": 10.929110536897733,
+      "learning_rate": 1.0441693388417282e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -386.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1157,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 7.84375,
+      "rewards/rejected": -24.875,
+      "step": 5870
+    },
+    {
+      "epoch": 0.7514857179372484,
+      "grad_norm": 6.2394999166233935,
+      "learning_rate": 1.0432810619146023e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -382.0,
+      "logps/rejected": -450.0,
+      "loss": 0.1514,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -25.0,
+      "step": 5880
+    },
+    {
+      "epoch": 0.7527637548725158,
+      "grad_norm": 7.915541168351391,
+      "learning_rate": 1.042395048114086e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -400.0,
+      "logps/rejected": -466.0,
+      "loss": 0.1188,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 7.3125,
+      "rewards/rejected": -24.375,
+      "step": 5890
+    },
+    {
+      "epoch": 0.7540417918077832,
+      "grad_norm": 6.570812357738413,
+      "learning_rate": 1.041511287846591e-06,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -386.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1084,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 7.53125,
+      "rewards/rejected": -25.125,
+      "step": 5900
+    },
+    {
+      "epoch": 0.7553198287430507,
+      "grad_norm": 8.216326669435428,
+      "learning_rate": 1.0406297715753674e-06,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.96484375,
+      "logps/chosen": -390.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1589,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 7.40625,
+      "rewards/rejected": -25.0,
+      "step": 5910
+    },
+    {
+      "epoch": 0.7565978656783181,
+      "grad_norm": 6.617681124925411,
+      "learning_rate": 1.0397504898200726e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -408.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1148,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 7.28125,
+      "rewards/rejected": -25.125,
+      "step": 5920
+    },
+    {
+      "epoch": 0.7578759026135855,
+      "grad_norm": 6.540742855435748,
+      "learning_rate": 1.0388734331563415e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -398.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1416,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 7.65625,
+      "rewards/rejected": -25.125,
+      "step": 5930
+    },
+    {
+      "epoch": 0.7591539395488529,
+      "grad_norm": 56.171606368970934,
+      "learning_rate": 1.037998592215364e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -410.0,
+      "logps/rejected": -454.0,
+      "loss": 0.1084,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 8.125,
+      "rewards/rejected": -25.625,
+      "step": 5940
+    },
+    {
+      "epoch": 0.7604319764841204,
+      "grad_norm": 4.556440020182315,
+      "learning_rate": 1.037125957683463e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -414.0,
+      "logps/rejected": -458.0,
+      "loss": 0.1227,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.875,
+      "rewards/rejected": -24.5,
+      "step": 5950
+    },
+    {
+      "epoch": 0.7617100134193878,
+      "grad_norm": 7.836444013217722,
+      "learning_rate": 1.0362555203016794e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -374.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1152,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 7.21875,
+      "rewards/rejected": -24.5,
+      "step": 5960
+    },
+    {
+      "epoch": 0.7629880503546552,
+      "grad_norm": 6.263694099353002,
+      "learning_rate": 1.035387270865359e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -396.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1316,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 8.0625,
+      "rewards/rejected": -25.375,
+      "step": 5970
+    },
+    {
+      "epoch": 0.7642660872899226,
+      "grad_norm": 6.411629605076929,
+      "learning_rate": 1.0345212002237434e-06,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -414.0,
+      "logps/rejected": -424.0,
+      "loss": 0.1202,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 6.46875,
+      "rewards/rejected": -23.875,
+      "step": 5980
+    },
+    {
+      "epoch": 0.7655441242251901,
+      "grad_norm": 5.880477189812311,
+      "learning_rate": 1.0336572992795644e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1034,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 8.375,
+      "rewards/rejected": -24.875,
+      "step": 5990
+    },
+    {
+      "epoch": 0.7668221611604575,
+      "grad_norm": 15.317178390366484,
+      "learning_rate": 1.0327955589886444e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.97265625,
+      "logps/chosen": -382.0,
+      "logps/rejected": -440.0,
+      "loss": 0.0933,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -24.75,
+      "step": 6000
+    },
+    {
+      "epoch": 0.7681001980957249,
+      "grad_norm": 7.3123446922387,
+      "learning_rate": 1.0319359703594971e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -388.0,
+      "logps/rejected": -438.0,
+      "loss": 0.0898,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 8.0625,
+      "rewards/rejected": -24.75,
+      "step": 6010
+    },
+    {
+      "epoch": 0.7693782350309923,
+      "grad_norm": 5.911182153775638,
+      "learning_rate": 1.0310785244529341e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.79296875,
+      "logps/chosen": -396.0,
+      "logps/rejected": -490.0,
+      "loss": 0.0807,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 8.625,
+      "rewards/rejected": -25.875,
+      "step": 6020
+    },
+    {
+      "epoch": 0.7706562719662599,
+      "grad_norm": 7.625268676276166,
+      "learning_rate": 1.0302232123816746e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.76953125,
+      "logps/chosen": -396.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1171,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 8.25,
+      "rewards/rejected": -26.25,
+      "step": 6030
+    },
+    {
+      "epoch": 0.7719343089015273,
+      "grad_norm": 6.662168171405007,
+      "learning_rate": 1.0293700253099576e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -376.0,
+      "logps/rejected": -432.0,
+      "loss": 0.0952,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 7.71875,
+      "rewards/rejected": -25.0,
+      "step": 6040
+    },
+    {
+      "epoch": 0.7732123458367947,
+      "grad_norm": 5.934641166586485,
+      "learning_rate": 1.02851895445316e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.859375,
+      "logps/chosen": -384.0,
+      "logps/rejected": -436.0,
+      "loss": 0.114,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 8.5,
+      "rewards/rejected": -25.625,
+      "step": 6050
+    },
+    {
+      "epoch": 0.7744903827720621,
+      "grad_norm": 6.554702045755405,
+      "learning_rate": 1.0276699910774159e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.7109375,
+      "logps/chosen": -398.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1058,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 8.6875,
+      "rewards/rejected": -25.625,
+      "step": 6060
+    },
+    {
+      "epoch": 0.7757684197073296,
+      "grad_norm": 6.566879151599004,
+      "learning_rate": 1.0268231264992398e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -372.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1421,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 7.65625,
+      "rewards/rejected": -24.75,
+      "step": 6070
+    },
+    {
+      "epoch": 0.777046456642597,
+      "grad_norm": 4.8241542110180955,
+      "learning_rate": 1.0259783520851542e-06,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -366.0,
+      "logps/rejected": -462.0,
+      "loss": 0.1079,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 9.25,
+      "rewards/rejected": -26.625,
+      "step": 6080
+    },
+    {
+      "epoch": 0.7783244935778644,
+      "grad_norm": 8.768909128360372,
+      "learning_rate": 1.0251356592513193e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.765625,
+      "logps/chosen": -398.0,
+      "logps/rejected": -438.0,
+      "loss": 0.0979,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 7.71875,
+      "rewards/rejected": -25.875,
+      "step": 6090
+    },
+    {
+      "epoch": 0.7796025305131318,
+      "grad_norm": 6.25007728023878,
+      "learning_rate": 1.0242950394631678e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.8359375,
+      "logps/chosen": -388.0,
+      "logps/rejected": -446.0,
+      "loss": 0.0863,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -25.5,
+      "step": 6100
+    },
+    {
+      "epoch": 0.7808805674483993,
+      "grad_norm": 11.49472101567702,
+      "learning_rate": 1.0234564842350404e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.78515625,
+      "logps/chosen": -380.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1046,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -25.25,
+      "step": 6110
+    },
+    {
+      "epoch": 0.7821586043836667,
+      "grad_norm": 6.636906398326096,
+      "learning_rate": 1.0226199851298272e-06,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -0.984375,
+      "logps/chosen": -388.0,
+      "logps/rejected": -450.0,
+      "loss": 0.1,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 8.125,
+      "rewards/rejected": -25.875,
+      "step": 6120
+    },
+    {
+      "epoch": 0.7834366413189341,
+      "grad_norm": 6.387151611650852,
+      "learning_rate": 1.0217855337586106e-06,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -0.8359375,
+      "logps/chosen": -400.0,
+      "logps/rejected": -462.0,
+      "loss": 0.0944,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 8.3125,
+      "rewards/rejected": -26.375,
+      "step": 6130
+    },
+    {
+      "epoch": 0.7847146782542015,
+      "grad_norm": 6.515939124091001,
+      "learning_rate": 1.0209531217803119e-06,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -404.0,
+      "logps/rejected": -462.0,
+      "loss": 0.0951,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 8.375,
+      "rewards/rejected": -26.5,
+      "step": 6140
+    },
+    {
+      "epoch": 0.785992715189469,
+      "grad_norm": 6.620623062895191,
+      "learning_rate": 1.0201227409013412e-06,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -388.0,
+      "logps/rejected": -472.0,
+      "loss": 0.1038,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 9.75,
+      "rewards/rejected": -27.375,
+      "step": 6150
+    },
+    {
+      "epoch": 0.7872707521247364,
+      "grad_norm": 5.932555699905493,
+      "learning_rate": 1.0192943828752509e-06,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.76953125,
+      "logps/chosen": -396.0,
+      "logps/rejected": -458.0,
+      "loss": 0.0998,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 9.125,
+      "rewards/rejected": -26.125,
+      "step": 6160
+    },
+    {
+      "epoch": 0.7885487890600038,
+      "grad_norm": 7.5665678213358225,
+      "learning_rate": 1.0184680395023912e-06,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -400.0,
+      "logps/rejected": -442.0,
+      "loss": 0.1147,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 8.3125,
+      "rewards/rejected": -25.25,
+      "step": 6170
+    },
+    {
+      "epoch": 0.7898268259952712,
+      "grad_norm": 5.213213976307602,
+      "learning_rate": 1.0176437026295688e-06,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.79296875,
+      "logps/chosen": -382.0,
+      "logps/rejected": -472.0,
+      "loss": 0.106,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 8.9375,
+      "rewards/rejected": -26.25,
+      "step": 6180
+    },
+    {
+      "epoch": 0.7911048629305387,
+      "grad_norm": 6.400163689909287,
+      "learning_rate": 1.0168213641497094e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.79296875,
+      "logps/chosen": -386.0,
+      "logps/rejected": -430.0,
+      "loss": 0.1077,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 8.625,
+      "rewards/rejected": -25.625,
+      "step": 6190
+    },
+    {
+      "epoch": 0.7923828998658061,
+      "grad_norm": 7.1809188619548046,
+      "learning_rate": 1.016001016001524e-06,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -398.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1124,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -25.375,
+      "step": 6200
+    },
+    {
+      "epoch": 0.7936609368010735,
+      "grad_norm": 9.644791495295987,
+      "learning_rate": 1.0151826501691747e-06,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -404.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1592,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -18.75,
+      "rewards/margins": 7.46875,
+      "rewards/rejected": -26.125,
+      "step": 6210
+    },
+    {
+      "epoch": 0.7949389737363409,
+      "grad_norm": 8.810395258458886,
+      "learning_rate": 1.0143662586819475e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -398.0,
+      "logps/rejected": -454.0,
+      "loss": 0.1206,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -18.25,
+      "rewards/margins": 7.53125,
+      "rewards/rejected": -25.75,
+      "step": 6220
+    },
+    {
+      "epoch": 0.7962170106716084,
+      "grad_norm": 4.8938088021872534,
+      "learning_rate": 1.0135518336139257e-06,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -394.0,
+      "logps/rejected": -442.0,
+      "loss": 0.114,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -18.625,
+      "rewards/margins": 6.78125,
+      "rewards/rejected": -25.375,
+      "step": 6230
+    },
+    {
+      "epoch": 0.7974950476068758,
+      "grad_norm": 6.076192040462074,
+      "learning_rate": 1.0127393670836666e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.84765625,
+      "logps/chosen": -398.0,
+      "logps/rejected": -450.0,
+      "loss": 0.1142,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -18.75,
+      "rewards/margins": 6.6875,
+      "rewards/rejected": -25.375,
+      "step": 6240
+    },
+    {
+      "epoch": 0.7987730845421432,
+      "grad_norm": 7.01949367774137,
+      "learning_rate": 1.0119288512538813e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.921875,
+      "logps/chosen": -396.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1448,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -18.875,
+      "rewards/margins": 6.9375,
+      "rewards/rejected": -25.75,
+      "step": 6250
+    },
+    {
+      "epoch": 0.8000511214774106,
+      "grad_norm": 5.9552982229514475,
+      "learning_rate": 1.0111202783311173e-06,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -410.0,
+      "logps/rejected": -452.0,
+      "loss": 0.0978,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -18.75,
+      "rewards/margins": 7.21875,
+      "rewards/rejected": -25.875,
+      "step": 6260
+    },
+    {
+      "epoch": 0.8013291584126782,
+      "grad_norm": 5.737453859649995,
+      "learning_rate": 1.010313640565443e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.80859375,
+      "logps/chosen": -392.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1296,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 7.09375,
+      "rewards/rejected": -24.875,
+      "step": 6270
+    },
+    {
+      "epoch": 0.8026071953479456,
+      "grad_norm": 5.069249650401188,
+      "learning_rate": 1.0095089302501373e-06,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -408.0,
+      "logps/rejected": -464.0,
+      "loss": 0.1221,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 7.84375,
+      "rewards/rejected": -25.5,
+      "step": 6280
+    },
+    {
+      "epoch": 0.803885232283213,
+      "grad_norm": 7.831264585211203,
+      "learning_rate": 1.0087061397213787e-06,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -0.7734375,
+      "logps/chosen": -388.0,
+      "logps/rejected": -446.0,
+      "loss": 0.114,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 7.9375,
+      "rewards/rejected": -25.25,
+      "step": 6290
+    },
+    {
+      "epoch": 0.8051632692184804,
+      "grad_norm": 7.748246644793888,
+      "learning_rate": 1.007905261357939e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.796875,
+      "logps/chosen": -362.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1005,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 8.6875,
+      "rewards/rejected": -25.75,
+      "step": 6300
+    },
+    {
+      "epoch": 0.8064413061537479,
+      "grad_norm": 6.383128487380736,
+      "learning_rate": 1.0071062875808811e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -456.0,
+      "loss": 0.1032,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 7.875,
+      "rewards/rejected": -26.0,
+      "step": 6310
+    },
+    {
+      "epoch": 0.8077193430890153,
+      "grad_norm": 4.853585559480419,
+      "learning_rate": 1.0063092108532552e-06,
+      "logits/chosen": -1.171875,
+      "logits/rejected": -0.75390625,
+      "logps/chosen": -402.0,
+      "logps/rejected": -450.0,
+      "loss": 0.1171,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 7.25,
+      "rewards/rejected": -25.25,
+      "step": 6320
+    },
+    {
+      "epoch": 0.8089973800242827,
+      "grad_norm": 6.681983791716586,
+      "learning_rate": 1.005514023679802e-06,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -392.0,
+      "logps/rejected": -452.0,
+      "loss": 0.1188,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 7.875,
+      "rewards/rejected": -25.625,
+      "step": 6330
+    },
+    {
+      "epoch": 0.8102754169595501,
+      "grad_norm": 6.501056012492706,
+      "learning_rate": 1.0047207186066567e-06,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -0.92578125,
+      "logps/chosen": -402.0,
+      "logps/rejected": -452.0,
+      "loss": 0.1045,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -18.375,
+      "rewards/margins": 8.1875,
+      "rewards/rejected": -26.5,
+      "step": 6340
+    },
+    {
+      "epoch": 0.8115534538948176,
+      "grad_norm": 8.410634525458537,
+      "learning_rate": 1.0039292882210538e-06,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -408.0,
+      "logps/rejected": -458.0,
+      "loss": 0.1139,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -18.25,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -25.625,
+      "step": 6350
+    },
+    {
+      "epoch": 0.812831490830085,
+      "grad_norm": 11.053303023835603,
+      "learning_rate": 1.0031397251510382e-06,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -390.0,
+      "logps/rejected": -464.0,
+      "loss": 0.1076,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 8.125,
+      "rewards/rejected": -26.125,
+      "step": 6360
+    },
+    {
+      "epoch": 0.8141095277653524,
+      "grad_norm": 7.57616709694438,
+      "learning_rate": 1.0023520220651762e-06,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -442.0,
+      "loss": 0.084,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 7.71875,
+      "rewards/rejected": -24.875,
+      "step": 6370
+    },
+    {
+      "epoch": 0.8153875647006198,
+      "grad_norm": 6.021616680957046,
+      "learning_rate": 1.0015661716722687e-06,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -376.0,
+      "logps/rejected": -466.0,
+      "loss": 0.1038,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 8.3125,
+      "rewards/rejected": -25.875,
+      "step": 6380
+    },
+    {
+      "epoch": 0.8166656016358873,
+      "grad_norm": 4.789011229577684,
+      "learning_rate": 1.0007821667210687e-06,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.8828125,
+      "logps/chosen": -408.0,
+      "logps/rejected": -456.0,
+      "loss": 0.1144,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 8.75,
+      "rewards/rejected": -26.625,
+      "step": 6390
+    },
+    {
+      "epoch": 0.8179436385711547,
+      "grad_norm": 8.814748129550921,
+      "learning_rate": 1e-06,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -388.0,
+      "logps/rejected": -464.0,
+      "loss": 0.1257,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -18.875,
+      "rewards/margins": 7.40625,
+      "rewards/rejected": -26.25,
+      "step": 6400
+    },
+    {
+      "epoch": 0.8192216755064221,
+      "grad_norm": 6.549530685597409,
+      "learning_rate": 9.992196643368784e-07,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.953125,
+      "logps/chosen": -386.0,
+      "logps/rejected": -454.0,
+      "loss": 0.1203,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -18.375,
+      "rewards/margins": 8.5625,
+      "rewards/rejected": -27.0,
+      "step": 6410
+    },
+    {
+      "epoch": 0.8204997124416895,
+      "grad_norm": 5.784310434502454,
+      "learning_rate": 9.984411525986355e-07,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.87109375,
+      "logps/chosen": -398.0,
+      "logps/rejected": -476.0,
+      "loss": 0.0935,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 8.875,
+      "rewards/rejected": -26.75,
+      "step": 6420
+    },
+    {
+      "epoch": 0.821777749376957,
+      "grad_norm": 4.4312818446826725,
+      "learning_rate": 9.97664457691046e-07,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -394.0,
+      "logps/rejected": -478.0,
+      "loss": 0.0928,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -18.5,
+      "rewards/margins": 8.625,
+      "rewards/rejected": -27.125,
+      "step": 6430
+    },
+    {
+      "epoch": 0.8230557863122244,
+      "grad_norm": 6.311658229466089,
+      "learning_rate": 9.968895725584535e-07,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.828125,
+      "logps/chosen": -400.0,
+      "logps/rejected": -450.0,
+      "loss": 0.1123,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 8.875,
+      "rewards/rejected": -27.0,
+      "step": 6440
+    },
+    {
+      "epoch": 0.8243338232474918,
+      "grad_norm": 6.266088921432867,
+      "learning_rate": 9.961164901835046e-07,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -386.0,
+      "logps/rejected": -460.0,
+      "loss": 0.0933,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 7.9375,
+      "rewards/rejected": -26.125,
+      "step": 6450
+    },
+    {
+      "epoch": 0.8256118601827592,
+      "grad_norm": 5.320047077446667,
+      "learning_rate": 9.95345203586879e-07,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -368.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1101,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 8.375,
+      "rewards/rejected": -26.125,
+      "step": 6460
+    },
+    {
+      "epoch": 0.8268898971180267,
+      "grad_norm": 6.278491590749454,
+      "learning_rate": 9.94575705827027e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -396.0,
+      "logps/rejected": -470.0,
+      "loss": 0.1082,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 8.6875,
+      "rewards/rejected": -26.0,
+      "step": 6470
+    },
+    {
+      "epoch": 0.8281679340532941,
+      "grad_norm": 6.927910052536261,
+      "learning_rate": 9.938079899999065e-07,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -398.0,
+      "logps/rejected": -428.0,
+      "loss": 0.1265,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 7.40625,
+      "rewards/rejected": -25.0,
+      "step": 6480
+    },
+    {
+      "epoch": 0.8294459709885615,
+      "grad_norm": 3.730617657383127,
+      "learning_rate": 9.930420492387219e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.984375,
+      "logps/chosen": -386.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1143,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 7.71875,
+      "rewards/rejected": -24.75,
+      "step": 6490
+    },
+    {
+      "epoch": 0.8307240079238289,
+      "grad_norm": 8.435510034690086,
+      "learning_rate": 9.922778767136676e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.8125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -468.0,
+      "loss": 0.1219,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -16.25,
+      "rewards/margins": 8.0625,
+      "rewards/rejected": -24.375,
+      "step": 6500
+    },
+    {
+      "epoch": 0.8320020448590965,
+      "grad_norm": 6.046793433191917,
+      "learning_rate": 9.915154656316713e-07,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1099,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -24.75,
+      "step": 6510
+    },
+    {
+      "epoch": 0.8332800817943639,
+      "grad_norm": 4.014980227140118,
+      "learning_rate": 9.907548092361398e-07,
+      "logits/chosen": -1.4453125,
+      "logits/rejected": -0.87890625,
+      "logps/chosen": -408.0,
+      "logps/rejected": -454.0,
+      "loss": 0.0995,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 7.75,
+      "rewards/rejected": -25.75,
+      "step": 6520
+    },
+    {
+      "epoch": 0.8345581187296313,
+      "grad_norm": 8.204096009672126,
+      "learning_rate": 9.899959008067097e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.96484375,
+      "logps/chosen": -394.0,
+      "logps/rejected": -480.0,
+      "loss": 0.1255,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -18.375,
+      "rewards/margins": 7.625,
+      "rewards/rejected": -26.0,
+      "step": 6530
+    },
+    {
+      "epoch": 0.8358361556648987,
+      "grad_norm": 4.974670854058527,
+      "learning_rate": 9.892387336589959e-07,
+      "logits/chosen": -1.1953125,
+      "logits/rejected": -0.80078125,
+      "logps/chosen": -386.0,
+      "logps/rejected": -430.0,
+      "loss": 0.0959,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.625,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -24.125,
+      "step": 6540
+    },
+    {
+      "epoch": 0.8371141926001662,
+      "grad_norm": 5.697582007476465,
+      "learning_rate": 9.884833011443446e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -360.0,
+      "logps/rejected": -442.0,
+      "loss": 0.1097,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 8.25,
+      "rewards/rejected": -24.25,
+      "step": 6550
+    },
+    {
+      "epoch": 0.8383922295354336,
+      "grad_norm": 9.899329556348572,
+      "learning_rate": 9.877295966495897e-07,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.85546875,
+      "logps/chosen": -352.0,
+      "logps/rejected": -406.0,
+      "loss": 0.1182,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -15.1875,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -22.75,
+      "step": 6560
+    },
+    {
+      "epoch": 0.839670266470701,
+      "grad_norm": 4.365381488415416,
+      "learning_rate": 9.86977613596807e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -358.0,
+      "logps/rejected": -412.0,
+      "loss": 0.1178,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.0,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -23.625,
+      "step": 6570
+    },
+    {
+      "epoch": 0.8409483034059684,
+      "grad_norm": 4.161449653350445,
+      "learning_rate": 9.862273454430757e-07,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -392.0,
+      "logps/rejected": -424.0,
+      "loss": 0.0873,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.71875,
+      "rewards/rejected": -24.5,
+      "step": 6580
+    },
+    {
+      "epoch": 0.8422263403412359,
+      "grad_norm": 6.3882055022707185,
+      "learning_rate": 9.85478785680238e-07,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -402.0,
+      "logps/rejected": -454.0,
+      "loss": 0.1067,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -25.5,
+      "step": 6590
+    },
+    {
+      "epoch": 0.8435043772765033,
+      "grad_norm": 4.5304316444756,
+      "learning_rate": 9.847319278346618e-07,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.953125,
+      "logps/chosen": -390.0,
+      "logps/rejected": -446.0,
+      "loss": 0.0988,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 8.3125,
+      "rewards/rejected": -24.875,
+      "step": 6600
+    },
+    {
+      "epoch": 0.8447824142117707,
+      "grad_norm": 6.684408905824012,
+      "learning_rate": 9.839867654670063e-07,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.875,
+      "logps/chosen": -390.0,
+      "logps/rejected": -444.0,
+      "loss": 0.0732,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 7.28125,
+      "rewards/rejected": -25.0,
+      "step": 6610
+    },
+    {
+      "epoch": 0.8460604511470381,
+      "grad_norm": 3.427491090092728,
+      "learning_rate": 9.832432921719876e-07,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.80078125,
+      "logps/chosen": -368.0,
+      "logps/rejected": -428.0,
+      "loss": 0.0894,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 8.5,
+      "rewards/rejected": -25.5,
+      "step": 6620
+    },
+    {
+      "epoch": 0.8473384880823056,
+      "grad_norm": 3.8996592501429173,
+      "learning_rate": 9.825015015781493e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.96484375,
+      "logps/chosen": -390.0,
+      "logps/rejected": -478.0,
+      "loss": 0.096,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 8.875,
+      "rewards/rejected": -26.0,
+      "step": 6630
+    },
+    {
+      "epoch": 0.848616525017573,
+      "grad_norm": 7.647528185762068,
+      "learning_rate": 9.81761387347632e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.9921875,
+      "logps/chosen": -376.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1037,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 9.125,
+      "rewards/rejected": -25.625,
+      "step": 6640
+    },
+    {
+      "epoch": 0.8498945619528404,
+      "grad_norm": 4.40796051425544,
+      "learning_rate": 9.810229431759452e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -382.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1031,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -16.375,
+      "rewards/margins": 9.125,
+      "rewards/rejected": -25.625,
+      "step": 6650
+    },
+    {
+      "epoch": 0.8511725988881078,
+      "grad_norm": 6.242943952937279,
+      "learning_rate": 9.802861627917437e-07,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -388.0,
+      "logps/rejected": -454.0,
+      "loss": 0.0819,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 7.71875,
+      "rewards/rejected": -25.5,
+      "step": 6660
+    },
+    {
+      "epoch": 0.8524506358233753,
+      "grad_norm": 6.6018444991740575,
+      "learning_rate": 9.795510399566016e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -394.0,
+      "logps/rejected": -442.0,
+      "loss": 0.1007,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 7.46875,
+      "rewards/rejected": -25.125,
+      "step": 6670
+    },
+    {
+      "epoch": 0.8537286727586427,
+      "grad_norm": 6.845894654697116,
+      "learning_rate": 9.788175684647926e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -394.0,
+      "logps/rejected": -462.0,
+      "loss": 0.1071,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 8.1875,
+      "rewards/rejected": -26.0,
+      "step": 6680
+    },
+    {
+      "epoch": 0.8550067096939101,
+      "grad_norm": 2.6121494961157143,
+      "learning_rate": 9.780857421430687e-07,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -392.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1123,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -18.5,
+      "rewards/margins": 7.96875,
+      "rewards/rejected": -26.5,
+      "step": 6690
+    },
+    {
+      "epoch": 0.8562847466291775,
+      "grad_norm": 7.685980620475692,
+      "learning_rate": 9.773555548504417e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.0,
+      "logps/chosen": -404.0,
+      "logps/rejected": -442.0,
+      "loss": 0.0901,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -18.25,
+      "rewards/margins": 8.125,
+      "rewards/rejected": -26.375,
+      "step": 6700
+    },
+    {
+      "epoch": 0.857562783564445,
+      "grad_norm": 4.609218126706455,
+      "learning_rate": 9.76627000477968e-07,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -382.0,
+      "logps/rejected": -442.0,
+      "loss": 0.0925,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -18.375,
+      "rewards/margins": 7.84375,
+      "rewards/rejected": -26.125,
+      "step": 6710
+    },
+    {
+      "epoch": 0.8588408204997124,
+      "grad_norm": 9.762553813772866,
+      "learning_rate": 9.75900072948533e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.9296875,
+      "logps/chosen": -396.0,
+      "logps/rejected": -458.0,
+      "loss": 0.0969,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -19.0,
+      "rewards/margins": 8.1875,
+      "rewards/rejected": -27.125,
+      "step": 6720
+    },
+    {
+      "epoch": 0.8601188574349798,
+      "grad_norm": 3.770133202281012,
+      "learning_rate": 9.751747662166388e-07,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -0.82421875,
+      "logps/chosen": -398.0,
+      "logps/rejected": -482.0,
+      "loss": 0.0983,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -18.625,
+      "rewards/margins": 8.8125,
+      "rewards/rejected": -27.5,
+      "step": 6730
+    },
+    {
+      "epoch": 0.8613968943702472,
+      "grad_norm": 4.936524679262009,
+      "learning_rate": 9.744510742681917e-07,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -404.0,
+      "logps/rejected": -468.0,
+      "loss": 0.0921,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -19.0,
+      "rewards/margins": 8.75,
+      "rewards/rejected": -27.75,
+      "step": 6740
+    },
+    {
+      "epoch": 0.8626749313055148,
+      "grad_norm": 10.063438619905556,
+      "learning_rate": 9.737289911202953e-07,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.87890625,
+      "logps/chosen": -372.0,
+      "logps/rejected": -436.0,
+      "loss": 0.1099,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 8.625,
+      "rewards/rejected": -26.375,
+      "step": 6750
+    },
+    {
+      "epoch": 0.8639529682407822,
+      "grad_norm": 5.837712714654185,
+      "learning_rate": 9.730085108210398e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -402.0,
+      "logps/rejected": -446.0,
+      "loss": 0.0862,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 8.6875,
+      "rewards/rejected": -25.875,
+      "step": 6760
+    },
+    {
+      "epoch": 0.8652310051760496,
+      "grad_norm": 10.422580355772604,
+      "learning_rate": 9.72289627449298e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -412.0,
+      "logps/rejected": -476.0,
+      "loss": 0.1027,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -19.375,
+      "rewards/margins": 8.375,
+      "rewards/rejected": -27.75,
+      "step": 6770
+    },
+    {
+      "epoch": 0.866509042111317,
+      "grad_norm": 5.852864363625125,
+      "learning_rate": 9.715723351145206e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.88671875,
+      "logps/chosen": -400.0,
+      "logps/rejected": -466.0,
+      "loss": 0.1052,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -19.125,
+      "rewards/margins": 8.875,
+      "rewards/rejected": -28.0,
+      "step": 6780
+    },
+    {
+      "epoch": 0.8677870790465845,
+      "grad_norm": 3.9690062351310864,
+      "learning_rate": 9.70856627956532e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -398.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1295,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -18.5,
+      "rewards/margins": 8.25,
+      "rewards/rejected": -26.625,
+      "step": 6790
+    },
+    {
+      "epoch": 0.8690651159818519,
+      "grad_norm": 10.903615330487535,
+      "learning_rate": 9.701425001453318e-07,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -380.0,
+      "logps/rejected": -436.0,
+      "loss": 0.0936,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 7.65625,
+      "rewards/rejected": -25.5,
+      "step": 6800
+    },
+    {
+      "epoch": 0.8703431529171193,
+      "grad_norm": 8.478400917677888,
+      "learning_rate": 9.694299458808932e-07,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -396.0,
+      "logps/rejected": -464.0,
+      "loss": 0.0885,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -18.25,
+      "rewards/margins": 8.375,
+      "rewards/rejected": -26.75,
+      "step": 6810
+    },
+    {
+      "epoch": 0.8716211898523867,
+      "grad_norm": 4.432238073585557,
+      "learning_rate": 9.687189593929655e-07,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.92578125,
+      "logps/chosen": -372.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1014,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 8.5625,
+      "rewards/rejected": -26.0,
+      "step": 6820
+    },
+    {
+      "epoch": 0.8728992267876542,
+      "grad_norm": 7.723634420720052,
+      "learning_rate": 9.680095349408789e-07,
+      "logits/chosen": -1.53125,
+      "logits/rejected": -0.97265625,
+      "logps/chosen": -396.0,
+      "logps/rejected": -468.0,
+      "loss": 0.1006,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 9.1875,
+      "rewards/rejected": -27.0,
+      "step": 6830
+    },
+    {
+      "epoch": 0.8741772637229216,
+      "grad_norm": 5.251886329520516,
+      "learning_rate": 9.673016668133487e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -0.98046875,
+      "logps/chosen": -412.0,
+      "logps/rejected": -470.0,
+      "loss": 0.1049,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -19.125,
+      "rewards/margins": 8.0625,
+      "rewards/rejected": -27.25,
+      "step": 6840
+    },
+    {
+      "epoch": 0.875455300658189,
+      "grad_norm": 7.398297144418387,
+      "learning_rate": 9.66595349328283e-07,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -0.97265625,
+      "logps/chosen": -408.0,
+      "logps/rejected": -470.0,
+      "loss": 0.0913,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -19.375,
+      "rewards/margins": 8.6875,
+      "rewards/rejected": -28.125,
+      "step": 6850
+    },
+    {
+      "epoch": 0.8767333375934564,
+      "grad_norm": 2.728111817679818,
+      "learning_rate": 9.658905768325902e-07,
+      "logits/chosen": -1.4609375,
+      "logits/rejected": -1.0,
+      "logps/chosen": -408.0,
+      "logps/rejected": -470.0,
+      "loss": 0.0967,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -20.0,
+      "rewards/margins": 7.71875,
+      "rewards/rejected": -27.75,
+      "step": 6860
+    },
+    {
+      "epoch": 0.8780113745287239,
+      "grad_norm": 8.066656507730912,
+      "learning_rate": 9.651873437019902e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -422.0,
+      "logps/rejected": -478.0,
+      "loss": 0.0977,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -19.875,
+      "rewards/margins": 7.84375,
+      "rewards/rejected": -27.75,
+      "step": 6870
+    },
+    {
+      "epoch": 0.8792894114639913,
+      "grad_norm": 4.619029253093423,
+      "learning_rate": 9.644856443408243e-07,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -396.0,
+      "logps/rejected": -476.0,
+      "loss": 0.0872,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -19.5,
+      "rewards/margins": 7.75,
+      "rewards/rejected": -27.25,
+      "step": 6880
+    },
+    {
+      "epoch": 0.8805674483992587,
+      "grad_norm": 9.352778625730608,
+      "learning_rate": 9.637854731818697e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -408.0,
+      "logps/rejected": -476.0,
+      "loss": 0.078,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -19.5,
+      "rewards/margins": 8.5,
+      "rewards/rejected": -28.0,
+      "step": 6890
+    },
+    {
+      "epoch": 0.8818454853345261,
+      "grad_norm": 14.518094546935496,
+      "learning_rate": 9.630868246861536e-07,
+      "logits/chosen": -1.484375,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -402.0,
+      "logps/rejected": -476.0,
+      "loss": 0.1351,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 9.0,
+      "rewards/rejected": -27.125,
+      "step": 6900
+    },
+    {
+      "epoch": 0.8831235222697936,
+      "grad_norm": 6.937353837574335,
+      "learning_rate": 9.623896933427685e-07,
+      "logits/chosen": -1.484375,
+      "logits/rejected": -0.99609375,
+      "logps/chosen": -402.0,
+      "logps/rejected": -476.0,
+      "loss": 0.0826,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 8.5625,
+      "rewards/rejected": -26.5,
+      "step": 6910
+    },
+    {
+      "epoch": 0.884401559205061,
+      "grad_norm": 9.978174165945111,
+      "learning_rate": 9.6169407366869e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.953125,
+      "logps/chosen": -390.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1159,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -18.375,
+      "rewards/margins": 8.125,
+      "rewards/rejected": -26.5,
+      "step": 6920
+    },
+    {
+      "epoch": 0.8856795961403284,
+      "grad_norm": 4.8529371512412585,
+      "learning_rate": 9.609999602085963e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.9921875,
+      "logps/chosen": -392.0,
+      "logps/rejected": -454.0,
+      "loss": 0.1107,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -18.25,
+      "rewards/margins": 8.375,
+      "rewards/rejected": -26.625,
+      "step": 6930
+    },
+    {
+      "epoch": 0.8869576330755958,
+      "grad_norm": 8.089860626350854,
+      "learning_rate": 9.603073475346872e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -412.0,
+      "logps/rejected": -462.0,
+      "loss": 0.1084,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -18.25,
+      "rewards/margins": 8.125,
+      "rewards/rejected": -26.375,
+      "step": 6940
+    },
+    {
+      "epoch": 0.8882356700108633,
+      "grad_norm": 7.294008139222566,
+      "learning_rate": 9.596162302465074e-07,
+      "logits/chosen": -1.4453125,
+      "logits/rejected": -0.875,
+      "logps/chosen": -392.0,
+      "logps/rejected": -466.0,
+      "loss": 0.0891,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 7.90625,
+      "rewards/rejected": -25.5,
+      "step": 6950
+    },
+    {
+      "epoch": 0.8895137069461307,
+      "grad_norm": 4.043691995023884,
+      "learning_rate": 9.589266029707683e-07,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -384.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1031,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 8.6875,
+      "rewards/rejected": -25.625,
+      "step": 6960
+    },
+    {
+      "epoch": 0.8907917438813981,
+      "grad_norm": 5.07831789334116,
+      "learning_rate": 9.582384603611731e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -378.0,
+      "logps/rejected": -444.0,
+      "loss": 0.1133,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 7.9375,
+      "rewards/rejected": -25.375,
+      "step": 6970
+    },
+    {
+      "epoch": 0.8920697808166655,
+      "grad_norm": 33.239597089873264,
+      "learning_rate": 9.575517970982428e-07,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -408.0,
+      "logps/rejected": -440.0,
+      "loss": 0.0927,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 8.0625,
+      "rewards/rejected": -25.375,
+      "step": 6980
+    },
+    {
+      "epoch": 0.8933478177519331,
+      "grad_norm": 11.72674455695384,
+      "learning_rate": 9.568666078891436e-07,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.95703125,
+      "logps/chosen": -398.0,
+      "logps/rejected": -470.0,
+      "loss": 0.1316,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 8.25,
+      "rewards/rejected": -26.125,
+      "step": 6990
+    },
+    {
+      "epoch": 0.8946258546872005,
+      "grad_norm": 3.164406572252071,
+      "learning_rate": 9.561828874675149e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -384.0,
+      "logps/rejected": -436.0,
+      "loss": 0.0911,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 8.375,
+      "rewards/rejected": -25.125,
+      "step": 7000
+    },
+    {
+      "epoch": 0.8959038916224679,
+      "grad_norm": 7.421200507784557,
+      "learning_rate": 9.555006305933e-07,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -0.83203125,
+      "logps/chosen": -384.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1128,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.75,
+      "rewards/rejected": -24.5,
+      "step": 7010
+    },
+    {
+      "epoch": 0.8971819285577353,
+      "grad_norm": 5.44998418218611,
+      "learning_rate": 9.548198320525771e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -0.99609375,
+      "logps/chosen": -386.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1166,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 7.875,
+      "rewards/rejected": -26.0,
+      "step": 7020
+    },
+    {
+      "epoch": 0.8984599654930028,
+      "grad_norm": 8.089769650696821,
+      "learning_rate": 9.54140486657392e-07,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.0703125,
+      "logps/chosen": -426.0,
+      "logps/rejected": -484.0,
+      "loss": 0.0881,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -18.625,
+      "rewards/margins": 8.25,
+      "rewards/rejected": -26.875,
+      "step": 7030
+    },
+    {
+      "epoch": 0.8997380024282702,
+      "grad_norm": 5.187482694950811,
+      "learning_rate": 9.534625892455922e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -386.0,
+      "logps/rejected": -454.0,
+      "loss": 0.0883,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 8.0625,
+      "rewards/rejected": -26.25,
+      "step": 7040
+    },
+    {
+      "epoch": 0.9010160393635376,
+      "grad_norm": 7.613999279252346,
+      "learning_rate": 9.527861346806618e-07,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -0.98046875,
+      "logps/chosen": -400.0,
+      "logps/rejected": -450.0,
+      "loss": 0.0852,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 8.4375,
+      "rewards/rejected": -26.25,
+      "step": 7050
+    },
+    {
+      "epoch": 0.902294076298805,
+      "grad_norm": 6.09773740170567,
+      "learning_rate": 9.521111178515582e-07,
+      "logits/chosen": -1.53125,
+      "logits/rejected": -1.171875,
+      "logps/chosen": -400.0,
+      "logps/rejected": -460.0,
+      "loss": 0.093,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 7.875,
+      "rewards/rejected": -25.75,
+      "step": 7060
+    },
+    {
+      "epoch": 0.9035721132340725,
+      "grad_norm": 6.407321880281207,
+      "learning_rate": 9.514375336725502e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.9375,
+      "logps/chosen": -408.0,
+      "logps/rejected": -466.0,
+      "loss": 0.0979,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -19.375,
+      "rewards/margins": 7.375,
+      "rewards/rejected": -26.75,
+      "step": 7070
+    },
+    {
+      "epoch": 0.9048501501693399,
+      "grad_norm": 6.334891520954573,
+      "learning_rate": 9.507653770830566e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -384.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1086,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 7.84375,
+      "rewards/rejected": -25.875,
+      "step": 7080
+    },
+    {
+      "epoch": 0.9061281871046073,
+      "grad_norm": 6.753405686892047,
+      "learning_rate": 9.500946430474869e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -386.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1117,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -18.25,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -25.875,
+      "step": 7090
+    },
+    {
+      "epoch": 0.9074062240398747,
+      "grad_norm": 6.592906933394224,
+      "learning_rate": 9.494253265550825e-07,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -0.9296875,
+      "logps/chosen": -384.0,
+      "logps/rejected": -462.0,
+      "loss": 0.1078,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 8.6875,
+      "rewards/rejected": -26.5,
+      "step": 7100
+    },
+    {
+      "epoch": 0.9086842609751422,
+      "grad_norm": 8.126876807444352,
+      "learning_rate": 9.4875742261976e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.0234375,
+      "logps/chosen": -400.0,
+      "logps/rejected": -442.0,
+      "loss": 0.1102,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -19.25,
+      "rewards/margins": 7.09375,
+      "rewards/rejected": -26.25,
+      "step": 7110
+    },
+    {
+      "epoch": 0.9099622979104096,
+      "grad_norm": 4.958478429146128,
+      "learning_rate": 9.480909262799544e-07,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.953125,
+      "logps/chosen": -396.0,
+      "logps/rejected": -452.0,
+      "loss": 0.1121,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -18.375,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -26.0,
+      "step": 7120
+    },
+    {
+      "epoch": 0.911240334845677,
+      "grad_norm": 9.150097717441465,
+      "learning_rate": 9.47425832598465e-07,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -382.0,
+      "logps/rejected": -434.0,
+      "loss": 0.106,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 7.1875,
+      "rewards/rejected": -24.875,
+      "step": 7130
+    },
+    {
+      "epoch": 0.9125183717809444,
+      "grad_norm": 5.191019569291211,
+      "learning_rate": 9.467621366623017e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.8203125,
+      "logps/chosen": -404.0,
+      "logps/rejected": -476.0,
+      "loss": 0.086,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 7.96875,
+      "rewards/rejected": -26.0,
+      "step": 7140
+    },
+    {
+      "epoch": 0.9137964087162119,
+      "grad_norm": 6.960854028688517,
+      "learning_rate": 9.46099833582532e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.9140625,
+      "logps/chosen": -390.0,
+      "logps/rejected": -466.0,
+      "loss": 0.0838,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 8.3125,
+      "rewards/rejected": -26.125,
+      "step": 7150
+    },
+    {
+      "epoch": 0.9150744456514793,
+      "grad_norm": 6.355485759356552,
+      "learning_rate": 9.45438918494131e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -394.0,
+      "logps/rejected": -468.0,
+      "loss": 0.1003,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 8.3125,
+      "rewards/rejected": -26.5,
+      "step": 7160
+    },
+    {
+      "epoch": 0.9163524825867467,
+      "grad_norm": 9.434968548311748,
+      "learning_rate": 9.447793865558291e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -394.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1123,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -18.625,
+      "rewards/margins": 7.8125,
+      "rewards/rejected": -26.5,
+      "step": 7170
+    },
+    {
+      "epoch": 0.9176305195220141,
+      "grad_norm": 4.911059852073827,
+      "learning_rate": 9.441212329499659e-07,
+      "logits/chosen": -1.25,
+      "logits/rejected": -0.90234375,
+      "logps/chosen": -388.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1034,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 8.375,
+      "rewards/rejected": -26.625,
+      "step": 7180
+    },
+    {
+      "epoch": 0.9189085564572816,
+      "grad_norm": 5.907759843742338,
+      "learning_rate": 9.434644528823399e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -390.0,
+      "logps/rejected": -434.0,
+      "loss": 0.0735,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 8.125,
+      "rewards/rejected": -25.625,
+      "step": 7190
+    },
+    {
+      "epoch": 0.920186593392549,
+      "grad_norm": 9.584345062828861,
+      "learning_rate": 9.428090415820634e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.7890625,
+      "logps/chosen": -394.0,
+      "logps/rejected": -464.0,
+      "loss": 0.1172,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -18.25,
+      "rewards/margins": 8.5,
+      "rewards/rejected": -26.75,
+      "step": 7200
+    },
+    {
+      "epoch": 0.9214646303278164,
+      "grad_norm": 10.047034632152846,
+      "learning_rate": 9.42154994301416e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.89453125,
+      "logps/chosen": -400.0,
+      "logps/rejected": -440.0,
+      "loss": 0.1035,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 8.6875,
+      "rewards/rejected": -26.375,
+      "step": 7210
+    },
+    {
+      "epoch": 0.9227426672630838,
+      "grad_norm": 7.2333815685600635,
+      "learning_rate": 9.415023063157008e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.890625,
+      "logps/chosen": -410.0,
+      "logps/rejected": -452.0,
+      "loss": 0.0935,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 9.375,
+      "rewards/rejected": -27.375,
+      "step": 7220
+    },
+    {
+      "epoch": 0.9240207041983514,
+      "grad_norm": 5.501270439789532,
+      "learning_rate": 9.408509729231009e-07,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.83984375,
+      "logps/chosen": -392.0,
+      "logps/rejected": -464.0,
+      "loss": 0.0834,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -19.0,
+      "rewards/margins": 8.6875,
+      "rewards/rejected": -27.625,
+      "step": 7230
+    },
+    {
+      "epoch": 0.9252987411336188,
+      "grad_norm": 4.833291006187099,
+      "learning_rate": 9.402009894445369e-07,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -398.0,
+      "logps/rejected": -442.0,
+      "loss": 0.1176,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -18.75,
+      "rewards/margins": 8.625,
+      "rewards/rejected": -27.375,
+      "step": 7240
+    },
+    {
+      "epoch": 0.9265767780688862,
+      "grad_norm": 6.382967446209594,
+      "learning_rate": 9.395523512235255e-07,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -402.0,
+      "logps/rejected": -460.0,
+      "loss": 0.1148,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -19.25,
+      "rewards/margins": 7.71875,
+      "rewards/rejected": -27.0,
+      "step": 7250
+    },
+    {
+      "epoch": 0.9278548150041536,
+      "grad_norm": 6.826747162713079,
+      "learning_rate": 9.389050536260404e-07,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.84765625,
+      "logps/chosen": -416.0,
+      "logps/rejected": -480.0,
+      "loss": 0.104,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -19.5,
+      "rewards/margins": 7.84375,
+      "rewards/rejected": -27.375,
+      "step": 7260
+    },
+    {
+      "epoch": 0.9291328519394211,
+      "grad_norm": 5.504342218205628,
+      "learning_rate": 9.382590920403722e-07,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.80078125,
+      "logps/chosen": -400.0,
+      "logps/rejected": -452.0,
+      "loss": 0.1068,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -19.0,
+      "rewards/margins": 7.46875,
+      "rewards/rejected": -26.5,
+      "step": 7270
+    },
+    {
+      "epoch": 0.9304108888746885,
+      "grad_norm": 4.3248279853358085,
+      "learning_rate": 9.376144618769908e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -400.0,
+      "logps/rejected": -464.0,
+      "loss": 0.085,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -19.0,
+      "rewards/margins": 8.25,
+      "rewards/rejected": -27.25,
+      "step": 7280
+    },
+    {
+      "epoch": 0.9316889258099559,
+      "grad_norm": 8.451426800686386,
+      "learning_rate": 9.369711585684086e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.859375,
+      "logps/chosen": -402.0,
+      "logps/rejected": -458.0,
+      "loss": 0.0905,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -19.125,
+      "rewards/margins": 8.1875,
+      "rewards/rejected": -27.375,
+      "step": 7290
+    },
+    {
+      "epoch": 0.9329669627452233,
+      "grad_norm": 5.726713343944587,
+      "learning_rate": 9.363291775690445e-07,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -0.80078125,
+      "logps/chosen": -404.0,
+      "logps/rejected": -482.0,
+      "loss": 0.1174,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 8.9375,
+      "rewards/rejected": -27.0,
+      "step": 7300
+    },
+    {
+      "epoch": 0.9342449996804908,
+      "grad_norm": 5.576985723031939,
+      "learning_rate": 9.356885143550886e-07,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.81640625,
+      "logps/chosen": -394.0,
+      "logps/rejected": -464.0,
+      "loss": 0.1018,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -18.75,
+      "rewards/margins": 8.6875,
+      "rewards/rejected": -27.5,
+      "step": 7310
+    },
+    {
+      "epoch": 0.9355230366157582,
+      "grad_norm": 5.972811136622307,
+      "learning_rate": 9.350491644243688e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -382.0,
+      "logps/rejected": -450.0,
+      "loss": 0.098,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -18.625,
+      "rewards/margins": 8.5,
+      "rewards/rejected": -27.25,
+      "step": 7320
+    },
+    {
+      "epoch": 0.9368010735510256,
+      "grad_norm": 7.655475567779487,
+      "learning_rate": 9.344111232962179e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.98828125,
+      "logps/chosen": -384.0,
+      "logps/rejected": -458.0,
+      "loss": 0.1028,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -18.875,
+      "rewards/margins": 8.5625,
+      "rewards/rejected": -27.5,
+      "step": 7330
+    },
+    {
+      "epoch": 0.938079110486293,
+      "grad_norm": 6.573266317396873,
+      "learning_rate": 9.337743865113415e-07,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -412.0,
+      "logps/rejected": -484.0,
+      "loss": 0.1221,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -18.5,
+      "rewards/margins": 8.1875,
+      "rewards/rejected": -26.75,
+      "step": 7340
+    },
+    {
+      "epoch": 0.9393571474215605,
+      "grad_norm": 11.89471121451416,
+      "learning_rate": 9.331389496316868e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -402.0,
+      "logps/rejected": -472.0,
+      "loss": 0.0947,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 9.125,
+      "rewards/rejected": -26.875,
+      "step": 7350
+    },
+    {
+      "epoch": 0.9406351843568279,
+      "grad_norm": 7.7692638705156485,
+      "learning_rate": 9.325048082403138e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -430.0,
+      "logps/rejected": -476.0,
+      "loss": 0.123,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -19.25,
+      "rewards/margins": 8.75,
+      "rewards/rejected": -28.0,
+      "step": 7360
+    },
+    {
+      "epoch": 0.9419132212920953,
+      "grad_norm": 5.316655370437257,
+      "learning_rate": 9.318719579412648e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.9765625,
+      "logps/chosen": -418.0,
+      "logps/rejected": -460.0,
+      "loss": 0.1152,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -18.375,
+      "rewards/margins": 8.5,
+      "rewards/rejected": -26.875,
+      "step": 7370
+    },
+    {
+      "epoch": 0.9431912582273627,
+      "grad_norm": 1.285284235137262,
+      "learning_rate": 9.312403943594374e-07,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -374.0,
+      "logps/rejected": -438.0,
+      "loss": 0.105,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -25.125,
+      "step": 7380
+    },
+    {
+      "epoch": 0.9444692951626302,
+      "grad_norm": 6.190707148050328,
+      "learning_rate": 9.306101131404582e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -372.0,
+      "logps/rejected": -444.0,
+      "loss": 0.0957,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -16.875,
+      "rewards/margins": 7.5,
+      "rewards/rejected": -24.375,
+      "step": 7390
+    },
+    {
+      "epoch": 0.9457473320978976,
+      "grad_norm": 27.002813907228703,
+      "learning_rate": 9.299811099505542e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.93359375,
+      "logps/chosen": -400.0,
+      "logps/rejected": -458.0,
+      "loss": 0.0975,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 8.125,
+      "rewards/rejected": -25.375,
+      "step": 7400
+    },
+    {
+      "epoch": 0.947025369033165,
+      "grad_norm": 8.88037788600425,
+      "learning_rate": 9.293533804764305e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.9609375,
+      "logps/chosen": -392.0,
+      "logps/rejected": -466.0,
+      "loss": 0.0906,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 8.75,
+      "rewards/rejected": -26.0,
+      "step": 7410
+    },
+    {
+      "epoch": 0.9483034059684324,
+      "grad_norm": 5.087323175580499,
+      "learning_rate": 9.28726920425144e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -1.0390625,
+      "logps/chosen": -400.0,
+      "logps/rejected": -470.0,
+      "loss": 0.0996,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -18.875,
+      "rewards/margins": 7.9375,
+      "rewards/rejected": -26.75,
+      "step": 7420
+    },
+    {
+      "epoch": 0.9495814429037,
+      "grad_norm": 8.416264189625831,
+      "learning_rate": 9.281017255239815e-07,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -0.84375,
+      "logps/chosen": -396.0,
+      "logps/rejected": -478.0,
+      "loss": 0.0998,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -18.25,
+      "rewards/margins": 8.4375,
+      "rewards/rejected": -26.625,
+      "step": 7430
+    },
+    {
+      "epoch": 0.9508594798389673,
+      "grad_norm": 5.961257507854672,
+      "learning_rate": 9.274777915203365e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.8515625,
+      "logps/chosen": -386.0,
+      "logps/rejected": -478.0,
+      "loss": 0.0834,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -25.25,
+      "step": 7440
+    },
+    {
+      "epoch": 0.9521375167742347,
+      "grad_norm": 13.721939553783256,
+      "learning_rate": 9.268551141815875e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -400.0,
+      "logps/rejected": -458.0,
+      "loss": 0.0994,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 7.59375,
+      "rewards/rejected": -25.0,
+      "step": 7450
+    },
+    {
+      "epoch": 0.9534155537095022,
+      "grad_norm": 8.700244446895944,
+      "learning_rate": 9.262336892949784e-07,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -0.8671875,
+      "logps/chosen": -380.0,
+      "logps/rejected": -432.0,
+      "loss": 0.1097,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -16.75,
+      "rewards/margins": 7.8125,
+      "rewards/rejected": -24.5,
+      "step": 7460
+    },
+    {
+      "epoch": 0.9546935906447697,
+      "grad_norm": 6.864476444338143,
+      "learning_rate": 9.256135126674977e-07,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -364.0,
+      "logps/rejected": -434.0,
+      "loss": 0.1045,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 7.96875,
+      "rewards/rejected": -25.25,
+      "step": 7470
+    },
+    {
+      "epoch": 0.9559716275800371,
+      "grad_norm": 10.278297604017423,
+      "learning_rate": 9.249945801257605e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -382.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1083,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 8.25,
+      "rewards/rejected": -25.625,
+      "step": 7480
+    },
+    {
+      "epoch": 0.9572496645153045,
+      "grad_norm": 7.7796694044799315,
+      "learning_rate": 9.243768875158902e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.86328125,
+      "logps/chosen": -388.0,
+      "logps/rejected": -438.0,
+      "loss": 0.1186,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 8.0625,
+      "rewards/rejected": -25.5,
+      "step": 7490
+    },
+    {
+      "epoch": 0.9585277014505719,
+      "grad_norm": 6.865296117430785,
+      "learning_rate": 9.23760430703401e-07,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.95703125,
+      "logps/chosen": -384.0,
+      "logps/rejected": -426.0,
+      "loss": 0.1071,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.125,
+      "rewards/margins": 7.6875,
+      "rewards/rejected": -24.875,
+      "step": 7500
+    },
+    {
+      "epoch": 0.9598057383858394,
+      "grad_norm": 12.666481395378009,
+      "learning_rate": 9.231452055730832e-07,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -0.94140625,
+      "logps/chosen": -406.0,
+      "logps/rejected": -446.0,
+      "loss": 0.0985,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 8.5,
+      "rewards/rejected": -26.5,
+      "step": 7510
+    },
+    {
+      "epoch": 0.9610837753211068,
+      "grad_norm": 4.4741941135913725,
+      "learning_rate": 9.225312080288851e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.8984375,
+      "logps/chosen": -368.0,
+      "logps/rejected": -438.0,
+      "loss": 0.0947,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 8.125,
+      "rewards/rejected": -24.75,
+      "step": 7520
+    },
+    {
+      "epoch": 0.9623618122563742,
+      "grad_norm": 9.133339910449749,
+      "learning_rate": 9.219184339938013e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -0.984375,
+      "logps/chosen": -392.0,
+      "logps/rejected": -462.0,
+      "loss": 0.1072,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 8.1875,
+      "rewards/rejected": -25.625,
+      "step": 7530
+    },
+    {
+      "epoch": 0.9636398491916416,
+      "grad_norm": 5.060741582793389,
+      "learning_rate": 9.213068794097574e-07,
+      "logits/chosen": -1.453125,
+      "logits/rejected": -0.96484375,
+      "logps/chosen": -390.0,
+      "logps/rejected": -466.0,
+      "loss": 0.0802,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 9.25,
+      "rewards/rejected": -26.5,
+      "step": 7540
+    },
+    {
+      "epoch": 0.9649178861269091,
+      "grad_norm": 7.551915584912899,
+      "learning_rate": 9.206965402374975e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -0.92578125,
+      "logps/chosen": -382.0,
+      "logps/rejected": -450.0,
+      "loss": 0.0913,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 8.125,
+      "rewards/rejected": -25.375,
+      "step": 7550
+    },
+    {
+      "epoch": 0.9661959230621765,
+      "grad_norm": 7.647164703167109,
+      "learning_rate": 9.200874124564723e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -412.0,
+      "logps/rejected": -446.0,
+      "loss": 0.1003,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 8.75,
+      "rewards/rejected": -26.5,
+      "step": 7560
+    },
+    {
+      "epoch": 0.9674739599974439,
+      "grad_norm": 14.083247565658889,
+      "learning_rate": 9.194794920647274e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.92578125,
+      "logps/chosen": -402.0,
+      "logps/rejected": -460.0,
+      "loss": 0.077,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 9.4375,
+      "rewards/rejected": -26.5,
+      "step": 7570
+    },
+    {
+      "epoch": 0.9687519969327113,
+      "grad_norm": 4.294840509915503,
+      "learning_rate": 9.188727750787932e-07,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -408.0,
+      "logps/rejected": -462.0,
+      "loss": 0.0919,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -18.25,
+      "rewards/margins": 8.6875,
+      "rewards/rejected": -26.875,
+      "step": 7580
+    },
+    {
+      "epoch": 0.9700300338679788,
+      "grad_norm": 8.539559868240332,
+      "learning_rate": 9.182672575335757e-07,
+      "logits/chosen": -1.46875,
+      "logits/rejected": -0.98828125,
+      "logps/chosen": -400.0,
+      "logps/rejected": -462.0,
+      "loss": 0.0947,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 8.4375,
+      "rewards/rejected": -26.375,
+      "step": 7590
+    },
+    {
+      "epoch": 0.9713080708032462,
+      "grad_norm": 7.153478060237526,
+      "learning_rate": 9.176629354822469e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -396.0,
+      "logps/rejected": -448.0,
+      "loss": 0.1082,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -18.25,
+      "rewards/margins": 7.5625,
+      "rewards/rejected": -25.875,
+      "step": 7600
+    },
+    {
+      "epoch": 0.9725861077385136,
+      "grad_norm": 10.221609457551647,
+      "learning_rate": 9.170598049961371e-07,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -0.98828125,
+      "logps/chosen": -422.0,
+      "logps/rejected": -480.0,
+      "loss": 0.0877,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -19.25,
+      "rewards/margins": 7.6875,
+      "rewards/rejected": -27.0,
+      "step": 7610
+    },
+    {
+      "epoch": 0.973864144673781,
+      "grad_norm": 4.363312597075587,
+      "learning_rate": 9.164578621646276e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.91796875,
+      "logps/chosen": -406.0,
+      "logps/rejected": -464.0,
+      "loss": 0.0766,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -19.125,
+      "rewards/margins": 8.1875,
+      "rewards/rejected": -27.25,
+      "step": 7620
+    },
+    {
+      "epoch": 0.9751421816090485,
+      "grad_norm": 3.2223617808202185,
+      "learning_rate": 9.15857103095044e-07,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -426.0,
+      "logps/rejected": -442.0,
+      "loss": 0.0945,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -18.5,
+      "rewards/margins": 8.0625,
+      "rewards/rejected": -26.625,
+      "step": 7630
+    },
+    {
+      "epoch": 0.9764202185443159,
+      "grad_norm": 6.850903058548948,
+      "learning_rate": 9.15257523912551e-07,
+      "logits/chosen": -1.5390625,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -406.0,
+      "logps/rejected": -476.0,
+      "loss": 0.1053,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -19.625,
+      "rewards/margins": 7.4375,
+      "rewards/rejected": -27.125,
+      "step": 7640
+    },
+    {
+      "epoch": 0.9776982554795833,
+      "grad_norm": 11.736143375545515,
+      "learning_rate": 9.146591207600472e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -0.94921875,
+      "logps/chosen": -406.0,
+      "logps/rejected": -468.0,
+      "loss": 0.1056,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -19.875,
+      "rewards/margins": 7.78125,
+      "rewards/rejected": -27.625,
+      "step": 7650
+    },
+    {
+      "epoch": 0.9789762924148507,
+      "grad_norm": 6.727989306817807,
+      "learning_rate": 9.140618897980601e-07,
+      "logits/chosen": -1.546875,
+      "logits/rejected": -1.0078125,
+      "logps/chosen": -402.0,
+      "logps/rejected": -464.0,
+      "loss": 0.0906,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -18.5,
+      "rewards/margins": 9.3125,
+      "rewards/rejected": -27.75,
+      "step": 7660
+    },
+    {
+      "epoch": 0.9802543293501182,
+      "grad_norm": 104.53418588652063,
+      "learning_rate": 9.134658272046442e-07,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -406.0,
+      "logps/rejected": -476.0,
+      "loss": 0.1049,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -18.5,
+      "rewards/margins": 8.625,
+      "rewards/rejected": -27.125,
+      "step": 7670
+    },
+    {
+      "epoch": 0.9815323662853856,
+      "grad_norm": 9.51009675428085,
+      "learning_rate": 9.128709291752768e-07,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -408.0,
+      "logps/rejected": -464.0,
+      "loss": 0.0921,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 8.3125,
+      "rewards/rejected": -26.125,
+      "step": 7680
+    },
+    {
+      "epoch": 0.982810403220653,
+      "grad_norm": 9.147336891936884,
+      "learning_rate": 9.122771919227568e-07,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -0.91015625,
+      "logps/chosen": -388.0,
+      "logps/rejected": -458.0,
+      "loss": 0.0829,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 8.625,
+      "rewards/rejected": -26.625,
+      "step": 7690
+    },
+    {
+      "epoch": 0.9840884401559205,
+      "grad_norm": 17.226195781674786,
+      "learning_rate": 9.116846116771035e-07,
+      "logits/chosen": -1.453125,
+      "logits/rejected": -0.87890625,
+      "logps/chosen": -402.0,
+      "logps/rejected": -458.0,
+      "loss": 0.0829,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -17.25,
+      "rewards/margins": 9.75,
+      "rewards/rejected": -27.0,
+      "step": 7700
+    },
+    {
+      "epoch": 0.985366477091188,
+      "grad_norm": 26.345225700671335,
+      "learning_rate": 9.110931846854553e-07,
+      "logits/chosen": -1.4609375,
+      "logits/rejected": -1.1171875,
+      "logps/chosen": -406.0,
+      "logps/rejected": -464.0,
+      "loss": 0.1107,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.75,
+      "rewards/margins": 8.75,
+      "rewards/rejected": -26.5,
+      "step": 7710
+    },
+    {
+      "epoch": 0.9866445140264554,
+      "grad_norm": 6.555563677629959,
+      "learning_rate": 9.105029072119708e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.046875,
+      "logps/chosen": -400.0,
+      "logps/rejected": -450.0,
+      "loss": 0.1069,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.875,
+      "rewards/margins": 7.90625,
+      "rewards/rejected": -25.75,
+      "step": 7720
+    },
+    {
+      "epoch": 0.9879225509617228,
+      "grad_norm": 9.248330165836487,
+      "learning_rate": 9.099137755377291e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.03125,
+      "logps/chosen": -386.0,
+      "logps/rejected": -440.0,
+      "loss": 0.0959,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -17.5,
+      "rewards/margins": 8.0625,
+      "rewards/rejected": -25.625,
+      "step": 7730
+    },
+    {
+      "epoch": 0.9892005878969902,
+      "grad_norm": 14.28383105363626,
+      "learning_rate": 9.093257859606311e-07,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -396.0,
+      "logps/rejected": -444.0,
+      "loss": 0.0927,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 7.9375,
+      "rewards/rejected": -25.875,
+      "step": 7740
+    },
+    {
+      "epoch": 0.9904786248322577,
+      "grad_norm": 6.751494047673582,
+      "learning_rate": 9.087389347953037e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -0.90625,
+      "logps/chosen": -398.0,
+      "logps/rejected": -438.0,
+      "loss": 0.0956,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 8.4375,
+      "rewards/rejected": -25.75,
+      "step": 7750
+    },
+    {
+      "epoch": 0.9917566617675251,
+      "grad_norm": 4.530916042350466,
+      "learning_rate": 9.081532183729995e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -388.0,
+      "logps/rejected": -456.0,
+      "loss": 0.0968,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -18.125,
+      "rewards/margins": 8.125,
+      "rewards/rejected": -26.25,
+      "step": 7760
+    },
+    {
+      "epoch": 0.9930346987027925,
+      "grad_norm": 5.32534263877865,
+      "learning_rate": 9.075686330415037e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.015625,
+      "logps/chosen": -398.0,
+      "logps/rejected": -448.0,
+      "loss": 0.098,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -18.0,
+      "rewards/margins": 8.3125,
+      "rewards/rejected": -26.375,
+      "step": 7770
+    },
+    {
+      "epoch": 0.9943127356380599,
+      "grad_norm": 5.674429330792858,
+      "learning_rate": 9.069851751650364e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.078125,
+      "logps/chosen": -384.0,
+      "logps/rejected": -466.0,
+      "loss": 0.0995,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 8.5625,
+      "rewards/rejected": -25.875,
+      "step": 7780
+    },
+    {
+      "epoch": 0.9955907725733274,
+      "grad_norm": 4.3769219729326,
+      "learning_rate": 9.064028411241582e-07,
+      "logits/chosen": -1.453125,
+      "logits/rejected": -0.96875,
+      "logps/chosen": -370.0,
+      "logps/rejected": -426.0,
+      "loss": 0.0778,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -17.0,
+      "rewards/margins": 8.3125,
+      "rewards/rejected": -25.375,
+      "step": 7790
+    },
+    {
+      "epoch": 0.9968688095085948,
+      "grad_norm": 8.665930873479642,
+      "learning_rate": 9.058216273156764e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.9453125,
+      "logps/chosen": -380.0,
+      "logps/rejected": -448.0,
+      "loss": 0.0969,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -16.5,
+      "rewards/margins": 8.4375,
+      "rewards/rejected": -24.875,
+      "step": 7800
+    },
+    {
+      "epoch": 0.9981468464438622,
+      "grad_norm": 16.355412702584342,
+      "learning_rate": 9.052415301525511e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -0.92578125,
+      "logps/chosen": -412.0,
+      "logps/rejected": -490.0,
+      "loss": 0.0957,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -17.375,
+      "rewards/margins": 9.0625,
+      "rewards/rejected": -26.375,
+      "step": 7810
+    },
+    {
+      "epoch": 0.9994248833791296,
+      "grad_norm": 7.527030438946665,
+      "learning_rate": 9.046625460638012e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -0.97265625,
+      "logps/chosen": -382.0,
+      "logps/rejected": -446.0,
+      "loss": 0.069,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -17.625,
+      "rewards/margins": 9.625,
+      "rewards/rejected": -27.25,
+      "step": 7820
+    },
+    {
+      "epoch": 0.9999360981532366,
+      "step": 7824,
+      "total_flos": 0.0,
+      "train_loss": 0.15463680950715253,
+      "train_runtime": 70743.949,
+      "train_samples_per_second": 14.157,
+      "train_steps_per_second": 0.111
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 7824,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 5000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}