diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,40427 @@
+{
+  "best_metric": 0.4587233364582062,
+  "best_model_checkpoint": "models/qwen2.5-3b-dpo-vanilla-no-tools/checkpoint-10000",
+  "epoch": 0.9999813887699838,
+  "eval_steps": 5000,
+  "global_step": 26865,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 3.722246003238354e-05,
+      "grad_norm": 1.9097327730480331,
+      "learning_rate": 1.8608113137327875e-10,
+      "logits/chosen": -2.65625,
+      "logits/rejected": -2.1875,
+      "logps/chosen": -128.0,
+      "logps/rejected": -128.0,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0003722246003238354,
+      "grad_norm": 2.2702518248859556,
+      "learning_rate": 1.8608113137327875e-09,
+      "logits/chosen": -2.484375,
+      "logits/rejected": -2.28125,
+      "logps/chosen": -166.0,
+      "logps/rejected": -165.0,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.0833333358168602,
+      "rewards/chosen": -0.000522613525390625,
+      "rewards/margins": -0.000904083251953125,
+      "rewards/rejected": 0.0003833770751953125,
+      "step": 10
+    },
+    {
+      "epoch": 0.0007444492006476708,
+      "grad_norm": 1.9495617640862475,
+      "learning_rate": 3.721622627465575e-09,
+      "logits/chosen": -2.4375,
+      "logits/rejected": -2.46875,
+      "logps/chosen": -196.0,
+      "logps/rejected": -175.0,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.3375000059604645,
+      "rewards/chosen": -4.57763671875e-05,
+      "rewards/margins": 0.00084686279296875,
+      "rewards/rejected": -0.00089263916015625,
+      "step": 20
+    },
+    {
+      "epoch": 0.0011166738009715062,
+      "grad_norm": 1.6726810534860268,
+      "learning_rate": 5.5824339411983625e-09,
+      "logits/chosen": -2.515625,
+      "logits/rejected": -2.28125,
+      "logps/chosen": -182.0,
+      "logps/rejected": -163.0,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.28125,
+      "rewards/chosen": -0.0004825592041015625,
+      "rewards/margins": -0.000640869140625,
+      "rewards/rejected": 0.000156402587890625,
+      "step": 30
+    },
+    {
+      "epoch": 0.0014888984012953416,
+      "grad_norm": 1.6462117598724253,
+      "learning_rate": 7.44324525493115e-09,
+      "logits/chosen": -2.53125,
+      "logits/rejected": -2.375,
+      "logps/chosen": -188.0,
+      "logps/rejected": -162.0,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": 0.0002956390380859375,
+      "rewards/margins": 0.000934600830078125,
+      "rewards/rejected": -0.000640869140625,
+      "step": 40
+    },
+    {
+      "epoch": 0.001861123001619177,
+      "grad_norm": 1.8766243763315376,
+      "learning_rate": 9.304056568663937e-09,
+      "logits/chosen": -2.484375,
+      "logits/rejected": -2.28125,
+      "logps/chosen": -177.0,
+      "logps/rejected": -153.0,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": 0.0012054443359375,
+      "rewards/margins": 0.000690460205078125,
+      "rewards/rejected": 0.000518798828125,
+      "step": 50
+    },
+    {
+      "epoch": 0.0022333476019430125,
+      "grad_norm": 1.8802283859316804,
+      "learning_rate": 1.1164867882396725e-08,
+      "logits/chosen": -2.65625,
+      "logits/rejected": -2.46875,
+      "logps/chosen": -170.0,
+      "logps/rejected": -145.0,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.35624998807907104,
+      "rewards/chosen": 0.0004062652587890625,
+      "rewards/margins": -2.9802322387695312e-05,
+      "rewards/rejected": 0.000438690185546875,
+      "step": 60
+    },
+    {
+      "epoch": 0.002605572202266848,
+      "grad_norm": 2.01305504328485,
+      "learning_rate": 1.3025679196129512e-08,
+      "logits/chosen": -2.5,
+      "logits/rejected": -2.328125,
+      "logps/chosen": -181.0,
+      "logps/rejected": -169.0,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.2874999940395355,
+      "rewards/chosen": -0.00029754638671875,
+      "rewards/margins": 0.00016021728515625,
+      "rewards/rejected": -0.000453948974609375,
+      "step": 70
+    },
+    {
+      "epoch": 0.002977796802590683,
+      "grad_norm": 1.8073154622979284,
+      "learning_rate": 1.48864905098623e-08,
+      "logits/chosen": -2.421875,
+      "logits/rejected": -2.234375,
+      "logps/chosen": -161.0,
+      "logps/rejected": -142.0,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.00177001953125,
+      "rewards/margins": 0.0014190673828125,
+      "rewards/rejected": -0.0031890869140625,
+      "step": 80
+    },
+    {
+      "epoch": 0.0033500214029145185,
+      "grad_norm": 1.8421541102872958,
+      "learning_rate": 1.6747301823595087e-08,
+      "logits/chosen": -2.515625,
+      "logits/rejected": -2.359375,
+      "logps/chosen": -196.0,
+      "logps/rejected": -186.0,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.3125,
+      "rewards/chosen": 0.0001392364501953125,
+      "rewards/margins": -0.00021648406982421875,
+      "rewards/rejected": 0.0003604888916015625,
+      "step": 90
+    },
+    {
+      "epoch": 0.003722246003238354,
+      "grad_norm": 1.9929974368949581,
+      "learning_rate": 1.8608113137327873e-08,
+      "logits/chosen": -2.484375,
+      "logits/rejected": -2.359375,
+      "logps/chosen": -212.0,
+      "logps/rejected": -179.0,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.35624998807907104,
+      "rewards/chosen": -0.00028228759765625,
+      "rewards/margins": 0.00028228759765625,
+      "rewards/rejected": -0.0005645751953125,
+      "step": 100
+    },
+    {
+      "epoch": 0.004094470603562189,
+      "grad_norm": 1.7727687424813967,
+      "learning_rate": 2.0468924451060663e-08,
+      "logits/chosen": -2.484375,
+      "logits/rejected": -2.328125,
+      "logps/chosen": -196.0,
+      "logps/rejected": -176.0,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.3062500059604645,
+      "rewards/chosen": -0.00115966796875,
+      "rewards/margins": -0.0005035400390625,
+      "rewards/rejected": -0.0006561279296875,
+      "step": 110
+    },
+    {
+      "epoch": 0.004466695203886025,
+      "grad_norm": 1.7306721315573685,
+      "learning_rate": 2.232973576479345e-08,
+      "logits/chosen": -2.546875,
+      "logits/rejected": -2.3125,
+      "logps/chosen": -182.0,
+      "logps/rejected": -173.0,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.26875001192092896,
+      "rewards/chosen": -0.00148773193359375,
+      "rewards/margins": -0.00162506103515625,
+      "rewards/rejected": 0.000141143798828125,
+      "step": 120
+    },
+    {
+      "epoch": 0.00483891980420986,
+      "grad_norm": 1.9897990343864889,
+      "learning_rate": 2.4190547078526237e-08,
+      "logits/chosen": -2.59375,
+      "logits/rejected": -2.328125,
+      "logps/chosen": -187.0,
+      "logps/rejected": -180.0,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -0.000782012939453125,
+      "rewards/margins": 0.0002498626708984375,
+      "rewards/rejected": -0.00102996826171875,
+      "step": 130
+    },
+    {
+      "epoch": 0.005211144404533696,
+      "grad_norm": 1.731446878836939,
+      "learning_rate": 2.6051358392259023e-08,
+      "logits/chosen": -2.515625,
+      "logits/rejected": -2.203125,
+      "logps/chosen": -188.0,
+      "logps/rejected": -167.0,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.3687500059604645,
+      "rewards/chosen": -3.818422555923462e-07,
+      "rewards/margins": 0.000812530517578125,
+      "rewards/rejected": -0.000812530517578125,
+      "step": 140
+    },
+    {
+      "epoch": 0.005583369004857531,
+      "grad_norm": 2.0915462289977245,
+      "learning_rate": 2.791216970599181e-08,
+      "logits/chosen": -2.53125,
+      "logits/rejected": -2.28125,
+      "logps/chosen": -180.0,
+      "logps/rejected": -158.0,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.35624998807907104,
+      "rewards/chosen": 1.4126300811767578e-05,
+      "rewards/margins": 0.0002956390380859375,
+      "rewards/rejected": -0.0002803802490234375,
+      "step": 150
+    },
+    {
+      "epoch": 0.005955593605181366,
+      "grad_norm": 1.974633221074281,
+      "learning_rate": 2.97729810197246e-08,
+      "logits/chosen": -2.578125,
+      "logits/rejected": -2.34375,
+      "logps/chosen": -176.0,
+      "logps/rejected": -156.0,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.3375000059604645,
+      "rewards/chosen": 4.5299530029296875e-05,
+      "rewards/margins": 1.6450881958007812e-05,
+      "rewards/rejected": 3.170967102050781e-05,
+      "step": 160
+    },
+    {
+      "epoch": 0.006327818205505202,
+      "grad_norm": 1.8969490991497713,
+      "learning_rate": 3.163379233345739e-08,
+      "logits/chosen": -2.484375,
+      "logits/rejected": -2.21875,
+      "logps/chosen": -171.0,
+      "logps/rejected": -158.0,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.3125,
+      "rewards/chosen": -0.00147247314453125,
+      "rewards/margins": 0.00012493133544921875,
+      "rewards/rejected": -0.00159454345703125,
+      "step": 170
+    },
+    {
+      "epoch": 0.006700042805829037,
+      "grad_norm": 1.744630438934527,
+      "learning_rate": 3.3494603647190173e-08,
+      "logits/chosen": -2.46875,
+      "logits/rejected": -2.40625,
+      "logps/chosen": -189.0,
+      "logps/rejected": -159.0,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.0002307891845703125,
+      "rewards/margins": 0.0001697540283203125,
+      "rewards/rejected": -0.0004062652587890625,
+      "step": 180
+    },
+    {
+      "epoch": 0.007072267406152873,
+      "grad_norm": 1.9454954155767545,
+      "learning_rate": 3.535541496092296e-08,
+      "logits/chosen": -2.53125,
+      "logits/rejected": -2.3125,
+      "logps/chosen": -166.0,
+      "logps/rejected": -157.0,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.33125001192092896,
+      "rewards/chosen": -0.0006256103515625,
+      "rewards/margins": -0.000751495361328125,
+      "rewards/rejected": 0.0001239776611328125,
+      "step": 190
+    },
+    {
+      "epoch": 0.007444492006476708,
+      "grad_norm": 1.9102221997085012,
+      "learning_rate": 3.721622627465575e-08,
+      "logits/chosen": -2.640625,
+      "logits/rejected": -2.375,
+      "logps/chosen": -195.0,
+      "logps/rejected": -182.0,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": -2.9802322387695312e-05,
+      "rewards/margins": 0.0004863739013671875,
+      "rewards/rejected": -0.000514984130859375,
+      "step": 200
+    },
+    {
+      "epoch": 0.007816716606800543,
+      "grad_norm": 2.0951816134962806,
+      "learning_rate": 3.9077037588388533e-08,
+      "logits/chosen": -2.46875,
+      "logits/rejected": -2.25,
+      "logps/chosen": -176.0,
+      "logps/rejected": -163.0,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": 0.002655029296875,
+      "rewards/margins": 0.0017242431640625,
+      "rewards/rejected": 0.0009307861328125,
+      "step": 210
+    },
+    {
+      "epoch": 0.008188941207124378,
+      "grad_norm": 1.7457974140727253,
+      "learning_rate": 4.093784890212133e-08,
+      "logits/chosen": -2.4375,
+      "logits/rejected": -2.1875,
+      "logps/chosen": -180.0,
+      "logps/rejected": -160.0,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.3687500059604645,
+      "rewards/chosen": 0.0011444091796875,
+      "rewards/margins": 0.00115203857421875,
+      "rewards/rejected": -1.6808509826660156e-05,
+      "step": 220
+    },
+    {
+      "epoch": 0.008561165807448215,
+      "grad_norm": 1.8752016004850744,
+      "learning_rate": 4.2798660215854113e-08,
+      "logits/chosen": -2.53125,
+      "logits/rejected": -2.25,
+      "logps/chosen": -172.0,
+      "logps/rejected": -153.0,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.41874998807907104,
+      "rewards/chosen": -0.0004711151123046875,
+      "rewards/margins": -0.0005950927734375,
+      "rewards/rejected": 0.000125885009765625,
+      "step": 230
+    },
+    {
+      "epoch": 0.00893339040777205,
+      "grad_norm": 1.597670525095004,
+      "learning_rate": 4.46594715295869e-08,
+      "logits/chosen": -2.5,
+      "logits/rejected": -2.375,
+      "logps/chosen": -187.0,
+      "logps/rejected": -156.0,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.00151824951171875,
+      "rewards/margins": 0.00341796875,
+      "rewards/rejected": -0.0019073486328125,
+      "step": 240
+    },
+    {
+      "epoch": 0.009305615008095885,
+      "grad_norm": 1.8259095323565484,
+      "learning_rate": 4.652028284331969e-08,
+      "logits/chosen": -2.4375,
+      "logits/rejected": -2.265625,
+      "logps/chosen": -183.0,
+      "logps/rejected": -177.0,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.4312500059604645,
+      "rewards/chosen": 0.00213623046875,
+      "rewards/margins": 0.00145721435546875,
+      "rewards/rejected": 0.000690460205078125,
+      "step": 250
+    },
+    {
+      "epoch": 0.00967783960841972,
+      "grad_norm": 1.8886451886739373,
+      "learning_rate": 4.8381094157052473e-08,
+      "logits/chosen": -2.578125,
+      "logits/rejected": -2.296875,
+      "logps/chosen": -181.0,
+      "logps/rejected": -178.0,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.38749998807907104,
+      "rewards/chosen": 0.00156402587890625,
+      "rewards/margins": 0.00145721435546875,
+      "rewards/rejected": 0.00010967254638671875,
+      "step": 260
+    },
+    {
+      "epoch": 0.010050064208743556,
+      "grad_norm": 1.8776534107788239,
+      "learning_rate": 5.024190547078526e-08,
+      "logits/chosen": -2.5625,
+      "logits/rejected": -2.1875,
+      "logps/chosen": -183.0,
+      "logps/rejected": -185.0,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.38749998807907104,
+      "rewards/chosen": 0.001953125,
+      "rewards/margins": 0.0012664794921875,
+      "rewards/rejected": 0.0006866455078125,
+      "step": 270
+    },
+    {
+      "epoch": 0.010422288809067391,
+      "grad_norm": 1.8604464869011383,
+      "learning_rate": 5.210271678451805e-08,
+      "logits/chosen": -2.5,
+      "logits/rejected": -2.265625,
+      "logps/chosen": -187.0,
+      "logps/rejected": -178.0,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 0.00177001953125,
+      "rewards/margins": 0.001953125,
+      "rewards/rejected": -0.000186920166015625,
+      "step": 280
+    },
+    {
+      "epoch": 0.010794513409391226,
+      "grad_norm": 1.9017588370457659,
+      "learning_rate": 5.3963528098250833e-08,
+      "logits/chosen": -2.53125,
+      "logits/rejected": -2.265625,
+      "logps/chosen": -185.0,
+      "logps/rejected": -163.0,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.0027618408203125,
+      "rewards/margins": 0.00225830078125,
+      "rewards/rejected": 0.000499725341796875,
+      "step": 290
+    },
+    {
+      "epoch": 0.011166738009715063,
+      "grad_norm": 1.795678440130463,
+      "learning_rate": 5.582433941198362e-08,
+      "logits/chosen": -2.46875,
+      "logits/rejected": -2.3125,
+      "logps/chosen": -185.0,
+      "logps/rejected": -174.0,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.41874998807907104,
+      "rewards/chosen": 0.0028228759765625,
+      "rewards/margins": 0.000949859619140625,
+      "rewards/rejected": 0.0018768310546875,
+      "step": 300
+    },
+    {
+      "epoch": 0.011538962610038898,
+      "grad_norm": 1.8500049991168492,
+      "learning_rate": 5.7685150725716413e-08,
+      "logits/chosen": -2.5625,
+      "logits/rejected": -2.421875,
+      "logps/chosen": -186.0,
+      "logps/rejected": -162.0,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.003448486328125,
+      "rewards/margins": 0.0016937255859375,
+      "rewards/rejected": 0.00176239013671875,
+      "step": 310
+    },
+    {
+      "epoch": 0.011911187210362733,
+      "grad_norm": 1.7819307818272416,
+      "learning_rate": 5.95459620394492e-08,
+      "logits/chosen": -2.65625,
+      "logits/rejected": -2.375,
+      "logps/chosen": -177.0,
+      "logps/rejected": -162.0,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": 0.00445556640625,
+      "rewards/margins": 0.003448486328125,
+      "rewards/rejected": 0.00099945068359375,
+      "step": 320
+    },
+    {
+      "epoch": 0.012283411810686568,
+      "grad_norm": 1.9329725404869653,
+      "learning_rate": 6.140677335318198e-08,
+      "logits/chosen": -2.484375,
+      "logits/rejected": -2.390625,
+      "logps/chosen": -168.0,
+      "logps/rejected": -152.0,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.003936767578125,
+      "rewards/margins": 0.00262451171875,
+      "rewards/rejected": 0.0012969970703125,
+      "step": 330
+    },
+    {
+      "epoch": 0.012655636411010404,
+      "grad_norm": 1.808538628844211,
+      "learning_rate": 6.326758466691477e-08,
+      "logits/chosen": -2.53125,
+      "logits/rejected": -2.1875,
+      "logps/chosen": -187.0,
+      "logps/rejected": -164.0,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 0.00531005859375,
+      "rewards/margins": 0.00347900390625,
+      "rewards/rejected": 0.0018310546875,
+      "step": 340
+    },
+    {
+      "epoch": 0.013027861011334239,
+      "grad_norm": 1.91375836254956,
+      "learning_rate": 6.512839598064757e-08,
+      "logits/chosen": -2.515625,
+      "logits/rejected": -2.265625,
+      "logps/chosen": -200.0,
+      "logps/rejected": -181.0,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": 0.007080078125,
+      "rewards/margins": 0.0029296875,
+      "rewards/rejected": 0.004150390625,
+      "step": 350
+    },
+    {
+      "epoch": 0.013400085611658074,
+      "grad_norm": 1.9145884011568952,
+      "learning_rate": 6.698920729438035e-08,
+      "logits/chosen": -2.578125,
+      "logits/rejected": -2.421875,
+      "logps/chosen": -182.0,
+      "logps/rejected": -167.0,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.007537841796875,
+      "rewards/margins": 0.0050048828125,
+      "rewards/rejected": 0.0025482177734375,
+      "step": 360
+    },
+    {
+      "epoch": 0.01377231021198191,
+      "grad_norm": 1.670827816536535,
+      "learning_rate": 6.885001860811314e-08,
+      "logits/chosen": -2.609375,
+      "logits/rejected": -2.359375,
+      "logps/chosen": -174.0,
+      "logps/rejected": -171.0,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0093994140625,
+      "rewards/margins": 0.0054931640625,
+      "rewards/rejected": 0.00390625,
+      "step": 370
+    },
+    {
+      "epoch": 0.014144534812305746,
+      "grad_norm": 1.8291731872457426,
+      "learning_rate": 7.071082992184592e-08,
+      "logits/chosen": -2.46875,
+      "logits/rejected": -2.21875,
+      "logps/chosen": -171.0,
+      "logps/rejected": -161.0,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.41874998807907104,
+      "rewards/chosen": 0.0108642578125,
+      "rewards/margins": 0.002960205078125,
+      "rewards/rejected": 0.0079345703125,
+      "step": 380
+    },
+    {
+      "epoch": 0.01451675941262958,
+      "grad_norm": 1.8166615721058887,
+      "learning_rate": 7.257164123557871e-08,
+      "logits/chosen": -2.59375,
+      "logits/rejected": -2.4375,
+      "logps/chosen": -188.0,
+      "logps/rejected": -156.0,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.0091552734375,
+      "rewards/margins": 0.00457763671875,
+      "rewards/rejected": 0.00457763671875,
+      "step": 390
+    },
+    {
+      "epoch": 0.014888984012953415,
+      "grad_norm": 1.977920783596711,
+      "learning_rate": 7.44324525493115e-08,
+      "logits/chosen": -2.625,
+      "logits/rejected": -2.484375,
+      "logps/chosen": -200.0,
+      "logps/rejected": -183.0,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.0111083984375,
+      "rewards/margins": 0.00823974609375,
+      "rewards/rejected": 0.002838134765625,
+      "step": 400
+    },
+    {
+      "epoch": 0.015261208613277252,
+      "grad_norm": 1.7431636454878188,
+      "learning_rate": 7.629326386304429e-08,
+      "logits/chosen": -2.625,
+      "logits/rejected": -2.4375,
+      "logps/chosen": -193.0,
+      "logps/rejected": -166.0,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.01141357421875,
+      "rewards/margins": 0.0059814453125,
+      "rewards/rejected": 0.00543212890625,
+      "step": 410
+    },
+    {
+      "epoch": 0.015633433213601087,
+      "grad_norm": 1.8562135125891985,
+      "learning_rate": 7.815407517677707e-08,
+      "logits/chosen": -2.59375,
+      "logits/rejected": -2.28125,
+      "logps/chosen": -183.0,
+      "logps/rejected": -169.0,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.01318359375,
+      "rewards/margins": 0.010498046875,
+      "rewards/rejected": 0.0026702880859375,
+      "step": 420
+    },
+    {
+      "epoch": 0.016005657813924924,
+      "grad_norm": 1.6984781110832412,
+      "learning_rate": 8.001488649050986e-08,
+      "logits/chosen": -2.5625,
+      "logits/rejected": -2.234375,
+      "logps/chosen": -179.0,
+      "logps/rejected": -168.0,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.01409912109375,
+      "rewards/margins": 0.00836181640625,
+      "rewards/rejected": 0.005767822265625,
+      "step": 430
+    },
+    {
+      "epoch": 0.016377882414248757,
+      "grad_norm": 1.8713009411972057,
+      "learning_rate": 8.187569780424265e-08,
+      "logits/chosen": -2.5,
+      "logits/rejected": -2.5,
+      "logps/chosen": -184.0,
+      "logps/rejected": -172.0,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.010498046875,
+      "rewards/margins": 0.00811767578125,
+      "rewards/rejected": 0.00238037109375,
+      "step": 440
+    },
+    {
+      "epoch": 0.016750107014572593,
+      "grad_norm": 1.844265594023156,
+      "learning_rate": 8.373650911797543e-08,
+      "logits/chosen": -2.609375,
+      "logits/rejected": -2.46875,
+      "logps/chosen": -179.0,
+      "logps/rejected": -164.0,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.01458740234375,
+      "rewards/margins": 0.01092529296875,
+      "rewards/rejected": 0.0036468505859375,
+      "step": 450
+    },
+    {
+      "epoch": 0.01712233161489643,
+      "grad_norm": 1.9249036266165669,
+      "learning_rate": 8.559732043170823e-08,
+      "logits/chosen": -2.359375,
+      "logits/rejected": -2.328125,
+      "logps/chosen": -193.0,
+      "logps/rejected": -162.0,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.0174560546875,
+      "rewards/margins": 0.0086669921875,
+      "rewards/rejected": 0.0087890625,
+      "step": 460
+    },
+    {
+      "epoch": 0.017494556215220263,
+      "grad_norm": 1.8947444885679214,
+      "learning_rate": 8.745813174544101e-08,
+      "logits/chosen": -2.5,
+      "logits/rejected": -2.25,
+      "logps/chosen": -176.0,
+      "logps/rejected": -171.0,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.0164794921875,
+      "rewards/margins": 0.01177978515625,
+      "rewards/rejected": 0.00469970703125,
+      "step": 470
+    },
+    {
+      "epoch": 0.0178667808155441,
+      "grad_norm": 1.8137950655296489,
+      "learning_rate": 8.93189430591738e-08,
+      "logits/chosen": -2.578125,
+      "logits/rejected": -2.53125,
+      "logps/chosen": -194.0,
+      "logps/rejected": -185.0,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.01068115234375,
+      "rewards/margins": 0.005340576171875,
+      "rewards/rejected": 0.005340576171875,
+      "step": 480
+    },
+    {
+      "epoch": 0.018239005415867936,
+      "grad_norm": 1.6754250068159429,
+      "learning_rate": 9.117975437290658e-08,
+      "logits/chosen": -2.578125,
+      "logits/rejected": -2.359375,
+      "logps/chosen": -187.0,
+      "logps/rejected": -169.0,
+      "loss": 0.6867,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.01470947265625,
+      "rewards/margins": 0.01171875,
+      "rewards/rejected": 0.00299072265625,
+      "step": 490
+    },
+    {
+      "epoch": 0.01861123001619177,
+      "grad_norm": 1.876860789975684,
+      "learning_rate": 9.304056568663937e-08,
+      "logits/chosen": -2.59375,
+      "logits/rejected": -2.390625,
+      "logps/chosen": -202.0,
+      "logps/rejected": -182.0,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 0.01123046875,
+      "rewards/margins": 0.0150146484375,
+      "rewards/rejected": -0.003753662109375,
+      "step": 500
+    },
+    {
+      "epoch": 0.018983454616515606,
+      "grad_norm": 1.9274375900677172,
+      "learning_rate": 9.490137700037215e-08,
+      "logits/chosen": -2.5,
+      "logits/rejected": -2.328125,
+      "logps/chosen": -194.0,
+      "logps/rejected": -170.0,
+      "loss": 0.6852,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.011474609375,
+      "rewards/margins": 0.01483154296875,
+      "rewards/rejected": -0.0033111572265625,
+      "step": 510
+    },
+    {
+      "epoch": 0.01935567921683944,
+      "grad_norm": 1.9795674434848012,
+      "learning_rate": 9.676218831410495e-08,
+      "logits/chosen": -2.515625,
+      "logits/rejected": -2.46875,
+      "logps/chosen": -200.0,
+      "logps/rejected": -182.0,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0106201171875,
+      "rewards/margins": 0.01458740234375,
+      "rewards/rejected": -0.003997802734375,
+      "step": 520
+    },
+    {
+      "epoch": 0.019727903817163276,
+      "grad_norm": 1.7867038683646506,
+      "learning_rate": 9.862299962783774e-08,
+      "logits/chosen": -2.671875,
+      "logits/rejected": -2.359375,
+      "logps/chosen": -191.0,
+      "logps/rejected": -174.0,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.0107421875,
+      "rewards/margins": 0.01385498046875,
+      "rewards/rejected": -0.0030975341796875,
+      "step": 530
+    },
+    {
+      "epoch": 0.020100128417487113,
+      "grad_norm": 2.321635422204204,
+      "learning_rate": 1.0048381094157052e-07,
+      "logits/chosen": -2.671875,
+      "logits/rejected": -2.390625,
+      "logps/chosen": -180.0,
+      "logps/rejected": -166.0,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.006866455078125,
+      "rewards/margins": 0.016845703125,
+      "rewards/rejected": -0.00994873046875,
+      "step": 540
+    },
+    {
+      "epoch": 0.020472353017810946,
+      "grad_norm": 1.8781699668032819,
+      "learning_rate": 1.0234462225530331e-07,
+      "logits/chosen": -2.59375,
+      "logits/rejected": -2.4375,
+      "logps/chosen": -191.0,
+      "logps/rejected": -186.0,
+      "loss": 0.6823,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.00433349609375,
+      "rewards/margins": 0.0206298828125,
+      "rewards/rejected": -0.0162353515625,
+      "step": 550
+    },
+    {
+      "epoch": 0.020844577618134782,
+      "grad_norm": 1.918664286905337,
+      "learning_rate": 1.042054335690361e-07,
+      "logits/chosen": -2.59375,
+      "logits/rejected": -2.421875,
+      "logps/chosen": -179.0,
+      "logps/rejected": -147.0,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0087890625,
+      "rewards/margins": 0.01458740234375,
+      "rewards/rejected": -0.00579833984375,
+      "step": 560
+    },
+    {
+      "epoch": 0.02121680221845862,
+      "grad_norm": 1.8305005682367024,
+      "learning_rate": 1.0606624488276889e-07,
+      "logits/chosen": -2.5625,
+      "logits/rejected": -2.484375,
+      "logps/chosen": -177.0,
+      "logps/rejected": -168.0,
+      "loss": 0.6809,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0096435546875,
+      "rewards/margins": 0.020263671875,
+      "rewards/rejected": -0.01055908203125,
+      "step": 570
+    },
+    {
+      "epoch": 0.021589026818782452,
+      "grad_norm": 1.8811354607810746,
+      "learning_rate": 1.0792705619650167e-07,
+      "logits/chosen": -2.71875,
+      "logits/rejected": -2.5,
+      "logps/chosen": -188.0,
+      "logps/rejected": -180.0,
+      "loss": 0.681,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.00518798828125,
+      "rewards/margins": 0.0255126953125,
+      "rewards/rejected": -0.020263671875,
+      "step": 580
+    },
+    {
+      "epoch": 0.02196125141910629,
+      "grad_norm": 1.8518048043763524,
+      "learning_rate": 1.0978786751023446e-07,
+      "logits/chosen": -2.640625,
+      "logits/rejected": -2.265625,
+      "logps/chosen": -180.0,
+      "logps/rejected": -182.0,
+      "loss": 0.6793,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.015625,
+      "rewards/margins": 0.0286865234375,
+      "rewards/rejected": -0.01312255859375,
+      "step": 590
+    },
+    {
+      "epoch": 0.022333476019430126,
+      "grad_norm": 1.959429186032332,
+      "learning_rate": 1.1164867882396724e-07,
+      "logits/chosen": -2.671875,
+      "logits/rejected": -2.6875,
+      "logps/chosen": -203.0,
+      "logps/rejected": -179.0,
+      "loss": 0.6792,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.0062255859375,
+      "rewards/margins": 0.030517578125,
+      "rewards/rejected": -0.0242919921875,
+      "step": 600
+    },
+    {
+      "epoch": 0.02270570061975396,
+      "grad_norm": 1.6860387154068144,
+      "learning_rate": 1.1350949013770003e-07,
+      "logits/chosen": -2.65625,
+      "logits/rejected": -2.453125,
+      "logps/chosen": -171.0,
+      "logps/rejected": -149.0,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.00469970703125,
+      "rewards/margins": 0.03271484375,
+      "rewards/rejected": -0.0279541015625,
+      "step": 610
+    },
+    {
+      "epoch": 0.023077925220077795,
+      "grad_norm": 1.9592725878142587,
+      "learning_rate": 1.1537030145143283e-07,
+      "logits/chosen": -2.703125,
+      "logits/rejected": -2.46875,
+      "logps/chosen": -170.0,
+      "logps/rejected": -165.0,
+      "loss": 0.6775,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.007049560546875,
+      "rewards/margins": 0.035400390625,
+      "rewards/rejected": -0.042236328125,
+      "step": 620
+    },
+    {
+      "epoch": 0.023450149820401632,
+      "grad_norm": 1.8802838684827163,
+      "learning_rate": 1.1723111276516561e-07,
+      "logits/chosen": -2.609375,
+      "logits/rejected": -2.34375,
+      "logps/chosen": -176.0,
+      "logps/rejected": -159.0,
+      "loss": 0.675,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.004730224609375,
+      "rewards/margins": 0.034423828125,
+      "rewards/rejected": -0.02978515625,
+      "step": 630
+    },
+    {
+      "epoch": 0.023822374420725465,
+      "grad_norm": 2.05762157911921,
+      "learning_rate": 1.190919240788984e-07,
+      "logits/chosen": -2.671875,
+      "logits/rejected": -2.421875,
+      "logps/chosen": -171.0,
+      "logps/rejected": -178.0,
+      "loss": 0.6758,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.005828857421875,
+      "rewards/margins": 0.0419921875,
+      "rewards/rejected": -0.047607421875,
+      "step": 640
+    },
+    {
+      "epoch": 0.024194599021049302,
+      "grad_norm": 1.9890772989214731,
+      "learning_rate": 1.2095273539263117e-07,
+      "logits/chosen": -2.625,
+      "logits/rejected": -2.421875,
+      "logps/chosen": -192.0,
+      "logps/rejected": -180.0,
+      "loss": 0.6754,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.01953125,
+      "rewards/margins": 0.038330078125,
+      "rewards/rejected": -0.057861328125,
+      "step": 650
+    },
+    {
+      "epoch": 0.024566823621373135,
+      "grad_norm": 1.9761320863608818,
+      "learning_rate": 1.2281354670636396e-07,
+      "logits/chosen": -2.671875,
+      "logits/rejected": -2.46875,
+      "logps/chosen": -193.0,
+      "logps/rejected": -180.0,
+      "loss": 0.673,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.025634765625,
+      "rewards/margins": 0.039306640625,
+      "rewards/rejected": -0.06494140625,
+      "step": 660
+    },
+    {
+      "epoch": 0.02493904822169697,
+      "grad_norm": 2.0911143498909963,
+      "learning_rate": 1.2467435802009675e-07,
+      "logits/chosen": -2.65625,
+      "logits/rejected": -2.515625,
+      "logps/chosen": -192.0,
+      "logps/rejected": -182.0,
+      "loss": 0.6727,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.035400390625,
+      "rewards/margins": 0.05078125,
+      "rewards/rejected": -0.0859375,
+      "step": 670
+    },
+    {
+      "epoch": 0.02531127282202081,
+      "grad_norm": 2.1226219039408925,
+      "learning_rate": 1.2653516933382955e-07,
+      "logits/chosen": -2.75,
+      "logits/rejected": -2.734375,
+      "logps/chosen": -215.0,
+      "logps/rejected": -187.0,
+      "loss": 0.6691,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.043212890625,
+      "rewards/margins": 0.0299072265625,
+      "rewards/rejected": -0.0732421875,
+      "step": 680
+    },
+    {
+      "epoch": 0.02568349742234464,
+      "grad_norm": 2.0052089494235776,
+      "learning_rate": 1.2839598064756231e-07,
+      "logits/chosen": -2.75,
+      "logits/rejected": -2.671875,
+      "logps/chosen": -191.0,
+      "logps/rejected": -171.0,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.05078125,
+      "rewards/margins": 0.050048828125,
+      "rewards/rejected": -0.10107421875,
+      "step": 690
+    },
+    {
+      "epoch": 0.026055722022668478,
+      "grad_norm": 1.9629218976934022,
+      "learning_rate": 1.3025679196129513e-07,
+      "logits/chosen": -2.765625,
+      "logits/rejected": -2.546875,
+      "logps/chosen": -174.0,
+      "logps/rejected": -175.0,
+      "loss": 0.6673,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.0615234375,
+      "rewards/margins": 0.045654296875,
+      "rewards/rejected": -0.107421875,
+      "step": 700
+    },
+    {
+      "epoch": 0.026427946622992315,
+      "grad_norm": 2.0838379595135574,
+      "learning_rate": 1.321176032750279e-07,
+      "logits/chosen": -2.75,
+      "logits/rejected": -2.59375,
+      "logps/chosen": -186.0,
+      "logps/rejected": -172.0,
+      "loss": 0.6667,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.06005859375,
+      "rewards/margins": 0.0478515625,
+      "rewards/rejected": -0.10791015625,
+      "step": 710
+    },
+    {
+      "epoch": 0.026800171223316148,
+      "grad_norm": 2.06780227169773,
+      "learning_rate": 1.339784145887607e-07,
+      "logits/chosen": -2.78125,
+      "logits/rejected": -2.578125,
+      "logps/chosen": -172.0,
+      "logps/rejected": -168.0,
+      "loss": 0.665,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.06396484375,
+      "rewards/margins": 0.07568359375,
+      "rewards/rejected": -0.1396484375,
+      "step": 720
+    },
+    {
+      "epoch": 0.027172395823639985,
+      "grad_norm": 2.0958108327517255,
+      "learning_rate": 1.3583922590249346e-07,
+      "logits/chosen": -2.75,
+      "logits/rejected": -2.546875,
+      "logps/chosen": -194.0,
+      "logps/rejected": -192.0,
+      "loss": 0.6608,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.07861328125,
+      "rewards/margins": 0.09033203125,
+      "rewards/rejected": -0.1689453125,
+      "step": 730
+    },
+    {
+      "epoch": 0.02754462042396382,
+      "grad_norm": 1.924679809717578,
+      "learning_rate": 1.3770003721622628e-07,
+      "logits/chosen": -2.6875,
+      "logits/rejected": -2.609375,
+      "logps/chosen": -187.0,
+      "logps/rejected": -158.0,
+      "loss": 0.6573,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.10791015625,
+      "rewards/margins": 0.0595703125,
+      "rewards/rejected": -0.1669921875,
+      "step": 740
+    },
+    {
+      "epoch": 0.027916845024287654,
+      "grad_norm": 1.9997145682786945,
+      "learning_rate": 1.3956084852995905e-07,
+      "logits/chosen": -2.8125,
+      "logits/rejected": -2.640625,
+      "logps/chosen": -226.0,
+      "logps/rejected": -191.0,
+      "loss": 0.6542,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.11572265625,
+      "rewards/margins": 0.09765625,
+      "rewards/rejected": -0.2138671875,
+      "step": 750
+    },
+    {
+      "epoch": 0.02828906962461149,
+      "grad_norm": 2.1187152286547537,
+      "learning_rate": 1.4142165984369184e-07,
+      "logits/chosen": -2.78125,
+      "logits/rejected": -2.515625,
+      "logps/chosen": -187.0,
+      "logps/rejected": -203.0,
+      "loss": 0.6512,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1357421875,
+      "rewards/margins": 0.10546875,
+      "rewards/rejected": -0.2412109375,
+      "step": 760
+    },
+    {
+      "epoch": 0.028661294224935328,
+      "grad_norm": 2.070753688487636,
+      "learning_rate": 1.432824711574246e-07,
+      "logits/chosen": -2.75,
+      "logits/rejected": -2.640625,
+      "logps/chosen": -194.0,
+      "logps/rejected": -179.0,
+      "loss": 0.6522,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1669921875,
+      "rewards/margins": 0.0859375,
+      "rewards/rejected": -0.251953125,
+      "step": 770
+    },
+    {
+      "epoch": 0.02903351882525916,
+      "grad_norm": 2.1226967473795466,
+      "learning_rate": 1.4514328247115743e-07,
+      "logits/chosen": -2.796875,
+      "logits/rejected": -2.734375,
+      "logps/chosen": -203.0,
+      "logps/rejected": -210.0,
+      "loss": 0.6444,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.2060546875,
+      "rewards/margins": 0.119140625,
+      "rewards/rejected": -0.326171875,
+      "step": 780
+    },
+    {
+      "epoch": 0.029405743425582997,
+      "grad_norm": 2.2539166318970545,
+      "learning_rate": 1.470040937848902e-07,
+      "logits/chosen": -2.890625,
+      "logits/rejected": -2.671875,
+      "logps/chosen": -217.0,
+      "logps/rejected": -225.0,
+      "loss": 0.6374,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.2294921875,
+      "rewards/margins": 0.1181640625,
+      "rewards/rejected": -0.34765625,
+      "step": 790
+    },
+    {
+      "epoch": 0.02977796802590683,
+      "grad_norm": 2.210526412343009,
+      "learning_rate": 1.48864905098623e-07,
+      "logits/chosen": -2.84375,
+      "logits/rejected": -2.625,
+      "logps/chosen": -237.0,
+      "logps/rejected": -240.0,
+      "loss": 0.6451,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.203125,
+      "rewards/margins": 0.2060546875,
+      "rewards/rejected": -0.408203125,
+      "step": 800
+    },
+    {
+      "epoch": 0.030150192626230667,
+      "grad_norm": 2.3093810768281626,
+      "learning_rate": 1.5072571641235578e-07,
+      "logits/chosen": -2.890625,
+      "logits/rejected": -2.71875,
+      "logps/chosen": -222.0,
+      "logps/rejected": -224.0,
+      "loss": 0.6382,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3046875,
+      "rewards/margins": 0.142578125,
+      "rewards/rejected": -0.4453125,
+      "step": 810
+    },
+    {
+      "epoch": 0.030522417226554504,
+      "grad_norm": 2.2902573111551523,
+      "learning_rate": 1.5258652772608857e-07,
+      "logits/chosen": -2.859375,
+      "logits/rejected": -2.671875,
+      "logps/chosen": -202.0,
+      "logps/rejected": -198.0,
+      "loss": 0.6368,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.35546875,
+      "rewards/margins": 0.0966796875,
+      "rewards/rejected": -0.451171875,
+      "step": 820
+    },
+    {
+      "epoch": 0.030894641826878337,
+      "grad_norm": 2.4108673549172104,
+      "learning_rate": 1.5444733903982134e-07,
+      "logits/chosen": -2.890625,
+      "logits/rejected": -2.625,
+      "logps/chosen": -206.0,
+      "logps/rejected": -209.0,
+      "loss": 0.6277,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.359375,
+      "rewards/margins": 0.158203125,
+      "rewards/rejected": -0.515625,
+      "step": 830
+    },
+    {
+      "epoch": 0.031266866427202174,
+      "grad_norm": 2.3517127798546222,
+      "learning_rate": 1.5630815035355413e-07,
+      "logits/chosen": -2.890625,
+      "logits/rejected": -2.65625,
+      "logps/chosen": -218.0,
+      "logps/rejected": -216.0,
+      "loss": 0.6243,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.404296875,
+      "rewards/margins": 0.2041015625,
+      "rewards/rejected": -0.609375,
+      "step": 840
+    },
+    {
+      "epoch": 0.03163909102752601,
+      "grad_norm": 2.2021853287907036,
+      "learning_rate": 1.5816896166728693e-07,
+      "logits/chosen": -2.984375,
+      "logits/rejected": -2.734375,
+      "logps/chosen": -197.0,
+      "logps/rejected": -207.0,
+      "loss": 0.6235,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.41796875,
+      "rewards/margins": 0.1904296875,
+      "rewards/rejected": -0.609375,
+      "step": 850
+    },
+    {
+      "epoch": 0.03201131562784985,
+      "grad_norm": 2.4382852409843205,
+      "learning_rate": 1.6002977298101972e-07,
+      "logits/chosen": -2.828125,
+      "logits/rejected": -2.75,
+      "logps/chosen": -231.0,
+      "logps/rejected": -204.0,
+      "loss": 0.6281,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.42578125,
+      "rewards/margins": 0.1240234375,
+      "rewards/rejected": -0.546875,
+      "step": 860
+    },
+    {
+      "epoch": 0.03238354022817368,
+      "grad_norm": 2.6006060997416434,
+      "learning_rate": 1.618905842947525e-07,
+      "logits/chosen": -3.046875,
+      "logits/rejected": -2.828125,
+      "logps/chosen": -238.0,
+      "logps/rejected": -245.0,
+      "loss": 0.6242,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.4765625,
+      "rewards/margins": 0.19921875,
+      "rewards/rejected": -0.67578125,
+      "step": 870
+    },
+    {
+      "epoch": 0.03275576482849751,
+      "grad_norm": 2.3624817402980183,
+      "learning_rate": 1.637513956084853e-07,
+      "logits/chosen": -2.84375,
+      "logits/rejected": -2.640625,
+      "logps/chosen": -234.0,
+      "logps/rejected": -224.0,
+      "loss": 0.6268,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.50390625,
+      "rewards/margins": 0.158203125,
+      "rewards/rejected": -0.6640625,
+      "step": 880
+    },
+    {
+      "epoch": 0.03312798942882135,
+      "grad_norm": 2.6704496844783945,
+      "learning_rate": 1.6561220692221807e-07,
+      "logits/chosen": -2.875,
+      "logits/rejected": -2.8125,
+      "logps/chosen": -221.0,
+      "logps/rejected": -223.0,
+      "loss": 0.616,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.474609375,
+      "rewards/margins": 0.2001953125,
+      "rewards/rejected": -0.67578125,
+      "step": 890
+    },
+    {
+      "epoch": 0.03350021402914519,
+      "grad_norm": 2.8514281391575014,
+      "learning_rate": 1.6747301823595087e-07,
+      "logits/chosen": -3.015625,
+      "logits/rejected": -2.84375,
+      "logps/chosen": -232.0,
+      "logps/rejected": -238.0,
+      "loss": 0.6084,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.53515625,
+      "rewards/margins": 0.2451171875,
+      "rewards/rejected": -0.78125,
+      "step": 900
+    },
+    {
+      "epoch": 0.03387243862946902,
+      "grad_norm": 2.731006607142271,
+      "learning_rate": 1.6933382954968363e-07,
+      "logits/chosen": -3.0,
+      "logits/rejected": -2.828125,
+      "logps/chosen": -244.0,
+      "logps/rejected": -262.0,
+      "loss": 0.5866,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.59765625,
+      "rewards/margins": 0.33984375,
+      "rewards/rejected": -0.9375,
+      "step": 910
+    },
+    {
+      "epoch": 0.03424466322979286,
+      "grad_norm": 2.7206444933470157,
+      "learning_rate": 1.7119464086341645e-07,
+      "logits/chosen": -3.09375,
+      "logits/rejected": -3.0,
+      "logps/chosen": -241.0,
+      "logps/rejected": -249.0,
+      "loss": 0.5963,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.65234375,
+      "rewards/margins": 0.2470703125,
+      "rewards/rejected": -0.8984375,
+      "step": 920
+    },
+    {
+      "epoch": 0.03461688783011669,
+      "grad_norm": 2.772360417113917,
+      "learning_rate": 1.7305545217714922e-07,
+      "logits/chosen": -2.953125,
+      "logits/rejected": -2.953125,
+      "logps/chosen": -252.0,
+      "logps/rejected": -248.0,
+      "loss": 0.597,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.609375,
+      "rewards/margins": 0.279296875,
+      "rewards/rejected": -0.88671875,
+      "step": 930
+    },
+    {
+      "epoch": 0.034989112430440526,
+      "grad_norm": 3.3559248071850027,
+      "learning_rate": 1.7491626349088201e-07,
+      "logits/chosen": -3.0625,
+      "logits/rejected": -2.953125,
+      "logps/chosen": -252.0,
+      "logps/rejected": -264.0,
+      "loss": 0.594,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.6953125,
+      "rewards/margins": 0.263671875,
+      "rewards/rejected": -0.95703125,
+      "step": 940
+    },
+    {
+      "epoch": 0.03536133703076436,
+      "grad_norm": 3.254191846218812,
+      "learning_rate": 1.767770748046148e-07,
+      "logits/chosen": -3.15625,
+      "logits/rejected": -2.984375,
+      "logps/chosen": -255.0,
+      "logps/rejected": -276.0,
+      "loss": 0.5861,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.71875,
+      "rewards/margins": 0.306640625,
+      "rewards/rejected": -1.0234375,
+      "step": 950
+    },
+    {
+      "epoch": 0.0357335616310882,
+      "grad_norm": 3.6457190559943125,
+      "learning_rate": 1.786378861183476e-07,
+      "logits/chosen": -3.09375,
+      "logits/rejected": -2.96875,
+      "logps/chosen": -251.0,
+      "logps/rejected": -268.0,
+      "loss": 0.5851,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.75390625,
+      "rewards/margins": 0.310546875,
+      "rewards/rejected": -1.0625,
+      "step": 960
+    },
+    {
+      "epoch": 0.036105786231412036,
+      "grad_norm": 3.210391264144676,
+      "learning_rate": 1.8049869743208037e-07,
+      "logits/chosen": -3.125,
+      "logits/rejected": -3.0625,
+      "logps/chosen": -268.0,
+      "logps/rejected": -274.0,
+      "loss": 0.5865,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.7421875,
+      "rewards/margins": 0.2578125,
+      "rewards/rejected": -1.0,
+      "step": 970
+    },
+    {
+      "epoch": 0.03647801083173587,
+      "grad_norm": 3.2479937962128496,
+      "learning_rate": 1.8235950874581316e-07,
+      "logits/chosen": -3.1875,
+      "logits/rejected": -3.09375,
+      "logps/chosen": -268.0,
+      "logps/rejected": -278.0,
+      "loss": 0.5786,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.7109375,
+      "rewards/margins": 0.35546875,
+      "rewards/rejected": -1.0703125,
+      "step": 980
+    },
+    {
+      "epoch": 0.0368502354320597,
+      "grad_norm": 3.2026336219407185,
+      "learning_rate": 1.8422032005954595e-07,
+      "logits/chosen": -3.15625,
+      "logits/rejected": -3.046875,
+      "logps/chosen": -278.0,
+      "logps/rejected": -300.0,
+      "loss": 0.5763,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.76171875,
+      "rewards/margins": 0.375,
+      "rewards/rejected": -1.1328125,
+      "step": 990
+    },
+    {
+      "epoch": 0.03722246003238354,
+      "grad_norm": 3.3934216128285093,
+      "learning_rate": 1.8608113137327875e-07,
+      "logits/chosen": -3.359375,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -270.0,
+      "logps/rejected": -318.0,
+      "loss": 0.5695,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.82421875,
+      "rewards/margins": 0.3828125,
+      "rewards/rejected": -1.2109375,
+      "step": 1000
+    },
+    {
+      "epoch": 0.037594684632707376,
+      "grad_norm": 4.811197922294435,
+      "learning_rate": 1.8794194268701151e-07,
+      "logits/chosen": -3.28125,
+      "logits/rejected": -3.15625,
+      "logps/chosen": -270.0,
+      "logps/rejected": -302.0,
+      "loss": 0.5625,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.8359375,
+      "rewards/margins": 0.419921875,
+      "rewards/rejected": -1.2578125,
+      "step": 1010
+    },
+    {
+      "epoch": 0.03796690923303121,
+      "grad_norm": 3.557646799500176,
+      "learning_rate": 1.898027540007443e-07,
+      "logits/chosen": -3.34375,
+      "logits/rejected": -3.21875,
+      "logps/chosen": -274.0,
+      "logps/rejected": -292.0,
+      "loss": 0.5716,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.91015625,
+      "rewards/margins": 0.5,
+      "rewards/rejected": -1.4140625,
+      "step": 1020
+    },
+    {
+      "epoch": 0.03833913383335505,
+      "grad_norm": 4.040727373831442,
+      "learning_rate": 1.916635653144771e-07,
+      "logits/chosen": -3.359375,
+      "logits/rejected": -3.203125,
+      "logps/chosen": -272.0,
+      "logps/rejected": -318.0,
+      "loss": 0.5626,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.921875,
+      "rewards/margins": 0.50390625,
+      "rewards/rejected": -1.4296875,
+      "step": 1030
+    },
+    {
+      "epoch": 0.03871135843367888,
+      "grad_norm": 3.476235870062827,
+      "learning_rate": 1.935243766282099e-07,
+      "logits/chosen": -3.34375,
+      "logits/rejected": -3.0625,
+      "logps/chosen": -262.0,
+      "logps/rejected": -302.0,
+      "loss": 0.5734,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.8984375,
+      "rewards/margins": 0.44921875,
+      "rewards/rejected": -1.34375,
+      "step": 1040
+    },
+    {
+      "epoch": 0.039083583034002715,
+      "grad_norm": 3.6266634291480746,
+      "learning_rate": 1.9538518794194266e-07,
+      "logits/chosen": -3.25,
+      "logits/rejected": -3.15625,
+      "logps/chosen": -276.0,
+      "logps/rejected": -330.0,
+      "loss": 0.5739,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.9140625,
+      "rewards/margins": 0.4453125,
+      "rewards/rejected": -1.359375,
+      "step": 1050
+    },
+    {
+      "epoch": 0.03945580763432655,
+      "grad_norm": 3.6956901467135386,
+      "learning_rate": 1.9724599925567548e-07,
+      "logits/chosen": -3.359375,
+      "logits/rejected": -3.203125,
+      "logps/chosen": -266.0,
+      "logps/rejected": -304.0,
+      "loss": 0.563,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.88671875,
+      "rewards/margins": 0.427734375,
+      "rewards/rejected": -1.3125,
+      "step": 1060
+    },
+    {
+      "epoch": 0.03982803223465039,
+      "grad_norm": 4.964415877760095,
+      "learning_rate": 1.9910681056940825e-07,
+      "logits/chosen": -3.328125,
+      "logits/rejected": -3.1875,
+      "logps/chosen": -282.0,
+      "logps/rejected": -344.0,
+      "loss": 0.5557,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.953125,
+      "rewards/margins": 0.57421875,
+      "rewards/rejected": -1.5234375,
+      "step": 1070
+    },
+    {
+      "epoch": 0.040200256834974225,
+      "grad_norm": 4.328769581569679,
+      "learning_rate": 2.0096762188314104e-07,
+      "logits/chosen": -3.296875,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -298.0,
+      "logps/rejected": -310.0,
+      "loss": 0.565,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0078125,
+      "rewards/margins": 0.419921875,
+      "rewards/rejected": -1.4296875,
+      "step": 1080
+    },
+    {
+      "epoch": 0.04057248143529806,
+      "grad_norm": 4.561685664853453,
+      "learning_rate": 2.028284331968738e-07,
+      "logits/chosen": -3.40625,
+      "logits/rejected": -3.25,
+      "logps/chosen": -288.0,
+      "logps/rejected": -320.0,
+      "loss": 0.566,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.94921875,
+      "rewards/margins": 0.447265625,
+      "rewards/rejected": -1.390625,
+      "step": 1090
+    },
+    {
+      "epoch": 0.04094470603562189,
+      "grad_norm": 4.33784550368845,
+      "learning_rate": 2.0468924451060663e-07,
+      "logits/chosen": -3.359375,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -276.0,
+      "logps/rejected": -314.0,
+      "loss": 0.5366,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.97265625,
+      "rewards/margins": 0.5859375,
+      "rewards/rejected": -1.5546875,
+      "step": 1100
+    },
+    {
+      "epoch": 0.04131693063594573,
+      "grad_norm": 4.505216621962622,
+      "learning_rate": 2.065500558243394e-07,
+      "logits/chosen": -3.28125,
+      "logits/rejected": -3.1875,
+      "logps/chosen": -276.0,
+      "logps/rejected": -324.0,
+      "loss": 0.5354,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.0234375,
+      "rewards/margins": 0.62109375,
+      "rewards/rejected": -1.640625,
+      "step": 1110
+    },
+    {
+      "epoch": 0.041689155236269565,
+      "grad_norm": 4.277688507231565,
+      "learning_rate": 2.084108671380722e-07,
+      "logits/chosen": -3.359375,
+      "logits/rejected": -3.203125,
+      "logps/chosen": -280.0,
+      "logps/rejected": -306.0,
+      "loss": 0.5761,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.0859375,
+      "rewards/margins": 0.482421875,
+      "rewards/rejected": -1.5703125,
+      "step": 1120
+    },
+    {
+      "epoch": 0.0420613798365934,
+      "grad_norm": 4.88849420841731,
+      "learning_rate": 2.1027167845180498e-07,
+      "logits/chosen": -3.40625,
+      "logits/rejected": -3.15625,
+      "logps/chosen": -272.0,
+      "logps/rejected": -328.0,
+      "loss": 0.5408,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.87890625,
+      "rewards/margins": 0.66796875,
+      "rewards/rejected": -1.546875,
+      "step": 1130
+    },
+    {
+      "epoch": 0.04243360443691724,
+      "grad_norm": 4.6659299965403624,
+      "learning_rate": 2.1213248976553777e-07,
+      "logits/chosen": -3.328125,
+      "logits/rejected": -3.21875,
+      "logps/chosen": -290.0,
+      "logps/rejected": -312.0,
+      "loss": 0.5596,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0859375,
+      "rewards/margins": 0.431640625,
+      "rewards/rejected": -1.515625,
+      "step": 1140
+    },
+    {
+      "epoch": 0.04280582903724107,
+      "grad_norm": 4.792599988977468,
+      "learning_rate": 2.1399330107927054e-07,
+      "logits/chosen": -3.421875,
+      "logits/rejected": -3.15625,
+      "logps/chosen": -274.0,
+      "logps/rejected": -312.0,
+      "loss": 0.5505,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.125,
+      "rewards/margins": 0.48046875,
+      "rewards/rejected": -1.6015625,
+      "step": 1150
+    },
+    {
+      "epoch": 0.043178053637564905,
+      "grad_norm": 4.90368027547217,
+      "learning_rate": 2.1585411239300333e-07,
+      "logits/chosen": -3.5,
+      "logits/rejected": -3.34375,
+      "logps/chosen": -294.0,
+      "logps/rejected": -334.0,
+      "loss": 0.5472,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.078125,
+      "rewards/margins": 0.515625,
+      "rewards/rejected": -1.59375,
+      "step": 1160
+    },
+    {
+      "epoch": 0.04355027823788874,
+      "grad_norm": 4.668380375082959,
+      "learning_rate": 2.1771492370673613e-07,
+      "logits/chosen": -3.34375,
+      "logits/rejected": -3.15625,
+      "logps/chosen": -274.0,
+      "logps/rejected": -320.0,
+      "loss": 0.5541,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.98828125,
+      "rewards/margins": 0.53125,
+      "rewards/rejected": -1.515625,
+      "step": 1170
+    },
+    {
+      "epoch": 0.04392250283821258,
+      "grad_norm": 5.313103441687499,
+      "learning_rate": 2.1957573502046892e-07,
+      "logits/chosen": -3.375,
+      "logits/rejected": -3.03125,
+      "logps/chosen": -272.0,
+      "logps/rejected": -326.0,
+      "loss": 0.5322,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.9453125,
+      "rewards/margins": 0.55859375,
+      "rewards/rejected": -1.5078125,
+      "step": 1180
+    },
+    {
+      "epoch": 0.044294727438536415,
+      "grad_norm": 4.434797396632511,
+      "learning_rate": 2.214365463342017e-07,
+      "logits/chosen": -3.34375,
+      "logits/rejected": -3.25,
+      "logps/chosen": -294.0,
+      "logps/rejected": -326.0,
+      "loss": 0.5541,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.1015625,
+      "rewards/margins": 0.53125,
+      "rewards/rejected": -1.6328125,
+      "step": 1190
+    },
+    {
+      "epoch": 0.04466695203886025,
+      "grad_norm": 5.283461502011228,
+      "learning_rate": 2.2329735764793448e-07,
+      "logits/chosen": -3.5,
+      "logits/rejected": -3.34375,
+      "logps/chosen": -286.0,
+      "logps/rejected": -338.0,
+      "loss": 0.5415,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.0859375,
+      "rewards/margins": 0.57421875,
+      "rewards/rejected": -1.65625,
+      "step": 1200
+    },
+    {
+      "epoch": 0.04503917663918408,
+      "grad_norm": 5.146962919353201,
+      "learning_rate": 2.2515816896166727e-07,
+      "logits/chosen": -3.4375,
+      "logits/rejected": -3.328125,
+      "logps/chosen": -292.0,
+      "logps/rejected": -324.0,
+      "loss": 0.5497,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.1015625,
+      "rewards/margins": 0.453125,
+      "rewards/rejected": -1.5546875,
+      "step": 1210
+    },
+    {
+      "epoch": 0.04541140123950792,
+      "grad_norm": 5.646450830202028,
+      "learning_rate": 2.2701898027540007e-07,
+      "logits/chosen": -3.359375,
+      "logits/rejected": -3.296875,
+      "logps/chosen": -294.0,
+      "logps/rejected": -334.0,
+      "loss": 0.5337,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1171875,
+      "rewards/margins": 0.42578125,
+      "rewards/rejected": -1.546875,
+      "step": 1220
+    },
+    {
+      "epoch": 0.045783625839831754,
+      "grad_norm": 4.85076328578327,
+      "learning_rate": 2.2887979158913283e-07,
+      "logits/chosen": -3.40625,
+      "logits/rejected": -3.328125,
+      "logps/chosen": -302.0,
+      "logps/rejected": -344.0,
+      "loss": 0.5377,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.1015625,
+      "rewards/margins": 0.62890625,
+      "rewards/rejected": -1.7265625,
+      "step": 1230
+    },
+    {
+      "epoch": 0.04615585044015559,
+      "grad_norm": 4.281822784521038,
+      "learning_rate": 2.3074060290286565e-07,
+      "logits/chosen": -3.390625,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -316.0,
+      "logps/rejected": -346.0,
+      "loss": 0.5678,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.1484375,
+      "rewards/margins": 0.423828125,
+      "rewards/rejected": -1.5703125,
+      "step": 1240
+    },
+    {
+      "epoch": 0.04652807504047943,
+      "grad_norm": 4.683379355406632,
+      "learning_rate": 2.3260141421659842e-07,
+      "logits/chosen": -3.390625,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -308.0,
+      "logps/rejected": -354.0,
+      "loss": 0.5461,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.109375,
+      "rewards/margins": 0.55859375,
+      "rewards/rejected": -1.6640625,
+      "step": 1250
+    },
+    {
+      "epoch": 0.046900299640803264,
+      "grad_norm": 7.902170570489184,
+      "learning_rate": 2.3446222553033121e-07,
+      "logits/chosen": -3.34375,
+      "logits/rejected": -3.296875,
+      "logps/chosen": -310.0,
+      "logps/rejected": -338.0,
+      "loss": 0.565,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.171875,
+      "rewards/margins": 0.45703125,
+      "rewards/rejected": -1.625,
+      "step": 1260
+    },
+    {
+      "epoch": 0.047272524241127094,
+      "grad_norm": 4.587829490700819,
+      "learning_rate": 2.3632303684406398e-07,
+      "logits/chosen": -3.3125,
+      "logits/rejected": -3.25,
+      "logps/chosen": -314.0,
+      "logps/rejected": -322.0,
+      "loss": 0.5364,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.1484375,
+      "rewards/margins": 0.46875,
+      "rewards/rejected": -1.6171875,
+      "step": 1270
+    },
+    {
+      "epoch": 0.04764474884145093,
+      "grad_norm": 4.9856893229597326,
+      "learning_rate": 2.381838481577968e-07,
+      "logits/chosen": -3.40625,
+      "logits/rejected": -3.359375,
+      "logps/chosen": -300.0,
+      "logps/rejected": -332.0,
+      "loss": 0.5293,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.078125,
+      "rewards/margins": 0.59765625,
+      "rewards/rejected": -1.671875,
+      "step": 1280
+    },
+    {
+      "epoch": 0.04801697344177477,
+      "grad_norm": 4.580889853163873,
+      "learning_rate": 2.4004465947152957e-07,
+      "logits/chosen": -3.40625,
+      "logits/rejected": -3.265625,
+      "logps/chosen": -318.0,
+      "logps/rejected": -352.0,
+      "loss": 0.5359,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.234375,
+      "rewards/margins": 0.58984375,
+      "rewards/rejected": -1.8203125,
+      "step": 1290
+    },
+    {
+      "epoch": 0.048389198042098604,
+      "grad_norm": 6.7019225186330456,
+      "learning_rate": 2.4190547078526233e-07,
+      "logits/chosen": -3.34375,
+      "logits/rejected": -3.40625,
+      "logps/chosen": -320.0,
+      "logps/rejected": -372.0,
+      "loss": 0.5438,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.2109375,
+      "rewards/margins": 0.7421875,
+      "rewards/rejected": -1.953125,
+      "step": 1300
+    },
+    {
+      "epoch": 0.04876142264242244,
+      "grad_norm": 5.250533761484681,
+      "learning_rate": 2.4376628209899515e-07,
+      "logits/chosen": -3.34375,
+      "logits/rejected": -3.28125,
+      "logps/chosen": -286.0,
+      "logps/rejected": -322.0,
+      "loss": 0.5272,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.0859375,
+      "rewards/margins": 0.66015625,
+      "rewards/rejected": -1.75,
+      "step": 1310
+    },
+    {
+      "epoch": 0.04913364724274627,
+      "grad_norm": 6.263136875889413,
+      "learning_rate": 2.456270934127279e-07,
+      "logits/chosen": -3.5625,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -314.0,
+      "logps/rejected": -360.0,
+      "loss": 0.527,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.2265625,
+      "rewards/margins": 0.6328125,
+      "rewards/rejected": -1.8515625,
+      "step": 1320
+    },
+    {
+      "epoch": 0.04950587184307011,
+      "grad_norm": 6.531224260550454,
+      "learning_rate": 2.4748790472646074e-07,
+      "logits/chosen": -3.46875,
+      "logits/rejected": -3.359375,
+      "logps/chosen": -310.0,
+      "logps/rejected": -340.0,
+      "loss": 0.5573,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.3125,
+      "rewards/margins": 0.40234375,
+      "rewards/rejected": -1.71875,
+      "step": 1330
+    },
+    {
+      "epoch": 0.04987809644339394,
+      "grad_norm": 5.769891227252786,
+      "learning_rate": 2.493487160401935e-07,
+      "logits/chosen": -3.53125,
+      "logits/rejected": -3.21875,
+      "logps/chosen": -288.0,
+      "logps/rejected": -354.0,
+      "loss": 0.5312,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.0859375,
+      "rewards/margins": 0.61328125,
+      "rewards/rejected": -1.703125,
+      "step": 1340
+    },
+    {
+      "epoch": 0.05025032104371778,
+      "grad_norm": 6.0555869835672125,
+      "learning_rate": 2.5120952735392633e-07,
+      "logits/chosen": -3.40625,
+      "logits/rejected": -3.375,
+      "logps/chosen": -314.0,
+      "logps/rejected": -360.0,
+      "loss": 0.5384,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.265625,
+      "rewards/margins": 0.59375,
+      "rewards/rejected": -1.859375,
+      "step": 1350
+    },
+    {
+      "epoch": 0.05062254564404162,
+      "grad_norm": 4.5935735855677,
+      "learning_rate": 2.530703386676591e-07,
+      "logits/chosen": -3.34375,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -334.0,
+      "logps/rejected": -366.0,
+      "loss": 0.5466,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.296875,
+      "rewards/margins": 0.515625,
+      "rewards/rejected": -1.8125,
+      "step": 1360
+    },
+    {
+      "epoch": 0.05099477024436545,
+      "grad_norm": 4.819563724248301,
+      "learning_rate": 2.5493114998139186e-07,
+      "logits/chosen": -3.421875,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -300.0,
+      "logps/rejected": -352.0,
+      "loss": 0.521,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.2265625,
+      "rewards/margins": 0.62109375,
+      "rewards/rejected": -1.8515625,
+      "step": 1370
+    },
+    {
+      "epoch": 0.05136699484468928,
+      "grad_norm": 5.997867654446387,
+      "learning_rate": 2.5679196129512463e-07,
+      "logits/chosen": -3.546875,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -318.0,
+      "logps/rejected": -354.0,
+      "loss": 0.5521,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.25,
+      "rewards/margins": 0.62890625,
+      "rewards/rejected": -1.875,
+      "step": 1380
+    },
+    {
+      "epoch": 0.05173921944501312,
+      "grad_norm": 6.842483364035436,
+      "learning_rate": 2.5865277260885745e-07,
+      "logits/chosen": -3.53125,
+      "logits/rejected": -3.375,
+      "logps/chosen": -308.0,
+      "logps/rejected": -378.0,
+      "loss": 0.5401,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.2734375,
+      "rewards/margins": 0.7421875,
+      "rewards/rejected": -2.015625,
+      "step": 1390
+    },
+    {
+      "epoch": 0.052111444045336956,
+      "grad_norm": 6.086281607964243,
+      "learning_rate": 2.6051358392259027e-07,
+      "logits/chosen": -3.609375,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -346.0,
+      "logps/rejected": -386.0,
+      "loss": 0.5179,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.53125,
+      "rewards/margins": 0.609375,
+      "rewards/rejected": -2.140625,
+      "step": 1400
+    },
+    {
+      "epoch": 0.05248366864566079,
+      "grad_norm": 7.925102827724738,
+      "learning_rate": 2.6237439523632303e-07,
+      "logits/chosen": -3.34375,
+      "logits/rejected": -3.296875,
+      "logps/chosen": -350.0,
+      "logps/rejected": -378.0,
+      "loss": 0.5349,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5078125,
+      "rewards/margins": 0.62109375,
+      "rewards/rejected": -2.125,
+      "step": 1410
+    },
+    {
+      "epoch": 0.05285589324598463,
+      "grad_norm": 5.562523376985507,
+      "learning_rate": 2.642352065500558e-07,
+      "logits/chosen": -3.390625,
+      "logits/rejected": -3.328125,
+      "logps/chosen": -330.0,
+      "logps/rejected": -378.0,
+      "loss": 0.5158,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.453125,
+      "rewards/margins": 0.73046875,
+      "rewards/rejected": -2.1875,
+      "step": 1420
+    },
+    {
+      "epoch": 0.05322811784630846,
+      "grad_norm": 6.273726179376068,
+      "learning_rate": 2.660960178637886e-07,
+      "logits/chosen": -3.296875,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -348.0,
+      "logps/rejected": -382.0,
+      "loss": 0.5082,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.484375,
+      "rewards/margins": 0.65625,
+      "rewards/rejected": -2.140625,
+      "step": 1430
+    },
+    {
+      "epoch": 0.053600342446632296,
+      "grad_norm": 5.899521967641372,
+      "learning_rate": 2.679568291775214e-07,
+      "logits/chosen": -3.453125,
+      "logits/rejected": -3.1875,
+      "logps/chosen": -342.0,
+      "logps/rejected": -400.0,
+      "loss": 0.5071,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.5703125,
+      "rewards/margins": 0.8125,
+      "rewards/rejected": -2.375,
+      "step": 1440
+    },
+    {
+      "epoch": 0.05397256704695613,
+      "grad_norm": 7.590023994842796,
+      "learning_rate": 2.6981764049125415e-07,
+      "logits/chosen": -3.359375,
+      "logits/rejected": -3.1875,
+      "logps/chosen": -338.0,
+      "logps/rejected": -404.0,
+      "loss": 0.502,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.5625,
+      "rewards/margins": 0.77734375,
+      "rewards/rejected": -2.34375,
+      "step": 1450
+    },
+    {
+      "epoch": 0.05434479164727997,
+      "grad_norm": 7.0787501168142075,
+      "learning_rate": 2.716784518049869e-07,
+      "logits/chosen": -3.265625,
+      "logits/rejected": -3.1875,
+      "logps/chosen": -368.0,
+      "logps/rejected": -404.0,
+      "loss": 0.5381,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7421875,
+      "rewards/margins": 0.6015625,
+      "rewards/rejected": -2.34375,
+      "step": 1460
+    },
+    {
+      "epoch": 0.054717016247603806,
+      "grad_norm": 5.524203931396336,
+      "learning_rate": 2.735392631187198e-07,
+      "logits/chosen": -3.34375,
+      "logits/rejected": -3.296875,
+      "logps/chosen": -342.0,
+      "logps/rejected": -382.0,
+      "loss": 0.5106,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.4453125,
+      "rewards/margins": 0.70703125,
+      "rewards/rejected": -2.15625,
+      "step": 1470
+    },
+    {
+      "epoch": 0.05508924084792764,
+      "grad_norm": 6.607050954002419,
+      "learning_rate": 2.7540007443245256e-07,
+      "logits/chosen": -3.3125,
+      "logits/rejected": -3.1875,
+      "logps/chosen": -372.0,
+      "logps/rejected": -436.0,
+      "loss": 0.5041,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.7734375,
+      "rewards/margins": 0.765625,
+      "rewards/rejected": -2.546875,
+      "step": 1480
+    },
+    {
+      "epoch": 0.05546146544825147,
+      "grad_norm": 5.8497032295822144,
+      "learning_rate": 2.7726088574618533e-07,
+      "logits/chosen": -3.484375,
+      "logits/rejected": -3.265625,
+      "logps/chosen": -328.0,
+      "logps/rejected": -408.0,
+      "loss": 0.5119,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.609375,
+      "rewards/margins": 0.78125,
+      "rewards/rejected": -2.390625,
+      "step": 1490
+    },
+    {
+      "epoch": 0.05583369004857531,
+      "grad_norm": 6.730674517789158,
+      "learning_rate": 2.791216970599181e-07,
+      "logits/chosen": -3.5,
+      "logits/rejected": -3.296875,
+      "logps/chosen": -322.0,
+      "logps/rejected": -380.0,
+      "loss": 0.5142,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.4140625,
+      "rewards/margins": 0.83984375,
+      "rewards/rejected": -2.25,
+      "step": 1500
+    },
+    {
+      "epoch": 0.056205914648899145,
+      "grad_norm": 8.257861813501814,
+      "learning_rate": 2.809825083736509e-07,
+      "logits/chosen": -3.484375,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -368.0,
+      "logps/rejected": -468.0,
+      "loss": 0.5097,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.84375,
+      "rewards/margins": 0.91015625,
+      "rewards/rejected": -2.75,
+      "step": 1510
+    },
+    {
+      "epoch": 0.05657813924922298,
+      "grad_norm": 5.752371300767705,
+      "learning_rate": 2.828433196873837e-07,
+      "logits/chosen": -3.4375,
+      "logits/rejected": -3.25,
+      "logps/chosen": -344.0,
+      "logps/rejected": -408.0,
+      "loss": 0.5177,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.609375,
+      "rewards/margins": 0.8046875,
+      "rewards/rejected": -2.40625,
+      "step": 1520
+    },
+    {
+      "epoch": 0.05695036384954682,
+      "grad_norm": 9.313332091007915,
+      "learning_rate": 2.8470413100111645e-07,
+      "logits/chosen": -3.296875,
+      "logits/rejected": -3.1875,
+      "logps/chosen": -334.0,
+      "logps/rejected": -406.0,
+      "loss": 0.5,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.53125,
+      "rewards/margins": 0.76171875,
+      "rewards/rejected": -2.28125,
+      "step": 1530
+    },
+    {
+      "epoch": 0.057322588449870655,
+      "grad_norm": 7.674209413267227,
+      "learning_rate": 2.865649423148492e-07,
+      "logits/chosen": -3.484375,
+      "logits/rejected": -3.296875,
+      "logps/chosen": -378.0,
+      "logps/rejected": -448.0,
+      "loss": 0.5129,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9296875,
+      "rewards/margins": 0.703125,
+      "rewards/rejected": -2.625,
+      "step": 1540
+    },
+    {
+      "epoch": 0.057694813050194485,
+      "grad_norm": 5.42621251994026,
+      "learning_rate": 2.884257536285821e-07,
+      "logits/chosen": -3.390625,
+      "logits/rejected": -3.28125,
+      "logps/chosen": -346.0,
+      "logps/rejected": -400.0,
+      "loss": 0.5054,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.515625,
+      "rewards/margins": 0.91015625,
+      "rewards/rejected": -2.4375,
+      "step": 1550
+    },
+    {
+      "epoch": 0.05806703765051832,
+      "grad_norm": 5.398220335499412,
+      "learning_rate": 2.9028656494231485e-07,
+      "logits/chosen": -3.375,
+      "logits/rejected": -3.203125,
+      "logps/chosen": -344.0,
+      "logps/rejected": -408.0,
+      "loss": 0.5208,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.6796875,
+      "rewards/margins": 0.8359375,
+      "rewards/rejected": -2.515625,
+      "step": 1560
+    },
+    {
+      "epoch": 0.05843926225084216,
+      "grad_norm": 7.249550264624127,
+      "learning_rate": 2.921473762560476e-07,
+      "logits/chosen": -3.375,
+      "logits/rejected": -3.203125,
+      "logps/chosen": -346.0,
+      "logps/rejected": -414.0,
+      "loss": 0.5091,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.6953125,
+      "rewards/margins": 0.88671875,
+      "rewards/rejected": -2.578125,
+      "step": 1570
+    },
+    {
+      "epoch": 0.058811486851165995,
+      "grad_norm": 6.7555584084417175,
+      "learning_rate": 2.940081875697804e-07,
+      "logits/chosen": -3.3125,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -408.0,
+      "logps/rejected": -450.0,
+      "loss": 0.504,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.9921875,
+      "rewards/margins": 0.7578125,
+      "rewards/rejected": -2.75,
+      "step": 1580
+    },
+    {
+      "epoch": 0.05918371145148983,
+      "grad_norm": 6.757377075014917,
+      "learning_rate": 2.958689988835132e-07,
+      "logits/chosen": -3.3125,
+      "logits/rejected": -3.203125,
+      "logps/chosen": -368.0,
+      "logps/rejected": -464.0,
+      "loss": 0.4881,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.84375,
+      "rewards/margins": 0.82421875,
+      "rewards/rejected": -2.671875,
+      "step": 1590
+    },
+    {
+      "epoch": 0.05955593605181366,
+      "grad_norm": 6.082845891238061,
+      "learning_rate": 2.97729810197246e-07,
+      "logits/chosen": -3.515625,
+      "logits/rejected": -3.21875,
+      "logps/chosen": -338.0,
+      "logps/rejected": -416.0,
+      "loss": 0.5036,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.765625,
+      "rewards/margins": 0.8828125,
+      "rewards/rejected": -2.65625,
+      "step": 1600
+    },
+    {
+      "epoch": 0.0599281606521375,
+      "grad_norm": 6.427557821854768,
+      "learning_rate": 2.9959062151097874e-07,
+      "logits/chosen": -3.390625,
+      "logits/rejected": -3.203125,
+      "logps/chosen": -360.0,
+      "logps/rejected": -436.0,
+      "loss": 0.4992,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.796875,
+      "rewards/margins": 0.84765625,
+      "rewards/rejected": -2.640625,
+      "step": 1610
+    },
+    {
+      "epoch": 0.060300385252461335,
+      "grad_norm": 6.3336369858688055,
+      "learning_rate": 3.0145143282471156e-07,
+      "logits/chosen": -3.34375,
+      "logits/rejected": -3.265625,
+      "logps/chosen": -356.0,
+      "logps/rejected": -430.0,
+      "loss": 0.5053,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.6640625,
+      "rewards/margins": 0.77734375,
+      "rewards/rejected": -2.4375,
+      "step": 1620
+    },
+    {
+      "epoch": 0.06067260985278517,
+      "grad_norm": 6.53782796114545,
+      "learning_rate": 3.033122441384444e-07,
+      "logits/chosen": -3.34375,
+      "logits/rejected": -3.125,
+      "logps/chosen": -374.0,
+      "logps/rejected": -476.0,
+      "loss": 0.4971,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.078125,
+      "rewards/margins": 0.890625,
+      "rewards/rejected": -2.96875,
+      "step": 1630
+    },
+    {
+      "epoch": 0.06104483445310901,
+      "grad_norm": 5.872611276225177,
+      "learning_rate": 3.0517305545217715e-07,
+      "logits/chosen": -3.375,
+      "logits/rejected": -3.25,
+      "logps/chosen": -364.0,
+      "logps/rejected": -428.0,
+      "loss": 0.4923,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.8203125,
+      "rewards/margins": 0.86328125,
+      "rewards/rejected": -2.6875,
+      "step": 1640
+    },
+    {
+      "epoch": 0.061417059053432844,
+      "grad_norm": 5.485636557023713,
+      "learning_rate": 3.070338667659099e-07,
+      "logits/chosen": -3.40625,
+      "logits/rejected": -3.328125,
+      "logps/chosen": -390.0,
+      "logps/rejected": -446.0,
+      "loss": 0.5244,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.0,
+      "rewards/margins": 0.7578125,
+      "rewards/rejected": -2.765625,
+      "step": 1650
+    },
+    {
+      "epoch": 0.061789283653756674,
+      "grad_norm": 12.16446146603116,
+      "learning_rate": 3.088946780796427e-07,
+      "logits/chosen": -3.421875,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -402.0,
+      "logps/rejected": -470.0,
+      "loss": 0.4816,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.09375,
+      "rewards/margins": 0.88671875,
+      "rewards/rejected": -2.984375,
+      "step": 1660
+    },
+    {
+      "epoch": 0.06216150825408051,
+      "grad_norm": 7.067013845513051,
+      "learning_rate": 3.107554893933755e-07,
+      "logits/chosen": -3.3125,
+      "logits/rejected": -3.1875,
+      "logps/chosen": -394.0,
+      "logps/rejected": -450.0,
+      "loss": 0.5243,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.9375,
+      "rewards/margins": 0.890625,
+      "rewards/rejected": -2.828125,
+      "step": 1670
+    },
+    {
+      "epoch": 0.06253373285440435,
+      "grad_norm": 5.530567691349678,
+      "learning_rate": 3.1261630070710827e-07,
+      "logits/chosen": -3.28125,
+      "logits/rejected": -3.1875,
+      "logps/chosen": -386.0,
+      "logps/rejected": -428.0,
+      "loss": 0.5085,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.90625,
+      "rewards/margins": 0.6875,
+      "rewards/rejected": -2.59375,
+      "step": 1680
+    },
+    {
+      "epoch": 0.06290595745472818,
+      "grad_norm": 7.007034753902651,
+      "learning_rate": 3.144771120208411e-07,
+      "logits/chosen": -3.3125,
+      "logits/rejected": -3.21875,
+      "logps/chosen": -384.0,
+      "logps/rejected": -450.0,
+      "loss": 0.4898,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.875,
+      "rewards/margins": 0.98046875,
+      "rewards/rejected": -2.859375,
+      "step": 1690
+    },
+    {
+      "epoch": 0.06327818205505202,
+      "grad_norm": 5.9945544048762835,
+      "learning_rate": 3.1633792333457385e-07,
+      "logits/chosen": -3.34375,
+      "logits/rejected": -3.28125,
+      "logps/chosen": -382.0,
+      "logps/rejected": -464.0,
+      "loss": 0.4971,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.953125,
+      "rewards/margins": 0.9375,
+      "rewards/rejected": -2.890625,
+      "step": 1700
+    },
+    {
+      "epoch": 0.06365040665537586,
+      "grad_norm": 7.539162609616699,
+      "learning_rate": 3.181987346483067e-07,
+      "logits/chosen": -3.5,
+      "logits/rejected": -3.3125,
+      "logps/chosen": -406.0,
+      "logps/rejected": -468.0,
+      "loss": 0.5071,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.203125,
+      "rewards/margins": 0.80078125,
+      "rewards/rejected": -3.0,
+      "step": 1710
+    },
+    {
+      "epoch": 0.0640226312556997,
+      "grad_norm": 6.758590827074712,
+      "learning_rate": 3.2005954596203944e-07,
+      "logits/chosen": -3.390625,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -398.0,
+      "logps/rejected": -472.0,
+      "loss": 0.4796,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.25,
+      "rewards/margins": 0.84765625,
+      "rewards/rejected": -3.09375,
+      "step": 1720
+    },
+    {
+      "epoch": 0.06439485585602353,
+      "grad_norm": 6.7152082532075115,
+      "learning_rate": 3.219203572757722e-07,
+      "logits/chosen": -3.40625,
+      "logits/rejected": -3.21875,
+      "logps/chosen": -364.0,
+      "logps/rejected": -434.0,
+      "loss": 0.511,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.9296875,
+      "rewards/margins": 0.82421875,
+      "rewards/rejected": -2.75,
+      "step": 1730
+    },
+    {
+      "epoch": 0.06476708045634735,
+      "grad_norm": 5.736720629813318,
+      "learning_rate": 3.23781168589505e-07,
+      "logits/chosen": -3.484375,
+      "logits/rejected": -3.28125,
+      "logps/chosen": -364.0,
+      "logps/rejected": -442.0,
+      "loss": 0.4983,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.7890625,
+      "rewards/margins": 0.9609375,
+      "rewards/rejected": -2.75,
+      "step": 1740
+    },
+    {
+      "epoch": 0.06513930505667119,
+      "grad_norm": 6.211718793245078,
+      "learning_rate": 3.256419799032378e-07,
+      "logits/chosen": -3.40625,
+      "logits/rejected": -3.265625,
+      "logps/chosen": -358.0,
+      "logps/rejected": -406.0,
+      "loss": 0.5174,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6640625,
+      "rewards/margins": 0.6640625,
+      "rewards/rejected": -2.328125,
+      "step": 1750
+    },
+    {
+      "epoch": 0.06551152965699503,
+      "grad_norm": 5.632562755835517,
+      "learning_rate": 3.275027912169706e-07,
+      "logits/chosen": -3.375,
+      "logits/rejected": -3.265625,
+      "logps/chosen": -412.0,
+      "logps/rejected": -486.0,
+      "loss": 0.4903,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.015625,
+      "rewards/margins": 0.87109375,
+      "rewards/rejected": -2.890625,
+      "step": 1760
+    },
+    {
+      "epoch": 0.06588375425731886,
+      "grad_norm": 5.578480380124134,
+      "learning_rate": 3.293636025307034e-07,
+      "logits/chosen": -3.453125,
+      "logits/rejected": -3.359375,
+      "logps/chosen": -426.0,
+      "logps/rejected": -496.0,
+      "loss": 0.4979,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.328125,
+      "rewards/margins": 0.91015625,
+      "rewards/rejected": -3.25,
+      "step": 1770
+    },
+    {
+      "epoch": 0.0662559788576427,
+      "grad_norm": 6.8437171680878714,
+      "learning_rate": 3.3122441384443615e-07,
+      "logits/chosen": -3.484375,
+      "logits/rejected": -3.328125,
+      "logps/chosen": -368.0,
+      "logps/rejected": -428.0,
+      "loss": 0.5024,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.8046875,
+      "rewards/margins": 0.875,
+      "rewards/rejected": -2.671875,
+      "step": 1780
+    },
+    {
+      "epoch": 0.06662820345796654,
+      "grad_norm": 6.0818060385390265,
+      "learning_rate": 3.3308522515816897e-07,
+      "logits/chosen": -3.53125,
+      "logits/rejected": -3.359375,
+      "logps/chosen": -380.0,
+      "logps/rejected": -448.0,
+      "loss": 0.4947,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9765625,
+      "rewards/margins": 0.8828125,
+      "rewards/rejected": -2.859375,
+      "step": 1790
+    },
+    {
+      "epoch": 0.06700042805829037,
+      "grad_norm": 9.832087595188957,
+      "learning_rate": 3.3494603647190173e-07,
+      "logits/chosen": -3.59375,
+      "logits/rejected": -3.40625,
+      "logps/chosen": -454.0,
+      "logps/rejected": -516.0,
+      "loss": 0.5041,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.5625,
+      "rewards/margins": 0.890625,
+      "rewards/rejected": -3.453125,
+      "step": 1800
+    },
+    {
+      "epoch": 0.06737265265861421,
+      "grad_norm": 6.901409024521085,
+      "learning_rate": 3.368068477856345e-07,
+      "logits/chosen": -3.46875,
+      "logits/rejected": -3.328125,
+      "logps/chosen": -416.0,
+      "logps/rejected": -480.0,
+      "loss": 0.4787,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.421875,
+      "rewards/margins": 0.8203125,
+      "rewards/rejected": -3.234375,
+      "step": 1810
+    },
+    {
+      "epoch": 0.06774487725893805,
+      "grad_norm": 7.4474348220804485,
+      "learning_rate": 3.3866765909936727e-07,
+      "logits/chosen": -3.609375,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -396.0,
+      "logps/rejected": -460.0,
+      "loss": 0.4891,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.9296875,
+      "rewards/margins": 1.0078125,
+      "rewards/rejected": -2.9375,
+      "step": 1820
+    },
+    {
+      "epoch": 0.06811710185926188,
+      "grad_norm": 6.50288727444381,
+      "learning_rate": 3.4052847041310014e-07,
+      "logits/chosen": -3.46875,
+      "logits/rejected": -3.1875,
+      "logps/chosen": -386.0,
+      "logps/rejected": -466.0,
+      "loss": 0.4927,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.171875,
+      "rewards/margins": 0.921875,
+      "rewards/rejected": -3.09375,
+      "step": 1830
+    },
+    {
+      "epoch": 0.06848932645958572,
+      "grad_norm": 7.608497450368566,
+      "learning_rate": 3.423892817268329e-07,
+      "logits/chosen": -3.5,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -416.0,
+      "logps/rejected": -494.0,
+      "loss": 0.4963,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.125,
+      "rewards/margins": 0.98828125,
+      "rewards/rejected": -3.125,
+      "step": 1840
+    },
+    {
+      "epoch": 0.06886155105990954,
+      "grad_norm": 10.258817701612603,
+      "learning_rate": 3.442500930405657e-07,
+      "logits/chosen": -3.578125,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -434.0,
+      "logps/rejected": -502.0,
+      "loss": 0.4814,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.40625,
+      "rewards/margins": 0.91015625,
+      "rewards/rejected": -3.3125,
+      "step": 1850
+    },
+    {
+      "epoch": 0.06923377566023338,
+      "grad_norm": 6.196807090101503,
+      "learning_rate": 3.4611090435429844e-07,
+      "logits/chosen": -3.453125,
+      "logits/rejected": -3.421875,
+      "logps/chosen": -422.0,
+      "logps/rejected": -492.0,
+      "loss": 0.4911,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.25,
+      "rewards/margins": 0.89453125,
+      "rewards/rejected": -3.140625,
+      "step": 1860
+    },
+    {
+      "epoch": 0.06960600026055722,
+      "grad_norm": 7.770073521207905,
+      "learning_rate": 3.4797171566803126e-07,
+      "logits/chosen": -3.375,
+      "logits/rejected": -3.28125,
+      "logps/chosen": -416.0,
+      "logps/rejected": -476.0,
+      "loss": 0.4657,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.1875,
+      "rewards/margins": 0.8671875,
+      "rewards/rejected": -3.046875,
+      "step": 1870
+    },
+    {
+      "epoch": 0.06997822486088105,
+      "grad_norm": 11.94458655741529,
+      "learning_rate": 3.4983252698176403e-07,
+      "logits/chosen": -3.421875,
+      "logits/rejected": -3.109375,
+      "logps/chosen": -454.0,
+      "logps/rejected": -520.0,
+      "loss": 0.5099,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.640625,
+      "rewards/margins": 0.8984375,
+      "rewards/rejected": -3.53125,
+      "step": 1880
+    },
+    {
+      "epoch": 0.07035044946120489,
+      "grad_norm": 8.647210574831504,
+      "learning_rate": 3.516933382954968e-07,
+      "logits/chosen": -3.453125,
+      "logits/rejected": -3.421875,
+      "logps/chosen": -408.0,
+      "logps/rejected": -486.0,
+      "loss": 0.4792,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.171875,
+      "rewards/margins": 0.9140625,
+      "rewards/rejected": -3.09375,
+      "step": 1890
+    },
+    {
+      "epoch": 0.07072267406152873,
+      "grad_norm": 8.253805566712586,
+      "learning_rate": 3.535541496092296e-07,
+      "logits/chosen": -3.53125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -482.0,
+      "logps/rejected": -548.0,
+      "loss": 0.4916,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.8125,
+      "rewards/margins": 0.89453125,
+      "rewards/rejected": -3.703125,
+      "step": 1900
+    },
+    {
+      "epoch": 0.07109489866185256,
+      "grad_norm": 6.8734464309524475,
+      "learning_rate": 3.5541496092296243e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -454.0,
+      "logps/rejected": -544.0,
+      "loss": 0.4968,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.65625,
+      "rewards/margins": 0.94921875,
+      "rewards/rejected": -3.609375,
+      "step": 1910
+    },
+    {
+      "epoch": 0.0714671232621764,
+      "grad_norm": 6.19491556402742,
+      "learning_rate": 3.572757722366952e-07,
+      "logits/chosen": -3.625,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -392.0,
+      "logps/rejected": -462.0,
+      "loss": 0.5004,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.265625,
+      "rewards/margins": 0.75390625,
+      "rewards/rejected": -3.015625,
+      "step": 1920
+    },
+    {
+      "epoch": 0.07183934786250024,
+      "grad_norm": 7.196402223453586,
+      "learning_rate": 3.5913658355042797e-07,
+      "logits/chosen": -3.578125,
+      "logits/rejected": -3.390625,
+      "logps/chosen": -436.0,
+      "logps/rejected": -498.0,
+      "loss": 0.5028,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.53125,
+      "rewards/margins": 0.7265625,
+      "rewards/rejected": -3.265625,
+      "step": 1930
+    },
+    {
+      "epoch": 0.07221157246282407,
+      "grad_norm": 8.42899046845527,
+      "learning_rate": 3.6099739486416073e-07,
+      "logits/chosen": -3.453125,
+      "logits/rejected": -3.3125,
+      "logps/chosen": -490.0,
+      "logps/rejected": -548.0,
+      "loss": 0.5011,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.875,
+      "rewards/margins": 0.85546875,
+      "rewards/rejected": -3.734375,
+      "step": 1940
+    },
+    {
+      "epoch": 0.07258379706314791,
+      "grad_norm": 5.646083907129506,
+      "learning_rate": 3.6285820617789355e-07,
+      "logits/chosen": -3.421875,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -432.0,
+      "logps/rejected": -496.0,
+      "loss": 0.5065,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.390625,
+      "rewards/margins": 0.8359375,
+      "rewards/rejected": -3.21875,
+      "step": 1950
+    },
+    {
+      "epoch": 0.07295602166347175,
+      "grad_norm": 7.9137815262402516,
+      "learning_rate": 3.647190174916263e-07,
+      "logits/chosen": -3.46875,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -426.0,
+      "logps/rejected": -498.0,
+      "loss": 0.5025,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.28125,
+      "rewards/margins": 0.76171875,
+      "rewards/rejected": -3.046875,
+      "step": 1960
+    },
+    {
+      "epoch": 0.07332824626379557,
+      "grad_norm": 7.919156114148352,
+      "learning_rate": 3.665798288053591e-07,
+      "logits/chosen": -3.53125,
+      "logits/rejected": -3.375,
+      "logps/chosen": -416.0,
+      "logps/rejected": -478.0,
+      "loss": 0.4789,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.46875,
+      "rewards/margins": 0.91796875,
+      "rewards/rejected": -3.375,
+      "step": 1970
+    },
+    {
+      "epoch": 0.0737004708641194,
+      "grad_norm": 6.235589300184533,
+      "learning_rate": 3.684406401190919e-07,
+      "logits/chosen": -3.515625,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -406.0,
+      "logps/rejected": -474.0,
+      "loss": 0.516,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.21875,
+      "rewards/margins": 0.8359375,
+      "rewards/rejected": -3.0625,
+      "step": 1980
+    },
+    {
+      "epoch": 0.07407269546444324,
+      "grad_norm": 7.351757597986148,
+      "learning_rate": 3.7030145143282473e-07,
+      "logits/chosen": -3.609375,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -432.0,
+      "logps/rejected": -508.0,
+      "loss": 0.4791,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.5625,
+      "rewards/margins": 0.94921875,
+      "rewards/rejected": -3.515625,
+      "step": 1990
+    },
+    {
+      "epoch": 0.07444492006476708,
+      "grad_norm": 6.655521716946184,
+      "learning_rate": 3.721622627465575e-07,
+      "logits/chosen": -3.46875,
+      "logits/rejected": -3.390625,
+      "logps/chosen": -476.0,
+      "logps/rejected": -552.0,
+      "loss": 0.4716,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.734375,
+      "rewards/margins": 0.99609375,
+      "rewards/rejected": -3.734375,
+      "step": 2000
+    },
+    {
+      "epoch": 0.07481714466509091,
+      "grad_norm": 7.419440727048614,
+      "learning_rate": 3.7402307406029026e-07,
+      "logits/chosen": -3.46875,
+      "logits/rejected": -3.28125,
+      "logps/chosen": -408.0,
+      "logps/rejected": -470.0,
+      "loss": 0.5067,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.3125,
+      "rewards/margins": 0.87890625,
+      "rewards/rejected": -3.1875,
+      "step": 2010
+    },
+    {
+      "epoch": 0.07518936926541475,
+      "grad_norm": 6.808603030463394,
+      "learning_rate": 3.7588388537402303e-07,
+      "logits/chosen": -3.515625,
+      "logits/rejected": -3.34375,
+      "logps/chosen": -406.0,
+      "logps/rejected": -468.0,
+      "loss": 0.4797,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.21875,
+      "rewards/margins": 0.97265625,
+      "rewards/rejected": -3.1875,
+      "step": 2020
+    },
+    {
+      "epoch": 0.07556159386573859,
+      "grad_norm": 9.804937954471944,
+      "learning_rate": 3.7774469668775585e-07,
+      "logits/chosen": -3.484375,
+      "logits/rejected": -3.3125,
+      "logps/chosen": -434.0,
+      "logps/rejected": -500.0,
+      "loss": 0.483,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.75,
+      "rewards/margins": 0.92578125,
+      "rewards/rejected": -3.6875,
+      "step": 2030
+    },
+    {
+      "epoch": 0.07593381846606242,
+      "grad_norm": 7.011321899662242,
+      "learning_rate": 3.796055080014886e-07,
+      "logits/chosen": -3.46875,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -456.0,
+      "logps/rejected": -524.0,
+      "loss": 0.4951,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.71875,
+      "rewards/margins": 0.83984375,
+      "rewards/rejected": -3.546875,
+      "step": 2040
+    },
+    {
+      "epoch": 0.07630604306638626,
+      "grad_norm": 8.34674921558812,
+      "learning_rate": 3.8146631931522143e-07,
+      "logits/chosen": -3.453125,
+      "logits/rejected": -3.375,
+      "logps/chosen": -454.0,
+      "logps/rejected": -552.0,
+      "loss": 0.4668,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.71875,
+      "rewards/margins": 1.15625,
+      "rewards/rejected": -3.875,
+      "step": 2050
+    },
+    {
+      "epoch": 0.0766782676667101,
+      "grad_norm": 7.914453901846385,
+      "learning_rate": 3.833271306289542e-07,
+      "logits/chosen": -3.5,
+      "logits/rejected": -3.328125,
+      "logps/chosen": -450.0,
+      "logps/rejected": -540.0,
+      "loss": 0.4623,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.859375,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -3.953125,
+      "step": 2060
+    },
+    {
+      "epoch": 0.07705049226703393,
+      "grad_norm": 8.048084008154206,
+      "learning_rate": 3.85187941942687e-07,
+      "logits/chosen": -3.515625,
+      "logits/rejected": -3.3125,
+      "logps/chosen": -472.0,
+      "logps/rejected": -556.0,
+      "loss": 0.4928,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.625,
+      "rewards/margins": 0.99609375,
+      "rewards/rejected": -3.625,
+      "step": 2070
+    },
+    {
+      "epoch": 0.07742271686735776,
+      "grad_norm": 7.206841484214931,
+      "learning_rate": 3.870487532564198e-07,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -462.0,
+      "logps/rejected": -536.0,
+      "loss": 0.4883,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.796875,
+      "rewards/margins": 0.890625,
+      "rewards/rejected": -3.6875,
+      "step": 2080
+    },
+    {
+      "epoch": 0.0777949414676816,
+      "grad_norm": 6.3124275295264685,
+      "learning_rate": 3.8890956457015255e-07,
+      "logits/chosen": -3.640625,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -430.0,
+      "logps/rejected": -516.0,
+      "loss": 0.4787,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.625,
+      "rewards/margins": 0.796875,
+      "rewards/rejected": -3.421875,
+      "step": 2090
+    },
+    {
+      "epoch": 0.07816716606800543,
+      "grad_norm": 6.387969253344187,
+      "learning_rate": 3.907703758838853e-07,
+      "logits/chosen": -3.484375,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -484.0,
+      "logps/rejected": -568.0,
+      "loss": 0.4716,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.875,
+      "rewards/margins": 1.1015625,
+      "rewards/rejected": -3.96875,
+      "step": 2100
+    },
+    {
+      "epoch": 0.07853939066832927,
+      "grad_norm": 7.3661811834257085,
+      "learning_rate": 3.9263118719761814e-07,
+      "logits/chosen": -3.53125,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -466.0,
+      "logps/rejected": -572.0,
+      "loss": 0.4716,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.828125,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -3.9375,
+      "step": 2110
+    },
+    {
+      "epoch": 0.0789116152686531,
+      "grad_norm": 7.830049539836104,
+      "learning_rate": 3.9449199851135096e-07,
+      "logits/chosen": -3.53125,
+      "logits/rejected": -3.390625,
+      "logps/chosen": -476.0,
+      "logps/rejected": -544.0,
+      "loss": 0.4993,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.03125,
+      "rewards/margins": 0.8984375,
+      "rewards/rejected": -3.9375,
+      "step": 2120
+    },
+    {
+      "epoch": 0.07928383986897694,
+      "grad_norm": 7.433179362073206,
+      "learning_rate": 3.9635280982508373e-07,
+      "logits/chosen": -3.609375,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -440.0,
+      "logps/rejected": -512.0,
+      "loss": 0.4601,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.609375,
+      "rewards/margins": 0.92578125,
+      "rewards/rejected": -3.53125,
+      "step": 2130
+    },
+    {
+      "epoch": 0.07965606446930078,
+      "grad_norm": 7.361658833026729,
+      "learning_rate": 3.982136211388165e-07,
+      "logits/chosen": -3.546875,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -488.0,
+      "logps/rejected": -568.0,
+      "loss": 0.4654,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.953125,
+      "rewards/margins": 1.078125,
+      "rewards/rejected": -4.03125,
+      "step": 2140
+    },
+    {
+      "epoch": 0.08002828906962461,
+      "grad_norm": 6.64152297996465,
+      "learning_rate": 4.000744324525493e-07,
+      "logits/chosen": -3.609375,
+      "logits/rejected": -3.359375,
+      "logps/chosen": -462.0,
+      "logps/rejected": -564.0,
+      "loss": 0.483,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.8125,
+      "rewards/margins": 1.0859375,
+      "rewards/rejected": -3.90625,
+      "step": 2150
+    },
+    {
+      "epoch": 0.08040051366994845,
+      "grad_norm": 8.565985887881476,
+      "learning_rate": 4.019352437662821e-07,
+      "logits/chosen": -3.53125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -448.0,
+      "logps/rejected": -512.0,
+      "loss": 0.4672,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.5625,
+      "rewards/margins": 0.84765625,
+      "rewards/rejected": -3.421875,
+      "step": 2160
+    },
+    {
+      "epoch": 0.08077273827027229,
+      "grad_norm": 8.502570785022007,
+      "learning_rate": 4.0379605508001485e-07,
+      "logits/chosen": -3.59375,
+      "logits/rejected": -3.421875,
+      "logps/chosen": -500.0,
+      "logps/rejected": -592.0,
+      "loss": 0.4865,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.015625,
+      "rewards/margins": 1.1484375,
+      "rewards/rejected": -4.15625,
+      "step": 2170
+    },
+    {
+      "epoch": 0.08114496287059612,
+      "grad_norm": 6.182879652012646,
+      "learning_rate": 4.056568663937476e-07,
+      "logits/chosen": -3.53125,
+      "logits/rejected": -3.265625,
+      "logps/chosen": -506.0,
+      "logps/rejected": -596.0,
+      "loss": 0.4739,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.1875,
+      "rewards/margins": 1.1171875,
+      "rewards/rejected": -4.3125,
+      "step": 2180
+    },
+    {
+      "epoch": 0.08151718747091995,
+      "grad_norm": 7.010299440446747,
+      "learning_rate": 4.075176777074805e-07,
+      "logits/chosen": -3.46875,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -464.0,
+      "logps/rejected": -528.0,
+      "loss": 0.4802,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.546875,
+      "rewards/margins": 0.9921875,
+      "rewards/rejected": -3.546875,
+      "step": 2190
+    },
+    {
+      "epoch": 0.08188941207124378,
+      "grad_norm": 5.568537062184612,
+      "learning_rate": 4.0937848902121325e-07,
+      "logits/chosen": -3.578125,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -512.0,
+      "logps/rejected": -580.0,
+      "loss": 0.481,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.328125,
+      "rewards/margins": 0.87109375,
+      "rewards/rejected": -4.1875,
+      "step": 2200
+    },
+    {
+      "epoch": 0.08226163667156762,
+      "grad_norm": 6.448164831889189,
+      "learning_rate": 4.11239300334946e-07,
+      "logits/chosen": -3.578125,
+      "logits/rejected": -3.421875,
+      "logps/chosen": -498.0,
+      "logps/rejected": -544.0,
+      "loss": 0.4887,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -3.140625,
+      "rewards/margins": 0.7578125,
+      "rewards/rejected": -3.890625,
+      "step": 2210
+    },
+    {
+      "epoch": 0.08263386127189146,
+      "grad_norm": 7.624705951730895,
+      "learning_rate": 4.131001116486788e-07,
+      "logits/chosen": -3.546875,
+      "logits/rejected": -3.5,
+      "logps/chosen": -502.0,
+      "logps/rejected": -572.0,
+      "loss": 0.4671,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -3.109375,
+      "rewards/margins": 0.90234375,
+      "rewards/rejected": -4.0,
+      "step": 2220
+    },
+    {
+      "epoch": 0.0830060858722153,
+      "grad_norm": 6.666954742432476,
+      "learning_rate": 4.149609229624116e-07,
+      "logits/chosen": -3.5,
+      "logits/rejected": -3.375,
+      "logps/chosen": -500.0,
+      "logps/rejected": -568.0,
+      "loss": 0.4793,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -3.0625,
+      "rewards/margins": 1.0,
+      "rewards/rejected": -4.0625,
+      "step": 2230
+    },
+    {
+      "epoch": 0.08337831047253913,
+      "grad_norm": 6.33594374057711,
+      "learning_rate": 4.168217342761444e-07,
+      "logits/chosen": -3.609375,
+      "logits/rejected": -3.421875,
+      "logps/chosen": -488.0,
+      "logps/rejected": -560.0,
+      "loss": 0.4747,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.96875,
+      "rewards/margins": 1.0546875,
+      "rewards/rejected": -4.03125,
+      "step": 2240
+    },
+    {
+      "epoch": 0.08375053507286297,
+      "grad_norm": 8.464776874422762,
+      "learning_rate": 4.1868254558987714e-07,
+      "logits/chosen": -3.484375,
+      "logits/rejected": -3.3125,
+      "logps/chosen": -482.0,
+      "logps/rejected": -540.0,
+      "loss": 0.4782,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.984375,
+      "rewards/margins": 0.84765625,
+      "rewards/rejected": -3.828125,
+      "step": 2250
+    },
+    {
+      "epoch": 0.0841227596731868,
+      "grad_norm": 9.029855119041432,
+      "learning_rate": 4.2054335690360996e-07,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -466.0,
+      "logps/rejected": -568.0,
+      "loss": 0.4776,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.875,
+      "rewards/margins": 1.03125,
+      "rewards/rejected": -3.90625,
+      "step": 2260
+    },
+    {
+      "epoch": 0.08449498427351064,
+      "grad_norm": 8.879201240189081,
+      "learning_rate": 4.224041682173428e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -500.0,
+      "logps/rejected": -600.0,
+      "loss": 0.4986,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -3.1875,
+      "rewards/margins": 0.8671875,
+      "rewards/rejected": -4.0625,
+      "step": 2270
+    },
+    {
+      "epoch": 0.08486720887383448,
+      "grad_norm": 5.848587871383628,
+      "learning_rate": 4.2426497953107555e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -520.0,
+      "logps/rejected": -600.0,
+      "loss": 0.469,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -3.296875,
+      "rewards/margins": 0.9375,
+      "rewards/rejected": -4.25,
+      "step": 2280
+    },
+    {
+      "epoch": 0.08523943347415831,
+      "grad_norm": 6.176726638784167,
+      "learning_rate": 4.261257908448083e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -520.0,
+      "logps/rejected": -600.0,
+      "loss": 0.4936,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.15625,
+      "rewards/margins": 1.0390625,
+      "rewards/rejected": -4.1875,
+      "step": 2290
+    },
+    {
+      "epoch": 0.08561165807448214,
+      "grad_norm": 5.633776596904817,
+      "learning_rate": 4.279866021585411e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -494.0,
+      "logps/rejected": -564.0,
+      "loss": 0.4794,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.015625,
+      "rewards/margins": 0.9375,
+      "rewards/rejected": -3.953125,
+      "step": 2300
+    },
+    {
+      "epoch": 0.08598388267480597,
+      "grad_norm": 5.987960772140025,
+      "learning_rate": 4.298474134722739e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -532.0,
+      "logps/rejected": -648.0,
+      "loss": 0.4429,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -3.390625,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -4.65625,
+      "step": 2310
+    },
+    {
+      "epoch": 0.08635610727512981,
+      "grad_norm": 7.931597077248074,
+      "learning_rate": 4.3170822478600667e-07,
+      "logits/chosen": -3.53125,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -516.0,
+      "logps/rejected": -600.0,
+      "loss": 0.473,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -3.234375,
+      "rewards/margins": 1.015625,
+      "rewards/rejected": -4.25,
+      "step": 2320
+    },
+    {
+      "epoch": 0.08672833187545365,
+      "grad_norm": 7.0217336118545965,
+      "learning_rate": 4.335690360997395e-07,
+      "logits/chosen": -3.625,
+      "logits/rejected": -3.375,
+      "logps/chosen": -510.0,
+      "logps/rejected": -596.0,
+      "loss": 0.4552,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.203125,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -4.3125,
+      "step": 2330
+    },
+    {
+      "epoch": 0.08710055647577748,
+      "grad_norm": 8.915164945587248,
+      "learning_rate": 4.3542984741347225e-07,
+      "logits/chosen": -3.609375,
+      "logits/rejected": -3.5,
+      "logps/chosen": -544.0,
+      "logps/rejected": -636.0,
+      "loss": 0.4716,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -3.53125,
+      "rewards/margins": 1.078125,
+      "rewards/rejected": -4.625,
+      "step": 2340
+    },
+    {
+      "epoch": 0.08747278107610132,
+      "grad_norm": 6.606079796847748,
+      "learning_rate": 4.372906587272051e-07,
+      "logits/chosen": -3.53125,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -536.0,
+      "logps/rejected": -616.0,
+      "loss": 0.4759,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.53125,
+      "rewards/margins": 1.0859375,
+      "rewards/rejected": -4.625,
+      "step": 2350
+    },
+    {
+      "epoch": 0.08784500567642516,
+      "grad_norm": 6.840756149949872,
+      "learning_rate": 4.3915147004093784e-07,
+      "logits/chosen": -3.609375,
+      "logits/rejected": -3.5,
+      "logps/chosen": -520.0,
+      "logps/rejected": -576.0,
+      "loss": 0.4554,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -3.4375,
+      "rewards/margins": 0.83203125,
+      "rewards/rejected": -4.28125,
+      "step": 2360
+    },
+    {
+      "epoch": 0.08821723027674899,
+      "grad_norm": 7.607432047893891,
+      "learning_rate": 4.410122813546706e-07,
+      "logits/chosen": -3.53125,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -488.0,
+      "logps/rejected": -544.0,
+      "loss": 0.4844,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -3.015625,
+      "rewards/margins": 1.0,
+      "rewards/rejected": -4.0,
+      "step": 2370
+    },
+    {
+      "epoch": 0.08858945487707283,
+      "grad_norm": 7.177824030903555,
+      "learning_rate": 4.428730926684034e-07,
+      "logits/chosen": -3.53125,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -520.0,
+      "logps/rejected": -576.0,
+      "loss": 0.5042,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -3.15625,
+      "rewards/margins": 0.89453125,
+      "rewards/rejected": -4.0625,
+      "step": 2380
+    },
+    {
+      "epoch": 0.08896167947739667,
+      "grad_norm": 5.609842370446836,
+      "learning_rate": 4.447339039821362e-07,
+      "logits/chosen": -3.609375,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -544.0,
+      "logps/rejected": -636.0,
+      "loss": 0.457,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -3.484375,
+      "rewards/margins": 1.1796875,
+      "rewards/rejected": -4.65625,
+      "step": 2390
+    },
+    {
+      "epoch": 0.0893339040777205,
+      "grad_norm": 6.631650078365204,
+      "learning_rate": 4.4659471529586896e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -474.0,
+      "logps/rejected": -592.0,
+      "loss": 0.4793,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.859375,
+      "rewards/margins": 1.140625,
+      "rewards/rejected": -4.0,
+      "step": 2400
+    },
+    {
+      "epoch": 0.08970612867804433,
+      "grad_norm": 8.43491277629513,
+      "learning_rate": 4.484555266096018e-07,
+      "logits/chosen": -3.59375,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -492.0,
+      "logps/rejected": -596.0,
+      "loss": 0.4743,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -3.34375,
+      "rewards/margins": 1.1875,
+      "rewards/rejected": -4.53125,
+      "step": 2410
+    },
+    {
+      "epoch": 0.09007835327836816,
+      "grad_norm": 5.688239496996797,
+      "learning_rate": 4.5031633792333455e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -548.0,
+      "logps/rejected": -624.0,
+      "loss": 0.4846,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -3.578125,
+      "rewards/margins": 1.078125,
+      "rewards/rejected": -4.65625,
+      "step": 2420
+    },
+    {
+      "epoch": 0.090450577878692,
+      "grad_norm": 6.594113719906323,
+      "learning_rate": 4.5217714923706737e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -494.0,
+      "logps/rejected": -604.0,
+      "loss": 0.4645,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -3.1875,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -4.4375,
+      "step": 2430
+    },
+    {
+      "epoch": 0.09082280247901584,
+      "grad_norm": 6.726664773853839,
+      "learning_rate": 4.5403796055080013e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -540.0,
+      "logps/rejected": -616.0,
+      "loss": 0.4543,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -3.28125,
+      "rewards/margins": 1.125,
+      "rewards/rejected": -4.40625,
+      "step": 2440
+    },
+    {
+      "epoch": 0.09119502707933967,
+      "grad_norm": 6.929343667676475,
+      "learning_rate": 4.558987718645329e-07,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -496.0,
+      "logps/rejected": -588.0,
+      "loss": 0.4583,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.265625,
+      "rewards/margins": 1.0546875,
+      "rewards/rejected": -4.3125,
+      "step": 2450
+    },
+    {
+      "epoch": 0.09156725167966351,
+      "grad_norm": 7.3622656961718835,
+      "learning_rate": 4.5775958317826567e-07,
+      "logits/chosen": -3.546875,
+      "logits/rejected": -3.5,
+      "logps/chosen": -552.0,
+      "logps/rejected": -604.0,
+      "loss": 0.4814,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.40625,
+      "rewards/margins": 0.91796875,
+      "rewards/rejected": -4.34375,
+      "step": 2460
+    },
+    {
+      "epoch": 0.09193947627998734,
+      "grad_norm": 6.779558467695923,
+      "learning_rate": 4.596203944919985e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -568.0,
+      "logps/rejected": -660.0,
+      "loss": 0.4841,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -3.796875,
+      "rewards/margins": 0.9765625,
+      "rewards/rejected": -4.78125,
+      "step": 2470
+    },
+    {
+      "epoch": 0.09231170088031118,
+      "grad_norm": 6.182721945798014,
+      "learning_rate": 4.614812058057313e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -548.0,
+      "logps/rejected": -600.0,
+      "loss": 0.4639,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -3.46875,
+      "rewards/margins": 0.890625,
+      "rewards/rejected": -4.375,
+      "step": 2480
+    },
+    {
+      "epoch": 0.09268392548063502,
+      "grad_norm": 5.874758959805965,
+      "learning_rate": 4.633420171194641e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.75,
+      "logps/chosen": -524.0,
+      "logps/rejected": -612.0,
+      "loss": 0.4943,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -3.296875,
+      "rewards/margins": 0.91796875,
+      "rewards/rejected": -4.21875,
+      "step": 2490
+    },
+    {
+      "epoch": 0.09305615008095885,
+      "grad_norm": 6.2020847366471985,
+      "learning_rate": 4.6520282843319684e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -532.0,
+      "logps/rejected": -616.0,
+      "loss": 0.4914,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -3.53125,
+      "rewards/margins": 0.93359375,
+      "rewards/rejected": -4.46875,
+      "step": 2500
+    },
+    {
+      "epoch": 0.09342837468128269,
+      "grad_norm": 7.270644951317942,
+      "learning_rate": 4.6706363974692966e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -552.0,
+      "logps/rejected": -620.0,
+      "loss": 0.4998,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.765625,
+      "rewards/margins": 0.97265625,
+      "rewards/rejected": -4.75,
+      "step": 2510
+    },
+    {
+      "epoch": 0.09380059928160653,
+      "grad_norm": 7.181528411096263,
+      "learning_rate": 4.6892445106066243e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -544.0,
+      "logps/rejected": -616.0,
+      "loss": 0.4715,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -3.4375,
+      "rewards/margins": 0.921875,
+      "rewards/rejected": -4.375,
+      "step": 2520
+    },
+    {
+      "epoch": 0.09417282388193035,
+      "grad_norm": 5.13615265178157,
+      "learning_rate": 4.707852623743952e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -592.0,
+      "logps/rejected": -664.0,
+      "loss": 0.4942,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.96875,
+      "rewards/margins": 1.0234375,
+      "rewards/rejected": -5.0,
+      "step": 2530
+    },
+    {
+      "epoch": 0.09454504848225419,
+      "grad_norm": 6.338054075847786,
+      "learning_rate": 4.7264607368812796e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -536.0,
+      "logps/rejected": -640.0,
+      "loss": 0.4802,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.53125,
+      "rewards/margins": 1.015625,
+      "rewards/rejected": -4.5625,
+      "step": 2540
+    },
+    {
+      "epoch": 0.09491727308257802,
+      "grad_norm": 7.379950985938358,
+      "learning_rate": 4.7450688500186083e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -584.0,
+      "logps/rejected": -660.0,
+      "loss": 0.4457,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.921875,
+      "rewards/margins": 0.97265625,
+      "rewards/rejected": -4.90625,
+      "step": 2550
+    },
+    {
+      "epoch": 0.09528949768290186,
+      "grad_norm": 7.680201919892533,
+      "learning_rate": 4.763676963155936e-07,
+      "logits/chosen": -3.59375,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -596.0,
+      "logps/rejected": -700.0,
+      "loss": 0.4737,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -4.09375,
+      "rewards/margins": 1.171875,
+      "rewards/rejected": -5.28125,
+      "step": 2560
+    },
+    {
+      "epoch": 0.0956617222832257,
+      "grad_norm": 8.454700735292763,
+      "learning_rate": 4.782285076293264e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -502.0,
+      "logps/rejected": -600.0,
+      "loss": 0.4657,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -3.203125,
+      "rewards/margins": 1.1171875,
+      "rewards/rejected": -4.3125,
+      "step": 2570
+    },
+    {
+      "epoch": 0.09603394688354953,
+      "grad_norm": 6.450134469200251,
+      "learning_rate": 4.800893189430591e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -556.0,
+      "logps/rejected": -648.0,
+      "loss": 0.4571,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -3.71875,
+      "rewards/margins": 1.1015625,
+      "rewards/rejected": -4.8125,
+      "step": 2580
+    },
+    {
+      "epoch": 0.09640617148387337,
+      "grad_norm": 5.940209728125408,
+      "learning_rate": 4.81950130256792e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -544.0,
+      "logps/rejected": -648.0,
+      "loss": 0.4613,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.625,
+      "rewards/margins": 1.2265625,
+      "rewards/rejected": -4.875,
+      "step": 2590
+    },
+    {
+      "epoch": 0.09677839608419721,
+      "grad_norm": 6.090480682330815,
+      "learning_rate": 4.838109415705247e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -528.0,
+      "logps/rejected": -608.0,
+      "loss": 0.4671,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.515625,
+      "rewards/margins": 0.921875,
+      "rewards/rejected": -4.4375,
+      "step": 2600
+    },
+    {
+      "epoch": 0.09715062068452104,
+      "grad_norm": 8.432048453882334,
+      "learning_rate": 4.856717528842575e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -540.0,
+      "logps/rejected": -624.0,
+      "loss": 0.4674,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -3.6875,
+      "rewards/margins": 0.94140625,
+      "rewards/rejected": -4.625,
+      "step": 2610
+    },
+    {
+      "epoch": 0.09752284528484488,
+      "grad_norm": 6.399208119692076,
+      "learning_rate": 4.875325641979903e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -488.0,
+      "logps/rejected": -572.0,
+      "loss": 0.4654,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -3.140625,
+      "rewards/margins": 1.0859375,
+      "rewards/rejected": -4.21875,
+      "step": 2620
+    },
+    {
+      "epoch": 0.09789506988516872,
+      "grad_norm": 6.048613419276874,
+      "learning_rate": 4.893933755117231e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -490.0,
+      "logps/rejected": -580.0,
+      "loss": 0.4703,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -3.0,
+      "rewards/margins": 1.0546875,
+      "rewards/rejected": -4.0625,
+      "step": 2630
+    },
+    {
+      "epoch": 0.09826729448549254,
+      "grad_norm": 5.926644748308866,
+      "learning_rate": 4.912541868254558e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -560.0,
+      "logps/rejected": -628.0,
+      "loss": 0.4862,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.609375,
+      "rewards/margins": 0.9765625,
+      "rewards/rejected": -4.5625,
+      "step": 2640
+    },
+    {
+      "epoch": 0.09863951908581638,
+      "grad_norm": 6.3255628420034835,
+      "learning_rate": 4.931149981391887e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -580.0,
+      "logps/rejected": -660.0,
+      "loss": 0.481,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -3.90625,
+      "rewards/margins": 0.921875,
+      "rewards/rejected": -4.8125,
+      "step": 2650
+    },
+    {
+      "epoch": 0.09901174368614021,
+      "grad_norm": 6.2756242030817955,
+      "learning_rate": 4.949758094529215e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -552.0,
+      "logps/rejected": -636.0,
+      "loss": 0.4846,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -3.671875,
+      "rewards/margins": 1.15625,
+      "rewards/rejected": -4.8125,
+      "step": 2660
+    },
+    {
+      "epoch": 0.09938396828646405,
+      "grad_norm": 6.211935806616986,
+      "learning_rate": 4.968366207666542e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -532.0,
+      "logps/rejected": -608.0,
+      "loss": 0.468,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.390625,
+      "rewards/margins": 1.0078125,
+      "rewards/rejected": -4.40625,
+      "step": 2670
+    },
+    {
+      "epoch": 0.09975619288678789,
+      "grad_norm": 7.110518955173448,
+      "learning_rate": 4.98697432080387e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -568.0,
+      "logps/rejected": -676.0,
+      "loss": 0.4666,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -3.8125,
+      "rewards/margins": 1.203125,
+      "rewards/rejected": -5.0,
+      "step": 2680
+    },
+    {
+      "epoch": 0.10012841748711172,
+      "grad_norm": 6.514305970839898,
+      "learning_rate": 4.999999810062151e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -568.0,
+      "logps/rejected": -624.0,
+      "loss": 0.4636,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -3.78125,
+      "rewards/margins": 0.84375,
+      "rewards/rejected": -4.625,
+      "step": 2690
+    },
+    {
+      "epoch": 0.10050064208743556,
+      "grad_norm": 7.306955106260322,
+      "learning_rate": 4.99999643339008e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -548.0,
+      "logps/rejected": -644.0,
+      "loss": 0.4637,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.703125,
+      "rewards/margins": 1.1015625,
+      "rewards/rejected": -4.8125,
+      "step": 2700
+    },
+    {
+      "epoch": 0.1008728666877594,
+      "grad_norm": 7.037716673658451,
+      "learning_rate": 4.999988835883477e-07,
+      "logits/chosen": -3.5625,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -548.0,
+      "logps/rejected": -632.0,
+      "loss": 0.4617,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.625,
+      "rewards/margins": 1.1015625,
+      "rewards/rejected": -4.71875,
+      "step": 2710
+    },
+    {
+      "epoch": 0.10124509128808323,
+      "grad_norm": 9.225651497922536,
+      "learning_rate": 4.999977017555171e-07,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.625,
+      "logps/chosen": -506.0,
+      "logps/rejected": -552.0,
+      "loss": 0.4767,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -3.015625,
+      "rewards/margins": 0.87109375,
+      "rewards/rejected": -3.890625,
+      "step": 2720
+    },
+    {
+      "epoch": 0.10161731588840707,
+      "grad_norm": 6.318792353332601,
+      "learning_rate": 4.999960978425113e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -532.0,
+      "logps/rejected": -596.0,
+      "loss": 0.4425,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.375,
+      "rewards/margins": 0.90625,
+      "rewards/rejected": -4.28125,
+      "step": 2730
+    },
+    {
+      "epoch": 0.1019895404887309,
+      "grad_norm": 7.445961075960431,
+      "learning_rate": 4.999940718520385e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -520.0,
+      "logps/rejected": -628.0,
+      "loss": 0.4751,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -3.453125,
+      "rewards/margins": 1.1171875,
+      "rewards/rejected": -4.5625,
+      "step": 2740
+    },
+    {
+      "epoch": 0.10236176508905473,
+      "grad_norm": 5.680014920863621,
+      "learning_rate": 4.999916237875191e-07,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -556.0,
+      "logps/rejected": -648.0,
+      "loss": 0.4433,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.734375,
+      "rewards/margins": 1.140625,
+      "rewards/rejected": -4.875,
+      "step": 2750
+    },
+    {
+      "epoch": 0.10273398968937857,
+      "grad_norm": 6.31065163996819,
+      "learning_rate": 4.999887536530864e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -568.0,
+      "logps/rejected": -652.0,
+      "loss": 0.4635,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.890625,
+      "rewards/margins": 1.09375,
+      "rewards/rejected": -4.96875,
+      "step": 2760
+    },
+    {
+      "epoch": 0.1031062142897024,
+      "grad_norm": 6.092896553196265,
+      "learning_rate": 4.999854614535859e-07,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -572.0,
+      "logps/rejected": -664.0,
+      "loss": 0.4584,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -3.859375,
+      "rewards/margins": 0.9296875,
+      "rewards/rejected": -4.78125,
+      "step": 2770
+    },
+    {
+      "epoch": 0.10347843889002624,
+      "grad_norm": 6.749870666157138,
+      "learning_rate": 4.999817471945762e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -560.0,
+      "logps/rejected": -636.0,
+      "loss": 0.4697,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -3.84375,
+      "rewards/margins": 0.9140625,
+      "rewards/rejected": -4.75,
+      "step": 2780
+    },
+    {
+      "epoch": 0.10385066349035008,
+      "grad_norm": 6.603742051475939,
+      "learning_rate": 4.99977610882328e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -592.0,
+      "logps/rejected": -684.0,
+      "loss": 0.4519,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.890625,
+      "rewards/margins": 1.1015625,
+      "rewards/rejected": -5.0,
+      "step": 2790
+    },
+    {
+      "epoch": 0.10422288809067391,
+      "grad_norm": 6.4601518863713405,
+      "learning_rate": 4.99973052523825e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -596.0,
+      "logps/rejected": -672.0,
+      "loss": 0.4633,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.0,
+      "rewards/margins": 0.9609375,
+      "rewards/rejected": -4.96875,
+      "step": 2800
+    },
+    {
+      "epoch": 0.10459511269099775,
+      "grad_norm": 7.234422738030497,
+      "learning_rate": 4.999680721267631e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -600.0,
+      "logps/rejected": -708.0,
+      "loss": 0.4633,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -4.09375,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -5.375,
+      "step": 2810
+    },
+    {
+      "epoch": 0.10496733729132159,
+      "grad_norm": 6.360501049327184,
+      "learning_rate": 4.999626696995509e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -576.0,
+      "logps/rejected": -652.0,
+      "loss": 0.4586,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.0,
+      "rewards/margins": 1.03125,
+      "rewards/rejected": -5.03125,
+      "step": 2820
+    },
+    {
+      "epoch": 0.10533956189164542,
+      "grad_norm": 7.093918545875841,
+      "learning_rate": 4.999568452513096e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -556.0,
+      "logps/rejected": -644.0,
+      "loss": 0.4756,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.640625,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -4.75,
+      "step": 2830
+    },
+    {
+      "epoch": 0.10571178649196926,
+      "grad_norm": 7.571224673884016,
+      "learning_rate": 4.999505987918727e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -548.0,
+      "logps/rejected": -640.0,
+      "loss": 0.4722,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -3.765625,
+      "rewards/margins": 1.015625,
+      "rewards/rejected": -4.78125,
+      "step": 2840
+    },
+    {
+      "epoch": 0.1060840110922931,
+      "grad_norm": 7.5048557009055035,
+      "learning_rate": 4.999439303317864e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -564.0,
+      "logps/rejected": -680.0,
+      "loss": 0.4814,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.84375,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -5.09375,
+      "step": 2850
+    },
+    {
+      "epoch": 0.10645623569261692,
+      "grad_norm": 9.07266968954365,
+      "learning_rate": 4.999368398823093e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.984375,
+      "logps/chosen": -540.0,
+      "logps/rejected": -616.0,
+      "loss": 0.4443,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -3.375,
+      "rewards/margins": 0.98046875,
+      "rewards/rejected": -4.375,
+      "step": 2860
+    },
+    {
+      "epoch": 0.10682846029294076,
+      "grad_norm": 7.251305724937602,
+      "learning_rate": 4.999293274554124e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -556.0,
+      "logps/rejected": -668.0,
+      "loss": 0.4664,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -3.71875,
+      "rewards/margins": 1.2421875,
+      "rewards/rejected": -4.9375,
+      "step": 2870
+    },
+    {
+      "epoch": 0.10720068489326459,
+      "grad_norm": 7.155242673178749,
+      "learning_rate": 4.999213930637793e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -576.0,
+      "logps/rejected": -688.0,
+      "loss": 0.457,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -3.859375,
+      "rewards/margins": 1.265625,
+      "rewards/rejected": -5.125,
+      "step": 2880
+    },
+    {
+      "epoch": 0.10757290949358843,
+      "grad_norm": 7.581529932420465,
+      "learning_rate": 4.999130367208059e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -592.0,
+      "logps/rejected": -676.0,
+      "loss": 0.4545,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.03125,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -5.15625,
+      "step": 2890
+    },
+    {
+      "epoch": 0.10794513409391226,
+      "grad_norm": 6.172272865891507,
+      "learning_rate": 4.999042584406005e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -576.0,
+      "logps/rejected": -684.0,
+      "loss": 0.4488,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.84375,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -5.09375,
+      "step": 2900
+    },
+    {
+      "epoch": 0.1083173586942361,
+      "grad_norm": 7.382912182723025,
+      "learning_rate": 4.998950582379836e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -600.0,
+      "logps/rejected": -700.0,
+      "loss": 0.4546,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.09375,
+      "rewards/margins": 1.125,
+      "rewards/rejected": -5.21875,
+      "step": 2910
+    },
+    {
+      "epoch": 0.10868958329455994,
+      "grad_norm": 5.758207620223827,
+      "learning_rate": 4.998854361284885e-07,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -524.0,
+      "logps/rejected": -616.0,
+      "loss": 0.4686,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -3.34375,
+      "rewards/margins": 0.98828125,
+      "rewards/rejected": -4.34375,
+      "step": 2920
+    },
+    {
+      "epoch": 0.10906180789488377,
+      "grad_norm": 6.845912737324073,
+      "learning_rate": 4.998753921283606e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -564.0,
+      "logps/rejected": -656.0,
+      "loss": 0.461,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -3.890625,
+      "rewards/margins": 1.015625,
+      "rewards/rejected": -4.90625,
+      "step": 2930
+    },
+    {
+      "epoch": 0.10943403249520761,
+      "grad_norm": 5.982964820305056,
+      "learning_rate": 4.998649262545573e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -556.0,
+      "logps/rejected": -660.0,
+      "loss": 0.4531,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.796875,
+      "rewards/margins": 1.2265625,
+      "rewards/rejected": -5.03125,
+      "step": 2940
+    },
+    {
+      "epoch": 0.10980625709553145,
+      "grad_norm": 6.292830318163489,
+      "learning_rate": 4.998540385247487e-07,
+      "logits/chosen": -3.640625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -520.0,
+      "logps/rejected": -584.0,
+      "loss": 0.4892,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -3.390625,
+      "rewards/margins": 0.8359375,
+      "rewards/rejected": -4.21875,
+      "step": 2950
+    },
+    {
+      "epoch": 0.11017848169585528,
+      "grad_norm": 6.841263806372628,
+      "learning_rate": 4.998427289573168e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -536.0,
+      "logps/rejected": -624.0,
+      "loss": 0.4887,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -3.5625,
+      "rewards/margins": 0.93359375,
+      "rewards/rejected": -4.5,
+      "step": 2960
+    },
+    {
+      "epoch": 0.11055070629617911,
+      "grad_norm": 5.26619136368238,
+      "learning_rate": 4.998309975713561e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -556.0,
+      "logps/rejected": -636.0,
+      "loss": 0.4815,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.84375,
+      "rewards/margins": 0.9375,
+      "rewards/rejected": -4.78125,
+      "step": 2970
+    },
+    {
+      "epoch": 0.11092293089650294,
+      "grad_norm": 5.953078813857384,
+      "learning_rate": 4.99818844386673e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -604.0,
+      "logps/rejected": -696.0,
+      "loss": 0.4653,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.28125,
+      "rewards/margins": 1.015625,
+      "rewards/rejected": -5.28125,
+      "step": 2980
+    },
+    {
+      "epoch": 0.11129515549682678,
+      "grad_norm": 6.937288858216694,
+      "learning_rate": 4.998062694237862e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -620.0,
+      "logps/rejected": -736.0,
+      "loss": 0.4545,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -4.3125,
+      "rewards/margins": 1.203125,
+      "rewards/rejected": -5.5,
+      "step": 2990
+    },
+    {
+      "epoch": 0.11166738009715062,
+      "grad_norm": 5.54695879592323,
+      "learning_rate": 4.997932727039265e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -548.0,
+      "logps/rejected": -640.0,
+      "loss": 0.4774,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.53125,
+      "rewards/margins": 1.078125,
+      "rewards/rejected": -4.625,
+      "step": 3000
+    },
+    {
+      "epoch": 0.11203960469747445,
+      "grad_norm": 6.279746283077127,
+      "learning_rate": 4.997798542490368e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -552.0,
+      "logps/rejected": -652.0,
+      "loss": 0.4379,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.78125,
+      "rewards/margins": 0.9921875,
+      "rewards/rejected": -4.78125,
+      "step": 3010
+    },
+    {
+      "epoch": 0.11241182929779829,
+      "grad_norm": 6.247032126592368,
+      "learning_rate": 4.997660140817717e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -632.0,
+      "logps/rejected": -732.0,
+      "loss": 0.4317,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -4.4375,
+      "rewards/margins": 1.1953125,
+      "rewards/rejected": -5.625,
+      "step": 3020
+    },
+    {
+      "epoch": 0.11278405389812213,
+      "grad_norm": 6.999244480996859,
+      "learning_rate": 4.997517522254984e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.40625,
+      "logps/chosen": -572.0,
+      "logps/rejected": -656.0,
+      "loss": 0.4495,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.796875,
+      "rewards/margins": 1.0546875,
+      "rewards/rejected": -4.84375,
+      "step": 3030
+    },
+    {
+      "epoch": 0.11315627849844596,
+      "grad_norm": 8.06872233784361,
+      "learning_rate": 4.997370687042956e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -624.0,
+      "logps/rejected": -712.0,
+      "loss": 0.4567,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.3125,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -5.4375,
+      "step": 3040
+    },
+    {
+      "epoch": 0.1135285030987698,
+      "grad_norm": 6.947687716311752,
+      "learning_rate": 4.997219635429538e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -580.0,
+      "logps/rejected": -704.0,
+      "loss": 0.4423,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -3.953125,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -5.21875,
+      "step": 3050
+    },
+    {
+      "epoch": 0.11390072769909364,
+      "grad_norm": 7.29382971914937,
+      "learning_rate": 4.997064367669758e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -600.0,
+      "logps/rejected": -692.0,
+      "loss": 0.4572,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.21875,
+      "rewards/margins": 0.9609375,
+      "rewards/rejected": -5.1875,
+      "step": 3060
+    },
+    {
+      "epoch": 0.11427295229941747,
+      "grad_norm": 5.646650773332122,
+      "learning_rate": 4.996904884025761e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -580.0,
+      "logps/rejected": -668.0,
+      "loss": 0.4628,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -3.828125,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -5.09375,
+      "step": 3070
+    },
+    {
+      "epoch": 0.11464517689974131,
+      "grad_norm": 5.925473322691006,
+      "learning_rate": 4.996741184766806e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -580.0,
+      "logps/rejected": -696.0,
+      "loss": 0.4445,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.0625,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -5.34375,
+      "step": 3080
+    },
+    {
+      "epoch": 0.11501740150006513,
+      "grad_norm": 6.7048257156530084,
+      "learning_rate": 4.996573270169275e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.875,
+      "logps/chosen": -572.0,
+      "logps/rejected": -652.0,
+      "loss": 0.4471,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.78125,
+      "rewards/margins": 1.078125,
+      "rewards/rejected": -4.875,
+      "step": 3090
+    },
+    {
+      "epoch": 0.11538962610038897,
+      "grad_norm": 6.293377102514235,
+      "learning_rate": 4.996401140516663e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -636.0,
+      "logps/rejected": -736.0,
+      "loss": 0.4436,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.53125,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -5.625,
+      "step": 3100
+    },
+    {
+      "epoch": 0.1157618507007128,
+      "grad_norm": 8.108184400087252,
+      "learning_rate": 4.996224796099584e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -560.0,
+      "logps/rejected": -668.0,
+      "loss": 0.4446,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -3.78125,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -5.0625,
+      "step": 3110
+    },
+    {
+      "epoch": 0.11613407530103664,
+      "grad_norm": 11.699226051929289,
+      "learning_rate": 4.996044237215765e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -596.0,
+      "logps/rejected": -704.0,
+      "loss": 0.4512,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -4.15625,
+      "rewards/margins": 1.140625,
+      "rewards/rejected": -5.28125,
+      "step": 3120
+    },
+    {
+      "epoch": 0.11650629990136048,
+      "grad_norm": 8.919296260065547,
+      "learning_rate": 4.995859464170051e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -620.0,
+      "logps/rejected": -728.0,
+      "loss": 0.4417,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.40625,
+      "rewards/margins": 1.1640625,
+      "rewards/rejected": -5.5625,
+      "step": 3130
+    },
+    {
+      "epoch": 0.11687852450168432,
+      "grad_norm": 6.148343995228796,
+      "learning_rate": 4.995670477274402e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -568.0,
+      "logps/rejected": -688.0,
+      "loss": 0.4663,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.9375,
+      "rewards/margins": 1.1796875,
+      "rewards/rejected": -5.125,
+      "step": 3140
+    },
+    {
+      "epoch": 0.11725074910200815,
+      "grad_norm": 6.671053026062253,
+      "learning_rate": 4.995477276847889e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -592.0,
+      "logps/rejected": -704.0,
+      "loss": 0.4481,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.21875,
+      "rewards/margins": 1.0625,
+      "rewards/rejected": -5.25,
+      "step": 3150
+    },
+    {
+      "epoch": 0.11762297370233199,
+      "grad_norm": 6.117612593879977,
+      "learning_rate": 4.995279863216701e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -556.0,
+      "logps/rejected": -644.0,
+      "loss": 0.4363,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -3.84375,
+      "rewards/margins": 1.0703125,
+      "rewards/rejected": -4.90625,
+      "step": 3160
+    },
+    {
+      "epoch": 0.11799519830265583,
+      "grad_norm": 7.176202883807436,
+      "learning_rate": 4.995078236714138e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -584.0,
+      "logps/rejected": -748.0,
+      "loss": 0.4499,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -4.03125,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -5.59375,
+      "step": 3170
+    },
+    {
+      "epoch": 0.11836742290297966,
+      "grad_norm": 6.911854486517442,
+      "learning_rate": 4.994872397680615e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -592.0,
+      "logps/rejected": -688.0,
+      "loss": 0.4671,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.21875,
+      "rewards/margins": 0.96484375,
+      "rewards/rejected": -5.15625,
+      "step": 3180
+    },
+    {
+      "epoch": 0.1187396475033035,
+      "grad_norm": 10.197222141423639,
+      "learning_rate": 4.994662346463655e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.75,
+      "logps/chosen": -592.0,
+      "logps/rejected": -664.0,
+      "loss": 0.4628,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.0,
+      "rewards/margins": 1.0546875,
+      "rewards/rejected": -5.0625,
+      "step": 3190
+    },
+    {
+      "epoch": 0.11911187210362732,
+      "grad_norm": 8.779132089887378,
+      "learning_rate": 4.994448083417896e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -616.0,
+      "logps/rejected": -740.0,
+      "loss": 0.4429,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.375,
+      "rewards/margins": 1.2109375,
+      "rewards/rejected": -5.59375,
+      "step": 3200
+    },
+    {
+      "epoch": 0.11948409670395116,
+      "grad_norm": 6.372450601862111,
+      "learning_rate": 4.994229608905087e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -596.0,
+      "logps/rejected": -768.0,
+      "loss": 0.4574,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -4.09375,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -5.875,
+      "step": 3210
+    },
+    {
+      "epoch": 0.119856321304275,
+      "grad_norm": 6.441216649020593,
+      "learning_rate": 4.994006923294085e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -568.0,
+      "logps/rejected": -680.0,
+      "loss": 0.457,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.0,
+      "rewards/margins": 1.15625,
+      "rewards/rejected": -5.1875,
+      "step": 3220
+    },
+    {
+      "epoch": 0.12022854590459883,
+      "grad_norm": 6.4785093399544555,
+      "learning_rate": 4.993780026960859e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -620.0,
+      "logps/rejected": -700.0,
+      "loss": 0.4752,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.09375,
+      "rewards/margins": 1.1015625,
+      "rewards/rejected": -5.1875,
+      "step": 3230
+    },
+    {
+      "epoch": 0.12060077050492267,
+      "grad_norm": 5.9512176119157045,
+      "learning_rate": 4.993548920288487e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -584.0,
+      "logps/rejected": -688.0,
+      "loss": 0.4618,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -4.0,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -5.25,
+      "step": 3240
+    },
+    {
+      "epoch": 0.1209729951052465,
+      "grad_norm": 6.575306736569084,
+      "learning_rate": 4.993313603667152e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -588.0,
+      "logps/rejected": -692.0,
+      "loss": 0.4422,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.15625,
+      "rewards/margins": 1.1484375,
+      "rewards/rejected": -5.28125,
+      "step": 3250
+    },
+    {
+      "epoch": 0.12134521970557034,
+      "grad_norm": 6.0838794532038225,
+      "learning_rate": 4.993074077494151e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -592.0,
+      "logps/rejected": -728.0,
+      "loss": 0.449,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.25,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -5.65625,
+      "step": 3260
+    },
+    {
+      "epoch": 0.12171744430589418,
+      "grad_norm": 7.95167940015155,
+      "learning_rate": 4.992830342173883e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -584.0,
+      "logps/rejected": -692.0,
+      "loss": 0.4238,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -3.796875,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -5.125,
+      "step": 3270
+    },
+    {
+      "epoch": 0.12208966890621802,
+      "grad_norm": 9.697344826786047,
+      "learning_rate": 4.992582398117854e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -580.0,
+      "logps/rejected": -716.0,
+      "loss": 0.45,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.09375,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -5.6875,
+      "step": 3280
+    },
+    {
+      "epoch": 0.12246189350654185,
+      "grad_norm": 6.074554707253789,
+      "learning_rate": 4.992330245744679e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -580.0,
+      "logps/rejected": -692.0,
+      "loss": 0.4562,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -3.96875,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -5.21875,
+      "step": 3290
+    },
+    {
+      "epoch": 0.12283411810686569,
+      "grad_norm": 6.037098881295751,
+      "learning_rate": 4.992073885480076e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.9375,
+      "logps/chosen": -572.0,
+      "logps/rejected": -624.0,
+      "loss": 0.4659,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -3.765625,
+      "rewards/margins": 0.88671875,
+      "rewards/rejected": -4.65625,
+      "step": 3300
+    },
+    {
+      "epoch": 0.12320634270718951,
+      "grad_norm": 6.821962308345998,
+      "learning_rate": 4.991813317756866e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -600.0,
+      "logps/rejected": -704.0,
+      "loss": 0.4542,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.1875,
+      "rewards/margins": 1.28125,
+      "rewards/rejected": -5.46875,
+      "step": 3310
+    },
+    {
+      "epoch": 0.12357856730751335,
+      "grad_norm": 7.229520240147668,
+      "learning_rate": 4.991548543014975e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -628.0,
+      "logps/rejected": -740.0,
+      "loss": 0.4769,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.4375,
+      "rewards/margins": 1.3125,
+      "rewards/rejected": -5.75,
+      "step": 3320
+    },
+    {
+      "epoch": 0.12395079190783719,
+      "grad_norm": 6.84456725737557,
+      "learning_rate": 4.991279561701434e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -572.0,
+      "logps/rejected": -692.0,
+      "loss": 0.4502,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -3.875,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -5.28125,
+      "step": 3330
+    },
+    {
+      "epoch": 0.12432301650816102,
+      "grad_norm": 7.139306112495771,
+      "learning_rate": 4.991006374270371e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -640.0,
+      "logps/rejected": -744.0,
+      "loss": 0.4545,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.4375,
+      "rewards/margins": 1.1796875,
+      "rewards/rejected": -5.625,
+      "step": 3340
+    },
+    {
+      "epoch": 0.12469524110848486,
+      "grad_norm": 8.202464001933487,
+      "learning_rate": 4.990728981183019e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -624.0,
+      "logps/rejected": -728.0,
+      "loss": 0.457,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -4.25,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -5.53125,
+      "step": 3350
+    },
+    {
+      "epoch": 0.1250674657088087,
+      "grad_norm": 6.183831591940218,
+      "learning_rate": 4.990447382907713e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -596.0,
+      "logps/rejected": -696.0,
+      "loss": 0.4735,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.15625,
+      "rewards/margins": 1.09375,
+      "rewards/rejected": -5.25,
+      "step": 3360
+    },
+    {
+      "epoch": 0.12543969030913252,
+      "grad_norm": 5.74197797394421,
+      "learning_rate": 4.990161579919882e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -616.0,
+      "logps/rejected": -716.0,
+      "loss": 0.4452,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.15625,
+      "rewards/margins": 1.1953125,
+      "rewards/rejected": -5.34375,
+      "step": 3370
+    },
+    {
+      "epoch": 0.12581191490945637,
+      "grad_norm": 5.972762359020475,
+      "learning_rate": 4.98987157270206e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -580.0,
+      "logps/rejected": -672.0,
+      "loss": 0.4539,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.890625,
+      "rewards/margins": 1.125,
+      "rewards/rejected": -5.03125,
+      "step": 3380
+    },
+    {
+      "epoch": 0.1261841395097802,
+      "grad_norm": 6.340524766277757,
+      "learning_rate": 4.989577361743875e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -604.0,
+      "logps/rejected": -696.0,
+      "loss": 0.4578,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.125,
+      "rewards/margins": 1.046875,
+      "rewards/rejected": -5.1875,
+      "step": 3390
+    },
+    {
+      "epoch": 0.12655636411010404,
+      "grad_norm": 6.680137616172863,
+      "learning_rate": 4.989278947542056e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.421875,
+      "logps/chosen": -648.0,
+      "logps/rejected": -744.0,
+      "loss": 0.4125,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.375,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -5.65625,
+      "step": 3400
+    },
+    {
+      "epoch": 0.12692858871042786,
+      "grad_norm": 7.890719574539212,
+      "learning_rate": 4.988976330600426e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -592.0,
+      "logps/rejected": -712.0,
+      "loss": 0.4605,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -4.21875,
+      "rewards/margins": 1.375,
+      "rewards/rejected": -5.59375,
+      "step": 3410
+    },
+    {
+      "epoch": 0.12730081331075171,
+      "grad_norm": 5.906282275914477,
+      "learning_rate": 4.988669511429902e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -608.0,
+      "logps/rejected": -704.0,
+      "loss": 0.4635,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -4.1875,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -5.3125,
+      "step": 3420
+    },
+    {
+      "epoch": 0.12767303791107554,
+      "grad_norm": 5.3903668131739835,
+      "learning_rate": 4.988358490548501e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -596.0,
+      "logps/rejected": -684.0,
+      "loss": 0.4765,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -3.921875,
+      "rewards/margins": 1.1484375,
+      "rewards/rejected": -5.0625,
+      "step": 3430
+    },
+    {
+      "epoch": 0.1280452625113994,
+      "grad_norm": 4.961272687874735,
+      "learning_rate": 4.988043268481329e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -616.0,
+      "logps/rejected": -748.0,
+      "loss": 0.4262,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -4.28125,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -5.75,
+      "step": 3440
+    },
+    {
+      "epoch": 0.1284174871117232,
+      "grad_norm": 8.068168331858757,
+      "learning_rate": 4.987723845760587e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -624.0,
+      "logps/rejected": -740.0,
+      "loss": 0.442,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -4.40625,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -5.6875,
+      "step": 3450
+    },
+    {
+      "epoch": 0.12878971171204706,
+      "grad_norm": 6.795034479962609,
+      "learning_rate": 4.987400222925569e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -652.0,
+      "logps/rejected": -748.0,
+      "loss": 0.4452,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.71875,
+      "rewards/margins": 0.99609375,
+      "rewards/rejected": -5.71875,
+      "step": 3460
+    },
+    {
+      "epoch": 0.12916193631237088,
+      "grad_norm": 5.796036927698797,
+      "learning_rate": 4.987072400522658e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -616.0,
+      "logps/rejected": -736.0,
+      "loss": 0.4276,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.78125,
+      "rewards/margins": 1.2109375,
+      "rewards/rejected": -5.96875,
+      "step": 3470
+    },
+    {
+      "epoch": 0.1295341609126947,
+      "grad_norm": 8.92117123663945,
+      "learning_rate": 4.986740379105328e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -604.0,
+      "logps/rejected": -712.0,
+      "loss": 0.4612,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.28125,
+      "rewards/margins": 1.0078125,
+      "rewards/rejected": -5.28125,
+      "step": 3480
+    },
+    {
+      "epoch": 0.12990638551301856,
+      "grad_norm": 5.223560106131392,
+      "learning_rate": 4.986404159234145e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -624.0,
+      "logps/rejected": -728.0,
+      "loss": 0.4321,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.21875,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -5.5,
+      "step": 3490
+    },
+    {
+      "epoch": 0.13027861011334238,
+      "grad_norm": 5.871849600072611,
+      "learning_rate": 4.986063741476759e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -664.0,
+      "logps/rejected": -784.0,
+      "loss": 0.4538,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -4.625,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -6.03125,
+      "step": 3500
+    },
+    {
+      "epoch": 0.13065083471366623,
+      "grad_norm": 6.987648018826682,
+      "learning_rate": 4.985719126407912e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -660.0,
+      "logps/rejected": -768.0,
+      "loss": 0.4375,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -4.78125,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -6.15625,
+      "step": 3510
+    },
+    {
+      "epoch": 0.13102305931399005,
+      "grad_norm": 5.901780901766237,
+      "learning_rate": 4.985370314609426e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -616.0,
+      "logps/rejected": -744.0,
+      "loss": 0.4456,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -4.21875,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -5.65625,
+      "step": 3520
+    },
+    {
+      "epoch": 0.1313952839143139,
+      "grad_norm": 7.071445326243194,
+      "learning_rate": 4.985017306670216e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -616.0,
+      "logps/rejected": -748.0,
+      "loss": 0.4518,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -4.25,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -5.78125,
+      "step": 3530
+    },
+    {
+      "epoch": 0.13176750851463773,
+      "grad_norm": 5.212204350054308,
+      "learning_rate": 4.984660103186278e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -676.0,
+      "logps/rejected": -796.0,
+      "loss": 0.4563,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -4.84375,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -6.09375,
+      "step": 3540
+    },
+    {
+      "epoch": 0.13213973311496158,
+      "grad_norm": 5.558822016819327,
+      "learning_rate": 4.984298704760689e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -656.0,
+      "logps/rejected": -756.0,
+      "loss": 0.4332,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.625,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -5.75,
+      "step": 3550
+    },
+    {
+      "epoch": 0.1325119577152854,
+      "grad_norm": 6.967443631896487,
+      "learning_rate": 4.983933112003615e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.75,
+      "logps/chosen": -624.0,
+      "logps/rejected": -752.0,
+      "loss": 0.439,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -4.4375,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -5.9375,
+      "step": 3560
+    },
+    {
+      "epoch": 0.13288418231560925,
+      "grad_norm": 6.3598928852554995,
+      "learning_rate": 4.983563325532295e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -644.0,
+      "logps/rejected": -764.0,
+      "loss": 0.4427,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.71875,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -6.0625,
+      "step": 3570
+    },
+    {
+      "epoch": 0.13325640691593307,
+      "grad_norm": 7.022481716988861,
+      "learning_rate": 4.983189345971056e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -644.0,
+      "logps/rejected": -744.0,
+      "loss": 0.4383,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.5625,
+      "rewards/margins": 1.171875,
+      "rewards/rejected": -5.71875,
+      "step": 3580
+    },
+    {
+      "epoch": 0.1336286315162569,
+      "grad_norm": 6.780733486647057,
+      "learning_rate": 4.982811173951301e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -616.0,
+      "logps/rejected": -720.0,
+      "loss": 0.4532,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.5625,
+      "rewards/margins": 1.1640625,
+      "rewards/rejected": -5.71875,
+      "step": 3590
+    },
+    {
+      "epoch": 0.13400085611658075,
+      "grad_norm": 5.637307376742751,
+      "learning_rate": 4.982428810111512e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.875,
+      "logps/chosen": -652.0,
+      "logps/rejected": -744.0,
+      "loss": 0.4605,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.5625,
+      "rewards/margins": 0.98046875,
+      "rewards/rejected": -5.53125,
+      "step": 3600
+    },
+    {
+      "epoch": 0.13437308071690457,
+      "grad_norm": 5.715037906817475,
+      "learning_rate": 4.982042255097245e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -624.0,
+      "logps/rejected": -744.0,
+      "loss": 0.4317,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -4.34375,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -5.75,
+      "step": 3610
+    },
+    {
+      "epoch": 0.13474530531722842,
+      "grad_norm": 7.235399557709921,
+      "learning_rate": 4.981651509561137e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -632.0,
+      "logps/rejected": -728.0,
+      "loss": 0.4471,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -4.4375,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -5.6875,
+      "step": 3620
+    },
+    {
+      "epoch": 0.13511752991755224,
+      "grad_norm": 6.8658883846755705,
+      "learning_rate": 4.981256574162897e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -628.0,
+      "logps/rejected": -760.0,
+      "loss": 0.4505,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -4.375,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -6.0,
+      "step": 3630
+    },
+    {
+      "epoch": 0.1354897545178761,
+      "grad_norm": 6.783324383771678,
+      "learning_rate": 4.980857449569309e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -636.0,
+      "logps/rejected": -748.0,
+      "loss": 0.4672,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -4.53125,
+      "rewards/margins": 1.2265625,
+      "rewards/rejected": -5.75,
+      "step": 3640
+    },
+    {
+      "epoch": 0.13586197911819992,
+      "grad_norm": 6.8706527455369395,
+      "learning_rate": 4.98045413645423e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -644.0,
+      "logps/rejected": -764.0,
+      "loss": 0.4365,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -4.375,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -5.84375,
+      "step": 3650
+    },
+    {
+      "epoch": 0.13623420371852377,
+      "grad_norm": 5.359628903637225,
+      "learning_rate": 4.980046635498589e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.390625,
+      "logps/chosen": -628.0,
+      "logps/rejected": -756.0,
+      "loss": 0.4544,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.59375,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -5.9375,
+      "step": 3660
+    },
+    {
+      "epoch": 0.1366064283188476,
+      "grad_norm": 6.08079369798337,
+      "learning_rate": 4.979634947390381e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -632.0,
+      "logps/rejected": -748.0,
+      "loss": 0.4597,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.4375,
+      "rewards/margins": 1.28125,
+      "rewards/rejected": -5.6875,
+      "step": 3670
+    },
+    {
+      "epoch": 0.13697865291917144,
+      "grad_norm": 5.318130278514382,
+      "learning_rate": 4.979219072824678e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -600.0,
+      "logps/rejected": -712.0,
+      "loss": 0.4446,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.3125,
+      "rewards/margins": 1.359375,
+      "rewards/rejected": -5.65625,
+      "step": 3680
+    },
+    {
+      "epoch": 0.13735087751949526,
+      "grad_norm": 5.614603740865042,
+      "learning_rate": 4.978799012503614e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -620.0,
+      "logps/rejected": -756.0,
+      "loss": 0.436,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -4.3125,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -5.75,
+      "step": 3690
+    },
+    {
+      "epoch": 0.13772310211981909,
+      "grad_norm": 8.205202469508833,
+      "learning_rate": 4.978374767136391e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -652.0,
+      "logps/rejected": -764.0,
+      "loss": 0.4578,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.75,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -6.0625,
+      "step": 3700
+    },
+    {
+      "epoch": 0.13809532672014294,
+      "grad_norm": 8.125967976566125,
+      "learning_rate": 4.977946337439281e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -668.0,
+      "logps/rejected": -748.0,
+      "loss": 0.4321,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -4.4375,
+      "rewards/margins": 1.265625,
+      "rewards/rejected": -5.71875,
+      "step": 3710
+    },
+    {
+      "epoch": 0.13846755132046676,
+      "grad_norm": 6.1826290654240745,
+      "learning_rate": 4.977513724135615e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -620.0,
+      "logps/rejected": -756.0,
+      "loss": 0.4453,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -4.40625,
+      "rewards/margins": 1.359375,
+      "rewards/rejected": -5.75,
+      "step": 3720
+    },
+    {
+      "epoch": 0.1388397759207906,
+      "grad_norm": 6.257212281885236,
+      "learning_rate": 4.977076927955792e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -644.0,
+      "logps/rejected": -740.0,
+      "loss": 0.4561,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.53125,
+      "rewards/margins": 1.1875,
+      "rewards/rejected": -5.75,
+      "step": 3730
+    },
+    {
+      "epoch": 0.13921200052111443,
+      "grad_norm": 5.8655856333891565,
+      "learning_rate": 4.976635949637268e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -632.0,
+      "logps/rejected": -744.0,
+      "loss": 0.4524,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.5,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -5.84375,
+      "step": 3740
+    },
+    {
+      "epoch": 0.13958422512143828,
+      "grad_norm": 8.972857669125604,
+      "learning_rate": 4.976190789924563e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -608.0,
+      "logps/rejected": -712.0,
+      "loss": 0.4422,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.3125,
+      "rewards/margins": 1.1640625,
+      "rewards/rejected": -5.5,
+      "step": 3750
+    },
+    {
+      "epoch": 0.1399564497217621,
+      "grad_norm": 5.990912754321117,
+      "learning_rate": 4.975741449569258e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -608.0,
+      "logps/rejected": -692.0,
+      "loss": 0.4611,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.46875,
+      "rewards/margins": 0.9375,
+      "rewards/rejected": -5.40625,
+      "step": 3760
+    },
+    {
+      "epoch": 0.14032867432208596,
+      "grad_norm": 5.121443166601498,
+      "learning_rate": 4.975287929329989e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -640.0,
+      "logps/rejected": -752.0,
+      "loss": 0.4561,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -4.46875,
+      "rewards/margins": 1.2421875,
+      "rewards/rejected": -5.71875,
+      "step": 3770
+    },
+    {
+      "epoch": 0.14070089892240978,
+      "grad_norm": 6.311626680869443,
+      "learning_rate": 4.974830229972452e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -608.0,
+      "logps/rejected": -732.0,
+      "loss": 0.4372,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.40625,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": -5.75,
+      "step": 3780
+    },
+    {
+      "epoch": 0.14107312352273363,
+      "grad_norm": 5.843621785655601,
+      "learning_rate": 4.974368352269397e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -624.0,
+      "logps/rejected": -756.0,
+      "loss": 0.4263,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -4.53125,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -5.84375,
+      "step": 3790
+    },
+    {
+      "epoch": 0.14144534812305745,
+      "grad_norm": 6.804867664066004,
+      "learning_rate": 4.973902297000628e-07,
+      "logits/chosen": -3.625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -644.0,
+      "logps/rejected": -744.0,
+      "loss": 0.4485,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.6875,
+      "rewards/margins": 1.1875,
+      "rewards/rejected": -5.875,
+      "step": 3800
+    },
+    {
+      "epoch": 0.14181757272338127,
+      "grad_norm": 6.218503436352911,
+      "learning_rate": 4.973432064953004e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -684.0,
+      "logps/rejected": -804.0,
+      "loss": 0.4724,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -5.0,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -6.25,
+      "step": 3810
+    },
+    {
+      "epoch": 0.14218979732370512,
+      "grad_norm": 5.2602914797478455,
+      "learning_rate": 4.972957656920434e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -616.0,
+      "logps/rejected": -748.0,
+      "loss": 0.435,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -4.34375,
+      "rewards/margins": 1.359375,
+      "rewards/rejected": -5.71875,
+      "step": 3820
+    },
+    {
+      "epoch": 0.14256202192402895,
+      "grad_norm": 5.770672774714267,
+      "learning_rate": 4.972479073703879e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -700.0,
+      "logps/rejected": -796.0,
+      "loss": 0.4591,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -4.875,
+      "rewards/margins": 1.125,
+      "rewards/rejected": -6.0,
+      "step": 3830
+    },
+    {
+      "epoch": 0.1429342465243528,
+      "grad_norm": 6.836653213684203,
+      "learning_rate": 4.971996316111347e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -712.0,
+      "logps/rejected": -808.0,
+      "loss": 0.4294,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -5.125,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -6.46875,
+      "step": 3840
+    },
+    {
+      "epoch": 0.14330647112467662,
+      "grad_norm": 6.904003354824577,
+      "learning_rate": 4.971509384957899e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.75,
+      "logps/chosen": -636.0,
+      "logps/rejected": -760.0,
+      "loss": 0.4264,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.78125,
+      "rewards/margins": 1.1953125,
+      "rewards/rejected": -6.0,
+      "step": 3850
+    },
+    {
+      "epoch": 0.14367869572500047,
+      "grad_norm": 5.902140943109142,
+      "learning_rate": 4.971018281065632e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -684.0,
+      "logps/rejected": -792.0,
+      "loss": 0.4138,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -4.8125,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -6.1875,
+      "step": 3860
+    },
+    {
+      "epoch": 0.1440509203253243,
+      "grad_norm": 5.803101839035741,
+      "learning_rate": 4.9705230052637e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -680.0,
+      "logps/rejected": -776.0,
+      "loss": 0.4571,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -4.6875,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -6.0,
+      "step": 3870
+    },
+    {
+      "epoch": 0.14442314492564814,
+      "grad_norm": 6.955879230784433,
+      "learning_rate": 4.970023558388294e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -652.0,
+      "logps/rejected": -744.0,
+      "loss": 0.445,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.625,
+      "rewards/margins": 1.171875,
+      "rewards/rejected": -5.8125,
+      "step": 3880
+    },
+    {
+      "epoch": 0.14479536952597197,
+      "grad_norm": 5.2387046632240235,
+      "learning_rate": 4.969519941282647e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -656.0,
+      "logps/rejected": -740.0,
+      "loss": 0.4292,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.5625,
+      "rewards/margins": 1.046875,
+      "rewards/rejected": -5.59375,
+      "step": 3890
+    },
+    {
+      "epoch": 0.14516759412629582,
+      "grad_norm": 26.07099639219989,
+      "learning_rate": 4.969012154797034e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -636.0,
+      "logps/rejected": -764.0,
+      "loss": 0.4395,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -4.53125,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -6.03125,
+      "step": 3900
+    },
+    {
+      "epoch": 0.14553981872661964,
+      "grad_norm": 7.604734590879799,
+      "learning_rate": 4.96850019978877e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -636.0,
+      "logps/rejected": -776.0,
+      "loss": 0.4515,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -4.65625,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -6.0625,
+      "step": 3910
+    },
+    {
+      "epoch": 0.1459120433269435,
+      "grad_norm": 5.464158017535742,
+      "learning_rate": 4.967984077122207e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -596.0,
+      "logps/rejected": -700.0,
+      "loss": 0.4641,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.125,
+      "rewards/margins": 1.1640625,
+      "rewards/rejected": -5.3125,
+      "step": 3920
+    },
+    {
+      "epoch": 0.14628426792726731,
+      "grad_norm": 6.304001836376101,
+      "learning_rate": 4.967463787668734e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -640.0,
+      "logps/rejected": -752.0,
+      "loss": 0.4384,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -4.34375,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": -5.6875,
+      "step": 3930
+    },
+    {
+      "epoch": 0.14665649252759114,
+      "grad_norm": 5.931843149222858,
+      "learning_rate": 4.966939332306773e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -672.0,
+      "logps/rejected": -792.0,
+      "loss": 0.45,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.875,
+      "rewards/margins": 1.125,
+      "rewards/rejected": -6.0,
+      "step": 3940
+    },
+    {
+      "epoch": 0.147028717127915,
+      "grad_norm": 5.714391921437342,
+      "learning_rate": 4.966410711921784e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -644.0,
+      "logps/rejected": -728.0,
+      "loss": 0.4353,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.75,
+      "rewards/margins": 0.9453125,
+      "rewards/rejected": -5.6875,
+      "step": 3950
+    },
+    {
+      "epoch": 0.1474009417282388,
+      "grad_norm": 5.96946748930747,
+      "learning_rate": 4.965877927406252e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -660.0,
+      "logps/rejected": -752.0,
+      "loss": 0.4478,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -4.875,
+      "rewards/margins": 1.140625,
+      "rewards/rejected": -6.03125,
+      "step": 3960
+    },
+    {
+      "epoch": 0.14777316632856266,
+      "grad_norm": 5.745987526680022,
+      "learning_rate": 4.965340979659699e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -672.0,
+      "logps/rejected": -776.0,
+      "loss": 0.4513,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.96875,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -6.0625,
+      "step": 3970
+    },
+    {
+      "epoch": 0.14814539092888648,
+      "grad_norm": 6.814180391539611,
+      "learning_rate": 4.964799869588673e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -632.0,
+      "logps/rejected": -756.0,
+      "loss": 0.4721,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.53125,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -5.8125,
+      "step": 3980
+    },
+    {
+      "epoch": 0.14851761552921033,
+      "grad_norm": 5.798104354113069,
+      "learning_rate": 4.964254598106751e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -640.0,
+      "logps/rejected": -752.0,
+      "loss": 0.4414,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -4.375,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -5.6875,
+      "step": 3990
+    },
+    {
+      "epoch": 0.14888984012953416,
+      "grad_norm": 6.114913289601737,
+      "learning_rate": 4.96370516613453e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -684.0,
+      "logps/rejected": -780.0,
+      "loss": 0.4562,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.9375,
+      "rewards/margins": 1.0,
+      "rewards/rejected": -5.9375,
+      "step": 4000
+    },
+    {
+      "epoch": 0.149262064729858,
+      "grad_norm": 5.01517407197322,
+      "learning_rate": 4.963151574599641e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -632.0,
+      "logps/rejected": -760.0,
+      "loss": 0.4557,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -4.625,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -5.96875,
+      "step": 4010
+    },
+    {
+      "epoch": 0.14963428933018183,
+      "grad_norm": 6.104422620962016,
+      "learning_rate": 4.962593824436731e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -644.0,
+      "logps/rejected": -772.0,
+      "loss": 0.432,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -4.59375,
+      "rewards/margins": 1.234375,
+      "rewards/rejected": -5.8125,
+      "step": 4020
+    },
+    {
+      "epoch": 0.15000651393050568,
+      "grad_norm": 5.493524217382667,
+      "learning_rate": 4.962031916587469e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -648.0,
+      "logps/rejected": -768.0,
+      "loss": 0.437,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -4.84375,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -6.1875,
+      "step": 4030
+    },
+    {
+      "epoch": 0.1503787385308295,
+      "grad_norm": 6.037008911338963,
+      "learning_rate": 4.961465852000545e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -680.0,
+      "logps/rejected": -792.0,
+      "loss": 0.4524,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.90625,
+      "rewards/margins": 1.234375,
+      "rewards/rejected": -6.125,
+      "step": 4040
+    },
+    {
+      "epoch": 0.15075096313115333,
+      "grad_norm": 6.206263363310757,
+      "learning_rate": 4.960895631631665e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.96875,
+      "logps/chosen": -700.0,
+      "logps/rejected": -812.0,
+      "loss": 0.4451,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -5.0,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -6.53125,
+      "step": 4050
+    },
+    {
+      "epoch": 0.15112318773147718,
+      "grad_norm": 6.228569331632899,
+      "learning_rate": 4.960321256443555e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -720.0,
+      "logps/rejected": -820.0,
+      "loss": 0.4434,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.96875,
+      "rewards/margins": 1.3671875,
+      "rewards/rejected": -6.3125,
+      "step": 4060
+    },
+    {
+      "epoch": 0.151495412331801,
+      "grad_norm": 5.122640947025669,
+      "learning_rate": 4.959742727405952e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -660.0,
+      "logps/rejected": -760.0,
+      "loss": 0.4226,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.65625,
+      "rewards/margins": 1.21875,
+      "rewards/rejected": -5.875,
+      "step": 4070
+    },
+    {
+      "epoch": 0.15186763693212485,
+      "grad_norm": 6.524389572637526,
+      "learning_rate": 4.959160045495607e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -616.0,
+      "logps/rejected": -736.0,
+      "loss": 0.4599,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -4.28125,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -5.5625,
+      "step": 4080
+    },
+    {
+      "epoch": 0.15223986153244867,
+      "grad_norm": 6.900189241327749,
+      "learning_rate": 4.958573211696285e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -668.0,
+      "logps/rejected": -788.0,
+      "loss": 0.4572,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.84375,
+      "rewards/margins": 1.2109375,
+      "rewards/rejected": -6.0625,
+      "step": 4090
+    },
+    {
+      "epoch": 0.15261208613277252,
+      "grad_norm": 6.16014597908324,
+      "learning_rate": 4.957982226998757e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -640.0,
+      "logps/rejected": -776.0,
+      "loss": 0.4537,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -4.6875,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -6.0,
+      "step": 4100
+    },
+    {
+      "epoch": 0.15298431073309635,
+      "grad_norm": 5.7726572436368055,
+      "learning_rate": 4.957387092400805e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -664.0,
+      "logps/rejected": -768.0,
+      "loss": 0.4201,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.8125,
+      "rewards/margins": 1.2265625,
+      "rewards/rejected": -6.03125,
+      "step": 4110
+    },
+    {
+      "epoch": 0.1533565353334202,
+      "grad_norm": 6.7406768979167575,
+      "learning_rate": 4.956787808907215e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -700.0,
+      "logps/rejected": -792.0,
+      "loss": 0.449,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -5.15625,
+      "rewards/margins": 1.046875,
+      "rewards/rejected": -6.21875,
+      "step": 4120
+    },
+    {
+      "epoch": 0.15372875993374402,
+      "grad_norm": 6.581215141816501,
+      "learning_rate": 4.95618437752978e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -708.0,
+      "logps/rejected": -836.0,
+      "loss": 0.4501,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -5.0625,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -6.5625,
+      "step": 4130
+    },
+    {
+      "epoch": 0.15410098453406787,
+      "grad_norm": 5.40509165991241,
+      "learning_rate": 4.955576799287296e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -664.0,
+      "logps/rejected": -768.0,
+      "loss": 0.4507,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -4.71875,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -6.0,
+      "step": 4140
+    },
+    {
+      "epoch": 0.1544732091343917,
+      "grad_norm": 5.800105089284638,
+      "learning_rate": 4.954965075205556e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -648.0,
+      "logps/rejected": -724.0,
+      "loss": 0.4528,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.6875,
+      "rewards/margins": 1.015625,
+      "rewards/rejected": -5.71875,
+      "step": 4150
+    },
+    {
+      "epoch": 0.15484543373471552,
+      "grad_norm": 5.700824199770932,
+      "learning_rate": 4.954349206317359e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -660.0,
+      "logps/rejected": -776.0,
+      "loss": 0.4487,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.8125,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -6.0625,
+      "step": 4160
+    },
+    {
+      "epoch": 0.15521765833503937,
+      "grad_norm": 5.951511567106983,
+      "learning_rate": 4.953729193662498e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -688.0,
+      "logps/rejected": -796.0,
+      "loss": 0.4371,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -5.09375,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -6.34375,
+      "step": 4170
+    },
+    {
+      "epoch": 0.1555898829353632,
+      "grad_norm": 5.924499558787697,
+      "learning_rate": 4.953105038287762e-07,
+      "logits/chosen": -4.125,
+      "logits/rejected": -3.96875,
+      "logps/chosen": -680.0,
+      "logps/rejected": -776.0,
+      "loss": 0.4463,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.8125,
+      "rewards/margins": 1.1796875,
+      "rewards/rejected": -6.0,
+      "step": 4180
+    },
+    {
+      "epoch": 0.15596210753568704,
+      "grad_norm": 4.908287229377769,
+      "learning_rate": 4.952476741246937e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -652.0,
+      "logps/rejected": -776.0,
+      "loss": 0.4318,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.84375,
+      "rewards/margins": 1.2421875,
+      "rewards/rejected": -6.0625,
+      "step": 4190
+    },
+    {
+      "epoch": 0.15633433213601086,
+      "grad_norm": 7.085053760895597,
+      "learning_rate": 4.951844303600798e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -692.0,
+      "logps/rejected": -804.0,
+      "loss": 0.4369,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -5.03125,
+      "rewards/margins": 1.234375,
+      "rewards/rejected": -6.25,
+      "step": 4200
+    },
+    {
+      "epoch": 0.1567065567363347,
+      "grad_norm": 7.299734039249005,
+      "learning_rate": 4.951207726417113e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -688.0,
+      "logps/rejected": -784.0,
+      "loss": 0.4433,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -5.03125,
+      "rewards/margins": 1.203125,
+      "rewards/rejected": -6.25,
+      "step": 4210
+    },
+    {
+      "epoch": 0.15707878133665854,
+      "grad_norm": 4.865575958043528,
+      "learning_rate": 4.950567010770638e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -652.0,
+      "logps/rejected": -760.0,
+      "loss": 0.4335,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -4.78125,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -6.0625,
+      "step": 4220
+    },
+    {
+      "epoch": 0.15745100593698239,
+      "grad_norm": 6.660088178189539,
+      "learning_rate": 4.949922157743116e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -696.0,
+      "logps/rejected": -804.0,
+      "loss": 0.4695,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -5.0625,
+      "rewards/margins": 1.0703125,
+      "rewards/rejected": -6.125,
+      "step": 4230
+    },
+    {
+      "epoch": 0.1578232305373062,
+      "grad_norm": 6.516346852986926,
+      "learning_rate": 4.949273168423277e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -740.0,
+      "logps/rejected": -824.0,
+      "loss": 0.4287,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -5.375,
+      "rewards/margins": 1.1328125,
+      "rewards/rejected": -6.5,
+      "step": 4240
+    },
+    {
+      "epoch": 0.15819545513763006,
+      "grad_norm": 5.779397019489463,
+      "learning_rate": 4.948620043906832e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -696.0,
+      "logps/rejected": -816.0,
+      "loss": 0.4355,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -5.03125,
+      "rewards/margins": 1.359375,
+      "rewards/rejected": -6.40625,
+      "step": 4250
+    },
+    {
+      "epoch": 0.15856767973795388,
+      "grad_norm": 5.6412601693042355,
+      "learning_rate": 4.947962785296475e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -724.0,
+      "logps/rejected": -852.0,
+      "loss": 0.4162,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.28125,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -6.75,
+      "step": 4260
+    },
+    {
+      "epoch": 0.1589399043382777,
+      "grad_norm": 5.212627894939003,
+      "learning_rate": 4.947301393701881e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -720.0,
+      "logps/rejected": -816.0,
+      "loss": 0.407,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -5.21875,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -6.5625,
+      "step": 4270
+    },
+    {
+      "epoch": 0.15931212893860155,
+      "grad_norm": 7.839232390844439,
+      "learning_rate": 4.946635870239699e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -744.0,
+      "logps/rejected": -864.0,
+      "loss": 0.4329,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -5.625,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -6.96875,
+      "step": 4280
+    },
+    {
+      "epoch": 0.15968435353892538,
+      "grad_norm": 5.305950553011658,
+      "learning_rate": 4.945966216033558e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -752.0,
+      "logps/rejected": -840.0,
+      "loss": 0.4365,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -5.46875,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -6.71875,
+      "step": 4290
+    },
+    {
+      "epoch": 0.16005657813924923,
+      "grad_norm": 6.092310208302023,
+      "learning_rate": 4.945292432214059e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -724.0,
+      "logps/rejected": -840.0,
+      "loss": 0.4135,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -5.5,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -6.8125,
+      "step": 4300
+    },
+    {
+      "epoch": 0.16042880273957305,
+      "grad_norm": 7.347003795038706,
+      "learning_rate": 4.944614519918775e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -728.0,
+      "logps/rejected": -872.0,
+      "loss": 0.4589,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -5.65625,
+      "rewards/margins": 1.375,
+      "rewards/rejected": -7.03125,
+      "step": 4310
+    },
+    {
+      "epoch": 0.1608010273398969,
+      "grad_norm": 5.525125255334156,
+      "learning_rate": 4.943932480292251e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -752.0,
+      "logps/rejected": -852.0,
+      "loss": 0.427,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -5.59375,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -6.875,
+      "step": 4320
+    },
+    {
+      "epoch": 0.16117325194022072,
+      "grad_norm": 7.152439469296344,
+      "learning_rate": 4.943246314485999e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -740.0,
+      "logps/rejected": -824.0,
+      "loss": 0.4411,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -5.34375,
+      "rewards/margins": 1.171875,
+      "rewards/rejected": -6.5,
+      "step": 4330
+    },
+    {
+      "epoch": 0.16154547654054457,
+      "grad_norm": 5.96617445849906,
+      "learning_rate": 4.942556023658497e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -724.0,
+      "logps/rejected": -804.0,
+      "loss": 0.4236,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -5.28125,
+      "rewards/margins": 1.2421875,
+      "rewards/rejected": -6.5,
+      "step": 4340
+    },
+    {
+      "epoch": 0.1619177011408684,
+      "grad_norm": 6.880916875781202,
+      "learning_rate": 4.941861608975188e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -728.0,
+      "logps/rejected": -880.0,
+      "loss": 0.4373,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -5.375,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -7.03125,
+      "step": 4350
+    },
+    {
+      "epoch": 0.16228992574119225,
+      "grad_norm": 5.3375870206157074,
+      "learning_rate": 4.941163071608479e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -716.0,
+      "logps/rejected": -848.0,
+      "loss": 0.4419,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -5.34375,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -6.9375,
+      "step": 4360
+    },
+    {
+      "epoch": 0.16266215034151607,
+      "grad_norm": 5.835199137212704,
+      "learning_rate": 4.940460412737733e-07,
+      "logits/chosen": -4.1875,
+      "logits/rejected": -3.9375,
+      "logps/chosen": -712.0,
+      "logps/rejected": -844.0,
+      "loss": 0.4285,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -5.34375,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -6.78125,
+      "step": 4370
+    },
+    {
+      "epoch": 0.1630343749418399,
+      "grad_norm": 5.430840526085023,
+      "learning_rate": 4.939753633549277e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -720.0,
+      "logps/rejected": -840.0,
+      "loss": 0.4278,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.59375,
+      "rewards/margins": 1.1484375,
+      "rewards/rejected": -6.75,
+      "step": 4380
+    },
+    {
+      "epoch": 0.16340659954216374,
+      "grad_norm": 6.420210453405993,
+      "learning_rate": 4.93904273523639e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -748.0,
+      "logps/rejected": -884.0,
+      "loss": 0.4468,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -5.6875,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": -7.03125,
+      "step": 4390
+    },
+    {
+      "epoch": 0.16377882414248757,
+      "grad_norm": 4.821559561645362,
+      "learning_rate": 4.93832771899931e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -716.0,
+      "logps/rejected": -800.0,
+      "loss": 0.438,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -5.34375,
+      "rewards/margins": 1.1640625,
+      "rewards/rejected": -6.5,
+      "step": 4400
+    },
+    {
+      "epoch": 0.16415104874281142,
+      "grad_norm": 5.9986907160594365,
+      "learning_rate": 4.937608586045223e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -740.0,
+      "logps/rejected": -848.0,
+      "loss": 0.4329,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -5.6875,
+      "rewards/margins": 1.2265625,
+      "rewards/rejected": -6.90625,
+      "step": 4410
+    },
+    {
+      "epoch": 0.16452327334313524,
+      "grad_norm": 5.135010039615432,
+      "learning_rate": 4.936885337588266e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -744.0,
+      "logps/rejected": -852.0,
+      "loss": 0.4462,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -5.5,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -6.75,
+      "step": 4420
+    },
+    {
+      "epoch": 0.1648954979434591,
+      "grad_norm": 4.956690386657707,
+      "learning_rate": 4.936157974849528e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -736.0,
+      "logps/rejected": -864.0,
+      "loss": 0.4538,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -5.59375,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -7.15625,
+      "step": 4430
+    },
+    {
+      "epoch": 0.1652677225437829,
+      "grad_norm": 5.3185852565852745,
+      "learning_rate": 4.93542649905704e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -732.0,
+      "logps/rejected": -848.0,
+      "loss": 0.4309,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.59375,
+      "rewards/margins": 1.265625,
+      "rewards/rejected": -6.84375,
+      "step": 4440
+    },
+    {
+      "epoch": 0.16563994714410676,
+      "grad_norm": 5.459650406863102,
+      "learning_rate": 4.934690911445782e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -708.0,
+      "logps/rejected": -836.0,
+      "loss": 0.4136,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -5.3125,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -6.59375,
+      "step": 4450
+    },
+    {
+      "epoch": 0.1660121717444306,
+      "grad_norm": 6.218550154302611,
+      "learning_rate": 4.933951213257669e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -772.0,
+      "logps/rejected": -892.0,
+      "loss": 0.455,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -5.75,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -7.0,
+      "step": 4460
+    },
+    {
+      "epoch": 0.16638439634475444,
+      "grad_norm": 5.821381057255457,
+      "learning_rate": 4.933207405741563e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.921875,
+      "logps/chosen": -736.0,
+      "logps/rejected": -832.0,
+      "loss": 0.4362,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -5.40625,
+      "rewards/margins": 1.1484375,
+      "rewards/rejected": -6.5625,
+      "step": 4470
+    },
+    {
+      "epoch": 0.16675662094507826,
+      "grad_norm": 5.978481687335648,
+      "learning_rate": 4.93245949015326e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -712.0,
+      "logps/rejected": -828.0,
+      "loss": 0.4439,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -5.21875,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -6.625,
+      "step": 4480
+    },
+    {
+      "epoch": 0.16712884554540208,
+      "grad_norm": 5.734106544288039,
+      "learning_rate": 4.931707467755495e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -748.0,
+      "logps/rejected": -880.0,
+      "loss": 0.443,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -5.46875,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -6.96875,
+      "step": 4490
+    },
+    {
+      "epoch": 0.16750107014572593,
+      "grad_norm": 5.006028842221933,
+      "learning_rate": 4.930951339817932e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -728.0,
+      "logps/rejected": -844.0,
+      "loss": 0.4441,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -5.59375,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -7.0,
+      "step": 4500
+    },
+    {
+      "epoch": 0.16787329474604976,
+      "grad_norm": 6.372329177266804,
+      "learning_rate": 4.93019110761717e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -696.0,
+      "logps/rejected": -824.0,
+      "loss": 0.4451,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.21875,
+      "rewards/margins": 1.3671875,
+      "rewards/rejected": -6.5625,
+      "step": 4510
+    },
+    {
+      "epoch": 0.1682455193463736,
+      "grad_norm": 5.321077186116423,
+      "learning_rate": 4.929426772436738e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -704.0,
+      "logps/rejected": -816.0,
+      "loss": 0.4601,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -5.3125,
+      "rewards/margins": 1.1953125,
+      "rewards/rejected": -6.5,
+      "step": 4520
+    },
+    {
+      "epoch": 0.16861774394669743,
+      "grad_norm": 5.780086491592659,
+      "learning_rate": 4.928658335567088e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -736.0,
+      "logps/rejected": -844.0,
+      "loss": 0.4323,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -5.4375,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -6.71875,
+      "step": 4530
+    },
+    {
+      "epoch": 0.16898996854702128,
+      "grad_norm": 5.583812927129396,
+      "learning_rate": 4.927885798305603e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -736.0,
+      "logps/rejected": -872.0,
+      "loss": 0.4313,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -5.4375,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -7.0625,
+      "step": 4540
+    },
+    {
+      "epoch": 0.1693621931473451,
+      "grad_norm": 5.476057964945678,
+      "learning_rate": 4.927109161956584e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -752.0,
+      "logps/rejected": -840.0,
+      "loss": 0.4495,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -5.6875,
+      "rewards/margins": 1.0078125,
+      "rewards/rejected": -6.71875,
+      "step": 4550
+    },
+    {
+      "epoch": 0.16973441774766895,
+      "grad_norm": 7.033847374914891,
+      "learning_rate": 4.926328427831254e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -732.0,
+      "logps/rejected": -840.0,
+      "loss": 0.4375,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -5.4375,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -6.75,
+      "step": 4560
+    },
+    {
+      "epoch": 0.17010664234799278,
+      "grad_norm": 5.1607785323824595,
+      "learning_rate": 4.925543597247756e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -724.0,
+      "logps/rejected": -888.0,
+      "loss": 0.4275,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -5.4375,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -7.15625,
+      "step": 4570
+    },
+    {
+      "epoch": 0.17047886694831663,
+      "grad_norm": 5.618927098882581,
+      "learning_rate": 4.924754671531145e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -728.0,
+      "logps/rejected": -856.0,
+      "loss": 0.401,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -5.5,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -6.9375,
+      "step": 4580
+    },
+    {
+      "epoch": 0.17085109154864045,
+      "grad_norm": 5.830772890973959,
+      "learning_rate": 4.923961652013396e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -744.0,
+      "logps/rejected": -852.0,
+      "loss": 0.4344,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -5.5,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -6.78125,
+      "step": 4590
+    },
+    {
+      "epoch": 0.17122331614896427,
+      "grad_norm": 6.239813343089435,
+      "learning_rate": 4.923164540033392e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -660.0,
+      "logps/rejected": -748.0,
+      "loss": 0.4385,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.90625,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -6.03125,
+      "step": 4600
+    },
+    {
+      "epoch": 0.17159554074928812,
+      "grad_norm": 4.968248889246611,
+      "learning_rate": 4.922363336936926e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -712.0,
+      "logps/rejected": -816.0,
+      "loss": 0.4246,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.21875,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": -6.5625,
+      "step": 4610
+    },
+    {
+      "epoch": 0.17196776534961195,
+      "grad_norm": 4.957565888521443,
+      "learning_rate": 4.921558044076696e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -692.0,
+      "logps/rejected": -824.0,
+      "loss": 0.4355,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -5.15625,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -6.59375,
+      "step": 4620
+    },
+    {
+      "epoch": 0.1723399899499358,
+      "grad_norm": 5.609337891090924,
+      "learning_rate": 4.920748662812309e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -756.0,
+      "logps/rejected": -848.0,
+      "loss": 0.4603,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -5.5625,
+      "rewards/margins": 1.0859375,
+      "rewards/rejected": -6.65625,
+      "step": 4630
+    },
+    {
+      "epoch": 0.17271221455025962,
+      "grad_norm": 5.668406015308072,
+      "learning_rate": 4.919935194510274e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -736.0,
+      "logps/rejected": -844.0,
+      "loss": 0.437,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -5.46875,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -6.8125,
+      "step": 4640
+    },
+    {
+      "epoch": 0.17308443915058347,
+      "grad_norm": 5.033927610969634,
+      "learning_rate": 4.919117640543996e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -748.0,
+      "logps/rejected": -848.0,
+      "loss": 0.441,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -5.46875,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -6.75,
+      "step": 4650
+    },
+    {
+      "epoch": 0.1734566637509073,
+      "grad_norm": 5.789939967450309,
+      "learning_rate": 4.918296002293782e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -756.0,
+      "logps/rejected": -836.0,
+      "loss": 0.4407,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.625,
+      "rewards/margins": 0.9296875,
+      "rewards/rejected": -6.5625,
+      "step": 4660
+    },
+    {
+      "epoch": 0.17382888835123114,
+      "grad_norm": 5.828642956821937,
+      "learning_rate": 4.917470281146836e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -728.0,
+      "logps/rejected": -864.0,
+      "loss": 0.4362,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -5.625,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -7.09375,
+      "step": 4670
+    },
+    {
+      "epoch": 0.17420111295155497,
+      "grad_norm": 6.765149293815343,
+      "learning_rate": 4.916640478497249e-07,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -740.0,
+      "logps/rejected": -856.0,
+      "loss": 0.4191,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -5.4375,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -6.84375,
+      "step": 4680
+    },
+    {
+      "epoch": 0.17457333755187882,
+      "grad_norm": 6.558713025037335,
+      "learning_rate": 4.91580659574601e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -772.0,
+      "logps/rejected": -880.0,
+      "loss": 0.4419,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -5.8125,
+      "rewards/margins": 1.28125,
+      "rewards/rejected": -7.09375,
+      "step": 4690
+    },
+    {
+      "epoch": 0.17494556215220264,
+      "grad_norm": 6.356682875651491,
+      "learning_rate": 4.914968634300993e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -732.0,
+      "logps/rejected": -900.0,
+      "loss": 0.4479,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.59375,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -7.25,
+      "step": 4700
+    },
+    {
+      "epoch": 0.17531778675252646,
+      "grad_norm": 6.113981869788465,
+      "learning_rate": 4.914126595576957e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -740.0,
+      "logps/rejected": -844.0,
+      "loss": 0.4237,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -5.375,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -6.6875,
+      "step": 4710
+    },
+    {
+      "epoch": 0.1756900113528503,
+      "grad_norm": 6.024599076238062,
+      "learning_rate": 4.913280480995547e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -752.0,
+      "logps/rejected": -864.0,
+      "loss": 0.454,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -5.53125,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -6.78125,
+      "step": 4720
+    },
+    {
+      "epoch": 0.17606223595317413,
+      "grad_norm": 8.184884356586648,
+      "learning_rate": 4.912430291985291e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -732.0,
+      "logps/rejected": -844.0,
+      "loss": 0.4376,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -5.5,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -6.90625,
+      "step": 4730
+    },
+    {
+      "epoch": 0.17643446055349798,
+      "grad_norm": 6.813540017641614,
+      "learning_rate": 4.911576029981591e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.5,
+      "logps/chosen": -756.0,
+      "logps/rejected": -904.0,
+      "loss": 0.4263,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -5.8125,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -7.25,
+      "step": 4740
+    },
+    {
+      "epoch": 0.1768066851538218,
+      "grad_norm": 6.563348472736759,
+      "learning_rate": 4.91071769642673e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -736.0,
+      "logps/rejected": -864.0,
+      "loss": 0.4304,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -5.4375,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -6.84375,
+      "step": 4750
+    },
+    {
+      "epoch": 0.17717890975414566,
+      "grad_norm": 5.825920486652797,
+      "learning_rate": 4.909855292769863e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -704.0,
+      "logps/rejected": -848.0,
+      "loss": 0.4513,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -5.125,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -6.71875,
+      "step": 4760
+    },
+    {
+      "epoch": 0.17755113435446948,
+      "grad_norm": 7.050683574404367,
+      "learning_rate": 4.908988820467018e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -728.0,
+      "logps/rejected": -832.0,
+      "loss": 0.4345,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -5.375,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -6.71875,
+      "step": 4770
+    },
+    {
+      "epoch": 0.17792335895479333,
+      "grad_norm": 5.943063447979361,
+      "learning_rate": 4.908118280981091e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -728.0,
+      "logps/rejected": -848.0,
+      "loss": 0.4401,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -5.40625,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -6.8125,
+      "step": 4780
+    },
+    {
+      "epoch": 0.17829558355511715,
+      "grad_norm": 5.5061494436906235,
+      "learning_rate": 4.907243675781847e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -728.0,
+      "logps/rejected": -840.0,
+      "loss": 0.4248,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -5.5,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -6.84375,
+      "step": 4790
+    },
+    {
+      "epoch": 0.178667808155441,
+      "grad_norm": 6.4331453916200685,
+      "learning_rate": 4.90636500634591e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -772.0,
+      "logps/rejected": -884.0,
+      "loss": 0.4209,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -5.84375,
+      "rewards/margins": 1.1953125,
+      "rewards/rejected": -7.03125,
+      "step": 4800
+    },
+    {
+      "epoch": 0.17904003275576483,
+      "grad_norm": 6.851308791165571,
+      "learning_rate": 4.905482274156773e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -756.0,
+      "logps/rejected": -892.0,
+      "loss": 0.4402,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -5.71875,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -7.125,
+      "step": 4810
+    },
+    {
+      "epoch": 0.17941225735608865,
+      "grad_norm": 6.118848900478091,
+      "learning_rate": 4.904595480704784e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -716.0,
+      "logps/rejected": -856.0,
+      "loss": 0.453,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -5.5,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -6.84375,
+      "step": 4820
+    },
+    {
+      "epoch": 0.1797844819564125,
+      "grad_norm": 6.02105358245143,
+      "learning_rate": 4.903704627487148e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -732.0,
+      "logps/rejected": -832.0,
+      "loss": 0.4205,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -5.53125,
+      "rewards/margins": 1.0625,
+      "rewards/rejected": -6.59375,
+      "step": 4830
+    },
+    {
+      "epoch": 0.18015670655673632,
+      "grad_norm": 5.612262348422165,
+      "learning_rate": 4.902809716007923e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -740.0,
+      "logps/rejected": -856.0,
+      "loss": 0.4487,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -5.71875,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -7.0625,
+      "step": 4840
+    },
+    {
+      "epoch": 0.18052893115706017,
+      "grad_norm": 6.268376969175533,
+      "learning_rate": 4.901910747778023e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -724.0,
+      "logps/rejected": -836.0,
+      "loss": 0.4451,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -5.4375,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -6.6875,
+      "step": 4850
+    },
+    {
+      "epoch": 0.180901155757384,
+      "grad_norm": 6.493458128988398,
+      "learning_rate": 4.901007724315209e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -732.0,
+      "logps/rejected": -856.0,
+      "loss": 0.4204,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -5.3125,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -6.6875,
+      "step": 4860
+    },
+    {
+      "epoch": 0.18127338035770785,
+      "grad_norm": 5.5586289014407555,
+      "learning_rate": 4.900100647144085e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -764.0,
+      "logps/rejected": -868.0,
+      "loss": 0.4445,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -5.65625,
+      "rewards/margins": 1.234375,
+      "rewards/rejected": -6.875,
+      "step": 4870
+    },
+    {
+      "epoch": 0.18164560495803167,
+      "grad_norm": 6.08621931637,
+      "learning_rate": 4.899189517796105e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -740.0,
+      "logps/rejected": -872.0,
+      "loss": 0.4362,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -5.6875,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -7.125,
+      "step": 4880
+    },
+    {
+      "epoch": 0.18201782955835552,
+      "grad_norm": 5.543220496129898,
+      "learning_rate": 4.898274337809562e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -764.0,
+      "logps/rejected": -916.0,
+      "loss": 0.4234,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -5.6875,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -7.34375,
+      "step": 4890
+    },
+    {
+      "epoch": 0.18239005415867934,
+      "grad_norm": 6.093505029561755,
+      "learning_rate": 4.897355108729585e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -764.0,
+      "logps/rejected": -892.0,
+      "loss": 0.4264,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -5.8125,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -7.21875,
+      "step": 4900
+    },
+    {
+      "epoch": 0.1827622787590032,
+      "grad_norm": 8.016223200841504,
+      "learning_rate": 4.896431832108143e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.625,
+      "logps/chosen": -764.0,
+      "logps/rejected": -888.0,
+      "loss": 0.4505,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -5.90625,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -7.25,
+      "step": 4910
+    },
+    {
+      "epoch": 0.18313450335932702,
+      "grad_norm": 5.795693174310396,
+      "learning_rate": 4.895504509504038e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -784.0,
+      "logps/rejected": -872.0,
+      "loss": 0.4154,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -5.8125,
+      "rewards/margins": 1.1953125,
+      "rewards/rejected": -7.03125,
+      "step": 4920
+    },
+    {
+      "epoch": 0.18350672795965087,
+      "grad_norm": 5.095859257672974,
+      "learning_rate": 4.894573142482902e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -744.0,
+      "logps/rejected": -880.0,
+      "loss": 0.4319,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -5.59375,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -7.03125,
+      "step": 4930
+    },
+    {
+      "epoch": 0.1838789525599747,
+      "grad_norm": 5.635877599621557,
+      "learning_rate": 4.893637732617196e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -740.0,
+      "logps/rejected": -856.0,
+      "loss": 0.4294,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -5.5,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -6.75,
+      "step": 4940
+    },
+    {
+      "epoch": 0.1842511771602985,
+      "grad_norm": 5.488866266104458,
+      "learning_rate": 4.892698281486206e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -792.0,
+      "logps/rejected": -916.0,
+      "loss": 0.4184,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -5.9375,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -7.4375,
+      "step": 4950
+    },
+    {
+      "epoch": 0.18462340176062236,
+      "grad_norm": 5.6824898097068575,
+      "learning_rate": 4.89175479067604e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -744.0,
+      "logps/rejected": -884.0,
+      "loss": 0.4338,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -5.6875,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -7.15625,
+      "step": 4960
+    },
+    {
+      "epoch": 0.18499562636094619,
+      "grad_norm": 6.015013465582994,
+      "learning_rate": 4.890807261779631e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -748.0,
+      "logps/rejected": -880.0,
+      "loss": 0.4398,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -5.625,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -7.03125,
+      "step": 4970
+    },
+    {
+      "epoch": 0.18536785096127004,
+      "grad_norm": 7.496988606774147,
+      "learning_rate": 4.889855696396722e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -724.0,
+      "logps/rejected": -844.0,
+      "loss": 0.4411,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -5.53125,
+      "rewards/margins": 1.15625,
+      "rewards/rejected": -6.6875,
+      "step": 4980
+    },
+    {
+      "epoch": 0.18574007556159386,
+      "grad_norm": 7.139245875854106,
+      "learning_rate": 4.88890009613388e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -736.0,
+      "logps/rejected": -856.0,
+      "loss": 0.4222,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -5.625,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": -6.96875,
+      "step": 4990
+    },
+    {
+      "epoch": 0.1861123001619177,
+      "grad_norm": 5.895380777165287,
+      "learning_rate": 4.887940462604475e-07,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.375,
+      "logps/chosen": -752.0,
+      "logps/rejected": -896.0,
+      "loss": 0.4428,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -5.59375,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -7.15625,
+      "step": 5000
+    },
+    {
+      "epoch": 0.1861123001619177,
+      "eval_logits/chosen": -3.8125,
+      "eval_logits/rejected": -3.625,
+      "eval_logps/chosen": -752.0,
+      "eval_logps/rejected": -844.0,
+      "eval_loss": 0.4641050100326538,
+      "eval_rewards/accuracies": 0.7533489465713501,
+      "eval_rewards/chosen": -5.53125,
+      "eval_rewards/margins": 1.203125,
+      "eval_rewards/rejected": -6.75,
+      "eval_runtime": 6274.7166,
+      "eval_samples_per_second": 30.449,
+      "eval_steps_per_second": 0.476,
+      "step": 5000
+    },
+    {
+      "epoch": 0.18648452476224153,
+      "grad_norm": 5.725945474561893,
+      "learning_rate": 4.886976797428694e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -740.0,
+      "logps/rejected": -844.0,
+      "loss": 0.4427,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -5.5,
+      "rewards/margins": 1.2109375,
+      "rewards/rejected": -6.71875,
+      "step": 5010
+    },
+    {
+      "epoch": 0.18685674936256538,
+      "grad_norm": 6.832914843918203,
+      "learning_rate": 4.886009102233528e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -724.0,
+      "logps/rejected": -844.0,
+      "loss": 0.445,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -5.28125,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -6.78125,
+      "step": 5020
+    },
+    {
+      "epoch": 0.1872289739628892,
+      "grad_norm": 6.7518725830357935,
+      "learning_rate": 4.88503737865277e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -728.0,
+      "logps/rejected": -856.0,
+      "loss": 0.4369,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -5.5625,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -6.90625,
+      "step": 5030
+    },
+    {
+      "epoch": 0.18760119856321306,
+      "grad_norm": 5.582802264723439,
+      "learning_rate": 4.884061628327018e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -752.0,
+      "logps/rejected": -872.0,
+      "loss": 0.4411,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -5.71875,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -7.03125,
+      "step": 5040
+    },
+    {
+      "epoch": 0.18797342316353688,
+      "grad_norm": 6.326476485002667,
+      "learning_rate": 4.883081852903665e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -728.0,
+      "logps/rejected": -860.0,
+      "loss": 0.399,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -5.40625,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -6.875,
+      "step": 5050
+    },
+    {
+      "epoch": 0.1883456477638607,
+      "grad_norm": 5.907025640438917,
+      "learning_rate": 4.882098054036901e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -760.0,
+      "logps/rejected": -872.0,
+      "loss": 0.4461,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -5.75,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -7.125,
+      "step": 5060
+    },
+    {
+      "epoch": 0.18871787236418455,
+      "grad_norm": 5.292269064459508,
+      "learning_rate": 4.881110233387711e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -740.0,
+      "logps/rejected": -876.0,
+      "loss": 0.4024,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -5.5,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -7.0,
+      "step": 5070
+    },
+    {
+      "epoch": 0.18909009696450838,
+      "grad_norm": 6.46023311091003,
+      "learning_rate": 4.880118392623869e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -716.0,
+      "logps/rejected": -852.0,
+      "loss": 0.4429,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -5.375,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -6.96875,
+      "step": 5080
+    },
+    {
+      "epoch": 0.18946232156483223,
+      "grad_norm": 6.0498403978099295,
+      "learning_rate": 4.879122533419933e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -748.0,
+      "logps/rejected": -872.0,
+      "loss": 0.4278,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -5.6875,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -7.1875,
+      "step": 5090
+    },
+    {
+      "epoch": 0.18983454616515605,
+      "grad_norm": 6.917627910061831,
+      "learning_rate": 4.87812265745725e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -736.0,
+      "logps/rejected": -832.0,
+      "loss": 0.4388,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.59375,
+      "rewards/margins": 1.2109375,
+      "rewards/rejected": -6.8125,
+      "step": 5100
+    },
+    {
+      "epoch": 0.1902067707654799,
+      "grad_norm": 5.854938819403657,
+      "learning_rate": 4.877118766423945e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -732.0,
+      "logps/rejected": -864.0,
+      "loss": 0.4207,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -5.5,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -6.78125,
+      "step": 5110
+    },
+    {
+      "epoch": 0.19057899536580372,
+      "grad_norm": 6.628461291408429,
+      "learning_rate": 4.876110862014926e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -776.0,
+      "logps/rejected": -908.0,
+      "loss": 0.4375,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -5.90625,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -7.34375,
+      "step": 5120
+    },
+    {
+      "epoch": 0.19095121996612757,
+      "grad_norm": 6.828581273814096,
+      "learning_rate": 4.875098945931873e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -740.0,
+      "logps/rejected": -876.0,
+      "loss": 0.4178,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -5.65625,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -7.25,
+      "step": 5130
+    },
+    {
+      "epoch": 0.1913234445664514,
+      "grad_norm": 7.889843254400278,
+      "learning_rate": 4.874083019883242e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -740.0,
+      "logps/rejected": -856.0,
+      "loss": 0.4515,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -5.78125,
+      "rewards/margins": 1.203125,
+      "rewards/rejected": -6.96875,
+      "step": 5140
+    },
+    {
+      "epoch": 0.19169566916677525,
+      "grad_norm": 6.252180105350043,
+      "learning_rate": 4.873063085584256e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -740.0,
+      "logps/rejected": -848.0,
+      "loss": 0.4389,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -5.6875,
+      "rewards/margins": 1.15625,
+      "rewards/rejected": -6.84375,
+      "step": 5150
+    },
+    {
+      "epoch": 0.19206789376709907,
+      "grad_norm": 7.349031142114769,
+      "learning_rate": 4.872039144756907e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -704.0,
+      "logps/rejected": -836.0,
+      "loss": 0.4165,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -5.28125,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -6.75,
+      "step": 5160
+    },
+    {
+      "epoch": 0.1924401183674229,
+      "grad_norm": 5.828708637714437,
+      "learning_rate": 4.871011199129953e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -744.0,
+      "logps/rejected": -872.0,
+      "loss": 0.4294,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.71875,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -7.125,
+      "step": 5170
+    },
+    {
+      "epoch": 0.19281234296774674,
+      "grad_norm": 8.44398046890037,
+      "learning_rate": 4.869979250438911e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -732.0,
+      "logps/rejected": -892.0,
+      "loss": 0.4269,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -5.6875,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -7.21875,
+      "step": 5180
+    },
+    {
+      "epoch": 0.19318456756807056,
+      "grad_norm": 6.091867051627648,
+      "learning_rate": 4.868943300426057e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -728.0,
+      "logps/rejected": -840.0,
+      "loss": 0.461,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -5.53125,
+      "rewards/margins": 1.2421875,
+      "rewards/rejected": -6.75,
+      "step": 5190
+    },
+    {
+      "epoch": 0.19355679216839441,
+      "grad_norm": 8.020364430687428,
+      "learning_rate": 4.867903350840423e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -724.0,
+      "logps/rejected": -804.0,
+      "loss": 0.437,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -5.40625,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -6.5,
+      "step": 5200
+    },
+    {
+      "epoch": 0.19392901676871824,
+      "grad_norm": 6.6012859196720886,
+      "learning_rate": 4.866859403437795e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -776.0,
+      "logps/rejected": -892.0,
+      "loss": 0.4028,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -5.75,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -7.125,
+      "step": 5210
+    },
+    {
+      "epoch": 0.1943012413690421,
+      "grad_norm": 6.373502263083278,
+      "learning_rate": 4.865811459980705e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -764.0,
+      "logps/rejected": -864.0,
+      "loss": 0.415,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -5.625,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -6.96875,
+      "step": 5220
+    },
+    {
+      "epoch": 0.1946734659693659,
+      "grad_norm": 6.487507137121604,
+      "learning_rate": 4.864759522238435e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -716.0,
+      "logps/rejected": -824.0,
+      "loss": 0.4563,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -5.46875,
+      "rewards/margins": 1.3125,
+      "rewards/rejected": -6.78125,
+      "step": 5230
+    },
+    {
+      "epoch": 0.19504569056968976,
+      "grad_norm": 4.992462986944008,
+      "learning_rate": 4.86370359198701e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -748.0,
+      "logps/rejected": -884.0,
+      "loss": 0.4317,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -5.46875,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -7.15625,
+      "step": 5240
+    },
+    {
+      "epoch": 0.19541791517001358,
+      "grad_norm": 5.488033190904126,
+      "learning_rate": 4.862643671009193e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -744.0,
+      "logps/rejected": -852.0,
+      "loss": 0.434,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -5.8125,
+      "rewards/margins": 1.1796875,
+      "rewards/rejected": -7.0,
+      "step": 5250
+    },
+    {
+      "epoch": 0.19579013977033743,
+      "grad_norm": 5.8995603064893665,
+      "learning_rate": 4.861579761094491e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -736.0,
+      "logps/rejected": -864.0,
+      "loss": 0.422,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -5.65625,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -7.15625,
+      "step": 5260
+    },
+    {
+      "epoch": 0.19616236437066126,
+      "grad_norm": 5.920617882957075,
+      "learning_rate": 4.860511864039139e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -740.0,
+      "logps/rejected": -880.0,
+      "loss": 0.4507,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -5.53125,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -7.03125,
+      "step": 5270
+    },
+    {
+      "epoch": 0.19653458897098508,
+      "grad_norm": 5.994071618489967,
+      "learning_rate": 4.859439981646106e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -676.0,
+      "logps/rejected": -804.0,
+      "loss": 0.4096,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -4.875,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -6.5,
+      "step": 5280
+    },
+    {
+      "epoch": 0.19690681357130893,
+      "grad_norm": 5.9493842806654165,
+      "learning_rate": 4.858364115725091e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -732.0,
+      "logps/rejected": -856.0,
+      "loss": 0.4245,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.5,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -6.8125,
+      "step": 5290
+    },
+    {
+      "epoch": 0.19727903817163275,
+      "grad_norm": 7.092812702959431,
+      "learning_rate": 4.857284268092516e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -732.0,
+      "logps/rejected": -864.0,
+      "loss": 0.437,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -5.75,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -7.15625,
+      "step": 5300
+    },
+    {
+      "epoch": 0.1976512627719566,
+      "grad_norm": 5.159932194970655,
+      "learning_rate": 4.856200440571529e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -712.0,
+      "logps/rejected": -856.0,
+      "loss": 0.4128,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -5.4375,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -7.125,
+      "step": 5310
+    },
+    {
+      "epoch": 0.19802348737228043,
+      "grad_norm": 7.17408143427172,
+      "learning_rate": 4.855112634991994e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -716.0,
+      "logps/rejected": -832.0,
+      "loss": 0.4257,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.375,
+      "rewards/margins": 1.28125,
+      "rewards/rejected": -6.65625,
+      "step": 5320
+    },
+    {
+      "epoch": 0.19839571197260428,
+      "grad_norm": 5.990694261618531,
+      "learning_rate": 4.854020853190492e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -756.0,
+      "logps/rejected": -892.0,
+      "loss": 0.434,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.6875,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -7.21875,
+      "step": 5330
+    },
+    {
+      "epoch": 0.1987679365729281,
+      "grad_norm": 7.813568158112514,
+      "learning_rate": 4.85292509701032e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -776.0,
+      "logps/rejected": -880.0,
+      "loss": 0.4224,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -5.875,
+      "rewards/margins": 1.203125,
+      "rewards/rejected": -7.0625,
+      "step": 5340
+    },
+    {
+      "epoch": 0.19914016117325195,
+      "grad_norm": 6.149892459263521,
+      "learning_rate": 4.85182536830148e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -720.0,
+      "logps/rejected": -876.0,
+      "loss": 0.417,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -5.53125,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -7.125,
+      "step": 5350
+    },
+    {
+      "epoch": 0.19951238577357577,
+      "grad_norm": 8.445225407753806,
+      "learning_rate": 4.850721668920684e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -700.0,
+      "logps/rejected": -816.0,
+      "loss": 0.4408,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -5.1875,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -6.625,
+      "step": 5360
+    },
+    {
+      "epoch": 0.19988461037389962,
+      "grad_norm": 5.7207478835359185,
+      "learning_rate": 4.84961400073135e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -700.0,
+      "logps/rejected": -812.0,
+      "loss": 0.423,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -5.25,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -6.5625,
+      "step": 5370
+    },
+    {
+      "epoch": 0.20025683497422345,
+      "grad_norm": 6.368419048288751,
+      "learning_rate": 4.848502365603593e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.984375,
+      "logps/chosen": -704.0,
+      "logps/rejected": -824.0,
+      "loss": 0.4271,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -5.1875,
+      "rewards/margins": 1.3125,
+      "rewards/rejected": -6.5,
+      "step": 5380
+    },
+    {
+      "epoch": 0.20062905957454727,
+      "grad_norm": 6.409322920707655,
+      "learning_rate": 4.847386765414227e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -764.0,
+      "logps/rejected": -852.0,
+      "loss": 0.4119,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -5.6875,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -7.0,
+      "step": 5390
+    },
+    {
+      "epoch": 0.20100128417487112,
+      "grad_norm": 6.9305270596595125,
+      "learning_rate": 4.84626720204676e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -784.0,
+      "logps/rejected": -896.0,
+      "loss": 0.4108,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -5.9375,
+      "rewards/margins": 1.234375,
+      "rewards/rejected": -7.1875,
+      "step": 5400
+    },
+    {
+      "epoch": 0.20137350877519494,
+      "grad_norm": 6.427921369036785,
+      "learning_rate": 4.845143677391392e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -760.0,
+      "logps/rejected": -892.0,
+      "loss": 0.4023,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -5.65625,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -7.09375,
+      "step": 5410
+    },
+    {
+      "epoch": 0.2017457333755188,
+      "grad_norm": 8.00449296114199,
+      "learning_rate": 4.84401619334501e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -768.0,
+      "logps/rejected": -880.0,
+      "loss": 0.4492,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -5.8125,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -7.0625,
+      "step": 5420
+    },
+    {
+      "epoch": 0.20211795797584262,
+      "grad_norm": 11.180049033802964,
+      "learning_rate": 4.842884751811185e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.875,
+      "logps/chosen": -760.0,
+      "logps/rejected": -892.0,
+      "loss": 0.4279,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -5.8125,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -7.3125,
+      "step": 5430
+    },
+    {
+      "epoch": 0.20249018257616647,
+      "grad_norm": 6.5389454437663765,
+      "learning_rate": 4.841749354700172e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -780.0,
+      "logps/rejected": -880.0,
+      "loss": 0.4376,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -5.75,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -7.09375,
+      "step": 5440
+    },
+    {
+      "epoch": 0.2028624071764903,
+      "grad_norm": 7.879662554611795,
+      "learning_rate": 4.840610003928902e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.953125,
+      "logps/chosen": -800.0,
+      "logps/rejected": -916.0,
+      "loss": 0.4231,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -5.96875,
+      "rewards/margins": 1.375,
+      "rewards/rejected": -7.34375,
+      "step": 5450
+    },
+    {
+      "epoch": 0.20323463177681414,
+      "grad_norm": 6.805461144988961,
+      "learning_rate": 4.839466701420985e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -804.0,
+      "logps/rejected": -912.0,
+      "loss": 0.4605,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.0,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -7.375,
+      "step": 5460
+    },
+    {
+      "epoch": 0.20360685637713796,
+      "grad_norm": 5.647063109039535,
+      "learning_rate": 4.838319449106697e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.875,
+      "logps/chosen": -736.0,
+      "logps/rejected": -844.0,
+      "loss": 0.4248,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -5.59375,
+      "rewards/margins": 1.203125,
+      "rewards/rejected": -6.78125,
+      "step": 5470
+    },
+    {
+      "epoch": 0.2039790809774618,
+      "grad_norm": 6.272824298573251,
+      "learning_rate": 4.837168248922986e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -796.0,
+      "logps/rejected": -908.0,
+      "loss": 0.4104,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -5.9375,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -7.25,
+      "step": 5480
+    },
+    {
+      "epoch": 0.20435130557778564,
+      "grad_norm": 6.782748839764093,
+      "learning_rate": 4.836013102813467e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -808.0,
+      "logps/rejected": -928.0,
+      "loss": 0.4153,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.125,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -7.65625,
+      "step": 5490
+    },
+    {
+      "epoch": 0.20472353017810946,
+      "grad_norm": 6.682845177389023,
+      "learning_rate": 4.834854012728412e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -780.0,
+      "logps/rejected": -888.0,
+      "loss": 0.4471,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -5.875,
+      "rewards/margins": 1.265625,
+      "rewards/rejected": -7.15625,
+      "step": 5500
+    },
+    {
+      "epoch": 0.2050957547784333,
+      "grad_norm": 7.451998046730023,
+      "learning_rate": 4.833690980624758e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -748.0,
+      "logps/rejected": -840.0,
+      "loss": 0.4428,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.59375,
+      "rewards/margins": 1.1484375,
+      "rewards/rejected": -6.71875,
+      "step": 5510
+    },
+    {
+      "epoch": 0.20546797937875713,
+      "grad_norm": 5.674745956020104,
+      "learning_rate": 4.832524008466091e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -764.0,
+      "logps/rejected": -872.0,
+      "loss": 0.4299,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -5.71875,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -7.03125,
+      "step": 5520
+    },
+    {
+      "epoch": 0.20584020397908098,
+      "grad_norm": 5.55775427016974,
+      "learning_rate": 4.831353098222655e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -772.0,
+      "logps/rejected": -912.0,
+      "loss": 0.4426,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.90625,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -7.4375,
+      "step": 5530
+    },
+    {
+      "epoch": 0.2062124285794048,
+      "grad_norm": 6.307489786502535,
+      "learning_rate": 4.83017825187134e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -756.0,
+      "logps/rejected": -904.0,
+      "loss": 0.4143,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -5.625,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -7.1875,
+      "step": 5540
+    },
+    {
+      "epoch": 0.20658465317972866,
+      "grad_norm": 6.242121457615083,
+      "learning_rate": 4.828999471395679e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -816.0,
+      "logps/rejected": -932.0,
+      "loss": 0.4179,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -7.65625,
+      "step": 5550
+    },
+    {
+      "epoch": 0.20695687778005248,
+      "grad_norm": 6.8558002150321595,
+      "learning_rate": 4.827816758785853e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -812.0,
+      "logps/rejected": -912.0,
+      "loss": 0.4186,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 1.1640625,
+      "rewards/rejected": -7.40625,
+      "step": 5560
+    },
+    {
+      "epoch": 0.20732910238037633,
+      "grad_norm": 7.3299794584129545,
+      "learning_rate": 4.826630116038677e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -800.0,
+      "logps/rejected": -904.0,
+      "loss": 0.4278,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.125,
+      "rewards/margins": 1.3671875,
+      "rewards/rejected": -7.5,
+      "step": 5570
+    },
+    {
+      "epoch": 0.20770132698070015,
+      "grad_norm": 6.5494812484368286,
+      "learning_rate": 4.825439545157603e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -772.0,
+      "logps/rejected": -896.0,
+      "loss": 0.4552,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -5.75,
+      "rewards/margins": 1.3671875,
+      "rewards/rejected": -7.125,
+      "step": 5580
+    },
+    {
+      "epoch": 0.208073551581024,
+      "grad_norm": 5.472702241786248,
+      "learning_rate": 4.824245048152715e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -748.0,
+      "logps/rejected": -856.0,
+      "loss": 0.4388,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.59375,
+      "rewards/margins": 1.203125,
+      "rewards/rejected": -6.78125,
+      "step": 5590
+    },
+    {
+      "epoch": 0.20844577618134782,
+      "grad_norm": 5.476594065118695,
+      "learning_rate": 4.823046627040725e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -748.0,
+      "logps/rejected": -856.0,
+      "loss": 0.4481,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -5.6875,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -7.0,
+      "step": 5600
+    },
+    {
+      "epoch": 0.20881800078167165,
+      "grad_norm": 5.988377094099928,
+      "learning_rate": 4.821844283844972e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -756.0,
+      "logps/rejected": -896.0,
+      "loss": 0.4269,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -5.59375,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -7.03125,
+      "step": 5610
+    },
+    {
+      "epoch": 0.2091902253819955,
+      "grad_norm": 6.901883765962637,
+      "learning_rate": 4.820638020595414e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -784.0,
+      "logps/rejected": -900.0,
+      "loss": 0.437,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -5.96875,
+      "rewards/margins": 1.234375,
+      "rewards/rejected": -7.1875,
+      "step": 5620
+    },
+    {
+      "epoch": 0.20956244998231932,
+      "grad_norm": 7.104942828033639,
+      "learning_rate": 4.819427839328632e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -760.0,
+      "logps/rejected": -872.0,
+      "loss": 0.434,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -5.875,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -7.28125,
+      "step": 5630
+    },
+    {
+      "epoch": 0.20993467458264317,
+      "grad_norm": 5.872803815540037,
+      "learning_rate": 4.818213742087815e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -788.0,
+      "logps/rejected": -912.0,
+      "loss": 0.4201,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -6.0,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -7.5,
+      "step": 5640
+    },
+    {
+      "epoch": 0.210306899182967,
+      "grad_norm": 7.01609954309212,
+      "learning_rate": 4.81699573092277e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -768.0,
+      "logps/rejected": -888.0,
+      "loss": 0.4127,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -5.90625,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -7.3125,
+      "step": 5650
+    },
+    {
+      "epoch": 0.21067912378329084,
+      "grad_norm": 6.5011599034795555,
+      "learning_rate": 4.815773807889907e-07,
+      "logits/chosen": -4.09375,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -748.0,
+      "logps/rejected": -892.0,
+      "loss": 0.4149,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.9375,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -7.375,
+      "step": 5660
+    },
+    {
+      "epoch": 0.21105134838361467,
+      "grad_norm": 5.619096198602472,
+      "learning_rate": 4.814547975052244e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -804.0,
+      "logps/rejected": -916.0,
+      "loss": 0.4433,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.1875,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -7.28125,
+      "step": 5670
+    },
+    {
+      "epoch": 0.21142357298393852,
+      "grad_norm": 8.68391073705843,
+      "learning_rate": 4.813318234479401e-07,
+      "logits/chosen": -4.125,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -772.0,
+      "logps/rejected": -892.0,
+      "loss": 0.4318,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.90625,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -7.21875,
+      "step": 5680
+    },
+    {
+      "epoch": 0.21179579758426234,
+      "grad_norm": 7.399522959076703,
+      "learning_rate": 4.812084588247592e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -792.0,
+      "logps/rejected": -904.0,
+      "loss": 0.4214,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.125,
+      "rewards/margins": 1.1796875,
+      "rewards/rejected": -7.3125,
+      "step": 5690
+    },
+    {
+      "epoch": 0.2121680221845862,
+      "grad_norm": 6.271029452514774,
+      "learning_rate": 4.810847038439626e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -772.0,
+      "logps/rejected": -884.0,
+      "loss": 0.4406,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.03125,
+      "rewards/margins": 1.171875,
+      "rewards/rejected": -7.1875,
+      "step": 5700
+    },
+    {
+      "epoch": 0.21254024678491001,
+      "grad_norm": 5.679004085620849,
+      "learning_rate": 4.809605587144904e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -796.0,
+      "logps/rejected": -936.0,
+      "loss": 0.4407,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -7.75,
+      "step": 5710
+    },
+    {
+      "epoch": 0.21291247138523384,
+      "grad_norm": 6.064190117711728,
+      "learning_rate": 4.808360236459412e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -788.0,
+      "logps/rejected": -928.0,
+      "loss": 0.4149,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.0625,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -7.625,
+      "step": 5720
+    },
+    {
+      "epoch": 0.2132846959855577,
+      "grad_norm": 5.34081328966269,
+      "learning_rate": 4.807110988485721e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -804.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4336,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.21875,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -7.90625,
+      "step": 5730
+    },
+    {
+      "epoch": 0.2136569205858815,
+      "grad_norm": 5.142041992023566,
+      "learning_rate": 4.805857845332983e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.921875,
+      "logps/chosen": -820.0,
+      "logps/rejected": -940.0,
+      "loss": 0.3852,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -7.65625,
+      "step": 5740
+    },
+    {
+      "epoch": 0.21402914518620536,
+      "grad_norm": 5.5563528347744215,
+      "learning_rate": 4.804600809116925e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.921875,
+      "logps/chosen": -800.0,
+      "logps/rejected": -940.0,
+      "loss": 0.4178,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.0625,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -7.65625,
+      "step": 5750
+    },
+    {
+      "epoch": 0.21440136978652918,
+      "grad_norm": 7.038481769129602,
+      "learning_rate": 4.803339881959845e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -768.0,
+      "logps/rejected": -888.0,
+      "loss": 0.4363,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -5.78125,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -7.25,
+      "step": 5760
+    },
+    {
+      "epoch": 0.21477359438685303,
+      "grad_norm": 7.308586162270306,
+      "learning_rate": 4.802075065990613e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -804.0,
+      "logps/rejected": -876.0,
+      "loss": 0.4378,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -6.09375,
+      "rewards/margins": 1.03125,
+      "rewards/rejected": -7.125,
+      "step": 5770
+    },
+    {
+      "epoch": 0.21514581898717686,
+      "grad_norm": 6.442639898934758,
+      "learning_rate": 4.800806363344663e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -784.0,
+      "logps/rejected": -900.0,
+      "loss": 0.4456,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.125,
+      "rewards/margins": 1.1328125,
+      "rewards/rejected": -7.25,
+      "step": 5780
+    },
+    {
+      "epoch": 0.2155180435875007,
+      "grad_norm": 5.545584310833082,
+      "learning_rate": 4.799533776163993e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -772.0,
+      "logps/rejected": -900.0,
+      "loss": 0.4252,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -6.0625,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -7.34375,
+      "step": 5790
+    },
+    {
+      "epoch": 0.21589026818782453,
+      "grad_norm": 6.0723328777733885,
+      "learning_rate": 4.798257306597156e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -804.0,
+      "logps/rejected": -908.0,
+      "loss": 0.4043,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.15625,
+      "rewards/margins": 1.078125,
+      "rewards/rejected": -7.21875,
+      "step": 5800
+    },
+    {
+      "epoch": 0.21626249278814838,
+      "grad_norm": 6.224187576028383,
+      "learning_rate": 4.796976956799264e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -824.0,
+      "logps/rejected": -940.0,
+      "loss": 0.4392,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -7.6875,
+      "step": 5810
+    },
+    {
+      "epoch": 0.2166347173884722,
+      "grad_norm": 6.411589765759591,
+      "learning_rate": 4.795692728931977e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.75,
+      "logps/chosen": -820.0,
+      "logps/rejected": -952.0,
+      "loss": 0.4378,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.40625,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -7.84375,
+      "step": 5820
+    },
+    {
+      "epoch": 0.21700694198879603,
+      "grad_norm": 5.717319870956347,
+      "learning_rate": 4.794404625163503e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -804.0,
+      "logps/rejected": -912.0,
+      "loss": 0.4175,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -6.15625,
+      "rewards/margins": 1.21875,
+      "rewards/rejected": -7.375,
+      "step": 5830
+    },
+    {
+      "epoch": 0.21737916658911988,
+      "grad_norm": 4.959897879077896,
+      "learning_rate": 4.793112647668594e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -744.0,
+      "logps/rejected": -888.0,
+      "loss": 0.4231,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -5.65625,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -7.21875,
+      "step": 5840
+    },
+    {
+      "epoch": 0.2177513911894437,
+      "grad_norm": 6.534440761415513,
+      "learning_rate": 4.791816798628544e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -760.0,
+      "logps/rejected": -896.0,
+      "loss": 0.4282,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -5.75,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -7.375,
+      "step": 5850
+    },
+    {
+      "epoch": 0.21812361578976755,
+      "grad_norm": 6.128108343721542,
+      "learning_rate": 4.790517080231179e-07,
+      "logits/chosen": -4.09375,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -832.0,
+      "logps/rejected": -940.0,
+      "loss": 0.4541,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -7.71875,
+      "step": 5860
+    },
+    {
+      "epoch": 0.21849584039009137,
+      "grad_norm": 8.19536165570185,
+      "learning_rate": 4.789213494670864e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -796.0,
+      "logps/rejected": -876.0,
+      "loss": 0.4372,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -5.9375,
+      "rewards/margins": 1.1015625,
+      "rewards/rejected": -7.03125,
+      "step": 5870
+    },
+    {
+      "epoch": 0.21886806499041522,
+      "grad_norm": 5.179616613587669,
+      "learning_rate": 4.787906044148489e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -736.0,
+      "logps/rejected": -868.0,
+      "loss": 0.4207,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -5.3125,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -6.90625,
+      "step": 5880
+    },
+    {
+      "epoch": 0.21924028959073905,
+      "grad_norm": 5.650400881885734,
+      "learning_rate": 4.786594730871467e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -800.0,
+      "logps/rejected": -916.0,
+      "loss": 0.4115,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -5.96875,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -7.375,
+      "step": 5890
+    },
+    {
+      "epoch": 0.2196125141910629,
+      "grad_norm": 6.225431096268802,
+      "learning_rate": 4.785279557053739e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -756.0,
+      "logps/rejected": -888.0,
+      "loss": 0.4517,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -5.875,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -7.15625,
+      "step": 5900
+    },
+    {
+      "epoch": 0.21998473879138672,
+      "grad_norm": 5.890086153536098,
+      "learning_rate": 4.78396052491576e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -756.0,
+      "logps/rejected": -908.0,
+      "loss": 0.4171,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -5.84375,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -7.46875,
+      "step": 5910
+    },
+    {
+      "epoch": 0.22035696339171057,
+      "grad_norm": 5.111685546177177,
+      "learning_rate": 4.782637636684499e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -772.0,
+      "logps/rejected": -900.0,
+      "loss": 0.4081,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -5.96875,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -7.40625,
+      "step": 5920
+    },
+    {
+      "epoch": 0.2207291879920344,
+      "grad_norm": 6.367102774287387,
+      "learning_rate": 4.781310894593437e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.921875,
+      "logps/chosen": -816.0,
+      "logps/rejected": -916.0,
+      "loss": 0.4363,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.0546875,
+      "rewards/rejected": -7.375,
+      "step": 5930
+    },
+    {
+      "epoch": 0.22110141259235822,
+      "grad_norm": 6.732957539192681,
+      "learning_rate": 4.779980300882559e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -788.0,
+      "logps/rejected": -948.0,
+      "loss": 0.4073,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -6.1875,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -7.84375,
+      "step": 5940
+    },
+    {
+      "epoch": 0.22147363719268207,
+      "grad_norm": 6.607356044114583,
+      "learning_rate": 4.778645857798357e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -752.0,
+      "logps/rejected": -856.0,
+      "loss": 0.4256,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -6.0625,
+      "rewards/margins": 0.98828125,
+      "rewards/rejected": -7.03125,
+      "step": 5950
+    },
+    {
+      "epoch": 0.2218458617930059,
+      "grad_norm": 6.235417597487092,
+      "learning_rate": 4.777307567593818e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -768.0,
+      "logps/rejected": -896.0,
+      "loss": 0.429,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -5.84375,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -7.28125,
+      "step": 5960
+    },
+    {
+      "epoch": 0.22221808639332974,
+      "grad_norm": 6.5454457344607,
+      "learning_rate": 4.775965432528426e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -808.0,
+      "logps/rejected": -932.0,
+      "loss": 0.4138,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -7.65625,
+      "step": 5970
+    },
+    {
+      "epoch": 0.22259031099365356,
+      "grad_norm": 6.331675313138522,
+      "learning_rate": 4.774619454868156e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -768.0,
+      "logps/rejected": -912.0,
+      "loss": 0.4224,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.0,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -7.625,
+      "step": 5980
+    },
+    {
+      "epoch": 0.2229625355939774,
+      "grad_norm": 5.751801674711064,
+      "learning_rate": 4.773269636885471e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -792.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4083,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.0,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -7.53125,
+      "step": 5990
+    },
+    {
+      "epoch": 0.22333476019430124,
+      "grad_norm": 5.848378657103043,
+      "learning_rate": 4.771915980859318e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -804.0,
+      "logps/rejected": -908.0,
+      "loss": 0.424,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.0625,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -7.3125,
+      "step": 6000
+    },
+    {
+      "epoch": 0.22370698479462509,
+      "grad_norm": 6.26990076919701,
+      "learning_rate": 4.770558489075124e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.9375,
+      "logps/chosen": -808.0,
+      "logps/rejected": -928.0,
+      "loss": 0.4112,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.09375,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -7.59375,
+      "step": 6010
+    },
+    {
+      "epoch": 0.2240792093949489,
+      "grad_norm": 7.548283260332925,
+      "learning_rate": 4.76919716382479e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -816.0,
+      "logps/rejected": -924.0,
+      "loss": 0.4202,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.21875,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -7.6875,
+      "step": 6020
+    },
+    {
+      "epoch": 0.22445143399527276,
+      "grad_norm": 5.6872152480335165,
+      "learning_rate": 4.7678320074066925e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -804.0,
+      "logps/rejected": -944.0,
+      "loss": 0.399,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.09375,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -7.78125,
+      "step": 6030
+    },
+    {
+      "epoch": 0.22482365859559658,
+      "grad_norm": 7.868638878167568,
+      "learning_rate": 4.766463022125673e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.96875,
+      "logps/chosen": -800.0,
+      "logps/rejected": -908.0,
+      "loss": 0.4192,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.0625,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -7.59375,
+      "step": 6040
+    },
+    {
+      "epoch": 0.22519588319592043,
+      "grad_norm": 12.322515515850705,
+      "learning_rate": 4.765090210293039e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -808.0,
+      "logps/rejected": -924.0,
+      "loss": 0.4352,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.28125,
+      "rewards/margins": 1.359375,
+      "rewards/rejected": -7.65625,
+      "step": 6050
+    },
+    {
+      "epoch": 0.22556810779624425,
+      "grad_norm": 5.527528242493144,
+      "learning_rate": 4.76371357422656e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.875,
+      "logps/chosen": -768.0,
+      "logps/rejected": -896.0,
+      "loss": 0.4129,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -5.90625,
+      "rewards/margins": 1.3125,
+      "rewards/rejected": -7.21875,
+      "step": 6060
+    },
+    {
+      "epoch": 0.22594033239656808,
+      "grad_norm": 6.530297056526231,
+      "learning_rate": 4.7623331162504585e-07,
+      "logits/chosen": -4.125,
+      "logits/rejected": -3.921875,
+      "logps/chosen": -760.0,
+      "logps/rejected": -900.0,
+      "loss": 0.4273,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -5.9375,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -7.4375,
+      "step": 6070
+    },
+    {
+      "epoch": 0.22631255699689193,
+      "grad_norm": 4.898526690129243,
+      "learning_rate": 4.7609488386954137e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -752.0,
+      "logps/rejected": -888.0,
+      "loss": 0.4315,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -5.78125,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -7.09375,
+      "step": 6080
+    },
+    {
+      "epoch": 0.22668478159721575,
+      "grad_norm": 6.132011810945695,
+      "learning_rate": 4.759560743898551e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -828.0,
+      "logps/rejected": -936.0,
+      "loss": 0.4062,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.0625,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -7.59375,
+      "step": 6090
+    },
+    {
+      "epoch": 0.2270570061975396,
+      "grad_norm": 5.023249321329233,
+      "learning_rate": 4.758168834203439e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -788.0,
+      "logps/rejected": -924.0,
+      "loss": 0.3967,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.0,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -7.625,
+      "step": 6100
+    },
+    {
+      "epoch": 0.22742923079786342,
+      "grad_norm": 8.489424090439904,
+      "learning_rate": 4.7567731119600916e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -752.0,
+      "logps/rejected": -932.0,
+      "loss": 0.4144,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -5.8125,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -7.59375,
+      "step": 6110
+    },
+    {
+      "epoch": 0.22780145539818727,
+      "grad_norm": 6.383030066432269,
+      "learning_rate": 4.755373579524957e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -796.0,
+      "logps/rejected": -920.0,
+      "loss": 0.4446,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 1.28125,
+      "rewards/rejected": -7.5625,
+      "step": 6120
+    },
+    {
+      "epoch": 0.2281736799985111,
+      "grad_norm": 5.122973760511068,
+      "learning_rate": 4.753970239260916e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -824.0,
+      "logps/rejected": -924.0,
+      "loss": 0.4561,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.46875,
+      "rewards/margins": 1.1640625,
+      "rewards/rejected": -7.625,
+      "step": 6130
+    },
+    {
+      "epoch": 0.22854590459883495,
+      "grad_norm": 5.306429521480451,
+      "learning_rate": 4.752563093537279e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -760.0,
+      "logps/rejected": -896.0,
+      "loss": 0.4162,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -5.84375,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -7.34375,
+      "step": 6140
+    },
+    {
+      "epoch": 0.22891812919915877,
+      "grad_norm": 5.682541141240045,
+      "learning_rate": 4.751152144729781e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -804.0,
+      "logps/rejected": -912.0,
+      "loss": 0.4244,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.15625,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": -7.5,
+      "step": 6150
+    },
+    {
+      "epoch": 0.22929035379948262,
+      "grad_norm": 6.436204816144081,
+      "learning_rate": 4.749737395220578e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.875,
+      "logps/chosen": -792.0,
+      "logps/rejected": -928.0,
+      "loss": 0.4059,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -5.90625,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -7.5,
+      "step": 6160
+    },
+    {
+      "epoch": 0.22966257839980644,
+      "grad_norm": 5.94076333669987,
+      "learning_rate": 4.748318847398242e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -788.0,
+      "logps/rejected": -924.0,
+      "loss": 0.4211,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.0,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -7.5,
+      "step": 6170
+    },
+    {
+      "epoch": 0.23003480300013027,
+      "grad_norm": 4.959858507272183,
+      "learning_rate": 4.746896503657759e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -752.0,
+      "logps/rejected": -892.0,
+      "loss": 0.4279,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -5.78125,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -7.3125,
+      "step": 6180
+    },
+    {
+      "epoch": 0.23040702760045412,
+      "grad_norm": 5.1969834553735135,
+      "learning_rate": 4.745470366400525e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -792.0,
+      "logps/rejected": -916.0,
+      "loss": 0.408,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -5.84375,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -7.375,
+      "step": 6190
+    },
+    {
+      "epoch": 0.23077925220077794,
+      "grad_norm": 6.445248230897788,
+      "learning_rate": 4.744040438034338e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -784.0,
+      "logps/rejected": -884.0,
+      "loss": 0.4294,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -5.9375,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -7.28125,
+      "step": 6200
+    },
+    {
+      "epoch": 0.2311514768011018,
+      "grad_norm": 5.817894647914261,
+      "learning_rate": 4.7426067209733977e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -760.0,
+      "logps/rejected": -864.0,
+      "loss": 0.4281,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -5.59375,
+      "rewards/margins": 1.3125,
+      "rewards/rejected": -6.90625,
+      "step": 6210
+    },
+    {
+      "epoch": 0.2315237014014256,
+      "grad_norm": 6.926605729963249,
+      "learning_rate": 4.7411692176383013e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -784.0,
+      "logps/rejected": -924.0,
+      "loss": 0.4083,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -5.9375,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -7.6875,
+      "step": 6220
+    },
+    {
+      "epoch": 0.23189592600174946,
+      "grad_norm": 5.851495287373424,
+      "learning_rate": 4.739727930456037e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -836.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4141,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.40625,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -7.90625,
+      "step": 6230
+    },
+    {
+      "epoch": 0.2322681506020733,
+      "grad_norm": 5.771059985402298,
+      "learning_rate": 4.738282861859982e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -812.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4299,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -7.875,
+      "step": 6240
+    },
+    {
+      "epoch": 0.23264037520239714,
+      "grad_norm": 5.489245233557804,
+      "learning_rate": 4.7368340142898983e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -764.0,
+      "logps/rejected": -864.0,
+      "loss": 0.4349,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -5.9375,
+      "rewards/margins": 1.1953125,
+      "rewards/rejected": -7.125,
+      "step": 6250
+    },
+    {
+      "epoch": 0.23301259980272096,
+      "grad_norm": 6.09632578490293,
+      "learning_rate": 4.7353813901919283e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -760.0,
+      "logps/rejected": -892.0,
+      "loss": 0.4199,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -5.90625,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -7.3125,
+      "step": 6260
+    },
+    {
+      "epoch": 0.2333848244030448,
+      "grad_norm": 6.522106259492334,
+      "learning_rate": 4.7339249920185885e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -816.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4152,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.28125,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -7.65625,
+      "step": 6270
+    },
+    {
+      "epoch": 0.23375704900336863,
+      "grad_norm": 5.71911587721337,
+      "learning_rate": 4.73246482222877e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -808.0,
+      "logps/rejected": -956.0,
+      "loss": 0.3941,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -7.9375,
+      "step": 6280
+    },
+    {
+      "epoch": 0.23412927360369246,
+      "grad_norm": 7.391656384188058,
+      "learning_rate": 4.73100088328773e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -836.0,
+      "logps/rejected": -952.0,
+      "loss": 0.4293,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.3125,
+      "rewards/rejected": -7.6875,
+      "step": 6290
+    },
+    {
+      "epoch": 0.2345014982040163,
+      "grad_norm": 5.114771672962533,
+      "learning_rate": 4.72953317766709e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.75,
+      "logps/chosen": -792.0,
+      "logps/rejected": -928.0,
+      "loss": 0.4358,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.03125,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -7.46875,
+      "step": 6300
+    },
+    {
+      "epoch": 0.23487372280434013,
+      "grad_norm": 5.680627173402873,
+      "learning_rate": 4.7280617078448294e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -784.0,
+      "logps/rejected": -916.0,
+      "loss": 0.4297,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.0625,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -7.375,
+      "step": 6310
+    },
+    {
+      "epoch": 0.23524594740466398,
+      "grad_norm": 5.933723635892915,
+      "learning_rate": 4.726586476305285e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -772.0,
+      "logps/rejected": -892.0,
+      "loss": 0.4288,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -5.90625,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -7.40625,
+      "step": 6320
+    },
+    {
+      "epoch": 0.2356181720049878,
+      "grad_norm": 5.22075790109594,
+      "learning_rate": 4.725107485539143e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -824.0,
+      "logps/rejected": -952.0,
+      "loss": 0.4388,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -7.8125,
+      "step": 6330
+    },
+    {
+      "epoch": 0.23599039660531165,
+      "grad_norm": 5.4705830083365194,
+      "learning_rate": 4.723624738043438e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -816.0,
+      "logps/rejected": -924.0,
+      "loss": 0.4292,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -7.59375,
+      "step": 6340
+    },
+    {
+      "epoch": 0.23636262120563548,
+      "grad_norm": 5.866031715136251,
+      "learning_rate": 4.7221382363215447e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -832.0,
+      "logps/rejected": -960.0,
+      "loss": 0.4357,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.59375,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -8.0625,
+      "step": 6350
+    },
+    {
+      "epoch": 0.23673484580595933,
+      "grad_norm": 5.799551873821158,
+      "learning_rate": 4.72064798288318e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -828.0,
+      "logps/rejected": -924.0,
+      "loss": 0.4073,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 1.2109375,
+      "rewards/rejected": -7.625,
+      "step": 6360
+    },
+    {
+      "epoch": 0.23710707040628315,
+      "grad_norm": 6.53867763652818,
+      "learning_rate": 4.7191539802443906e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -808.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4169,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -7.84375,
+      "step": 6370
+    },
+    {
+      "epoch": 0.237479295006607,
+      "grad_norm": 5.611965867491773,
+      "learning_rate": 4.717656230927557e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -820.0,
+      "logps/rejected": -948.0,
+      "loss": 0.427,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -7.875,
+      "step": 6380
+    },
+    {
+      "epoch": 0.23785151960693082,
+      "grad_norm": 6.7368464566277035,
+      "learning_rate": 4.7161547374613817e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -816.0,
+      "logps/rejected": -932.0,
+      "loss": 0.4174,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -7.71875,
+      "step": 6390
+    },
+    {
+      "epoch": 0.23822374420725465,
+      "grad_norm": 6.3362822283526175,
+      "learning_rate": 4.714649502380893e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -808.0,
+      "logps/rejected": -940.0,
+      "loss": 0.4183,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.21875,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -7.625,
+      "step": 6400
+    },
+    {
+      "epoch": 0.2385959688075785,
+      "grad_norm": 6.36364465001046,
+      "learning_rate": 4.7131405282274315e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -836.0,
+      "logps/rejected": -912.0,
+      "loss": 0.448,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.0703125,
+      "rewards/rejected": -7.59375,
+      "step": 6410
+    },
+    {
+      "epoch": 0.23896819340790232,
+      "grad_norm": 5.118738021745892,
+      "learning_rate": 4.7116278175486546e-07,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -784.0,
+      "logps/rejected": -904.0,
+      "loss": 0.4465,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.1875,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -7.53125,
+      "step": 6420
+    },
+    {
+      "epoch": 0.23934041800822617,
+      "grad_norm": 4.7360386085029536,
+      "learning_rate": 4.7101113728985253e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -792.0,
+      "logps/rejected": -920.0,
+      "loss": 0.4212,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.03125,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -7.40625,
+      "step": 6430
+    },
+    {
+      "epoch": 0.23971264260855,
+      "grad_norm": 5.656427810531944,
+      "learning_rate": 4.7085911968373135e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -788.0,
+      "logps/rejected": -904.0,
+      "loss": 0.4055,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.125,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": -7.46875,
+      "step": 6440
+    },
+    {
+      "epoch": 0.24008486720887384,
+      "grad_norm": 5.47845254609012,
+      "learning_rate": 4.7070672919315856e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -816.0,
+      "logps/rejected": -912.0,
+      "loss": 0.4203,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.21875,
+      "rewards/margins": 1.234375,
+      "rewards/rejected": -7.4375,
+      "step": 6450
+    },
+    {
+      "epoch": 0.24045709180919767,
+      "grad_norm": 6.347307076455001,
+      "learning_rate": 4.705539660754207e-07,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -816.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4281,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -7.78125,
+      "step": 6460
+    },
+    {
+      "epoch": 0.24082931640952152,
+      "grad_norm": 5.266163040844717,
+      "learning_rate": 4.704008305884332e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -832.0,
+      "logps/rejected": -932.0,
+      "loss": 0.4424,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.21875,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -7.6875,
+      "step": 6470
+    },
+    {
+      "epoch": 0.24120154100984534,
+      "grad_norm": 6.02256912553513,
+      "learning_rate": 4.7024732299074023e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -804.0,
+      "logps/rejected": -920.0,
+      "loss": 0.4236,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.1875,
+      "rewards/margins": 1.359375,
+      "rewards/rejected": -7.53125,
+      "step": 6480
+    },
+    {
+      "epoch": 0.2415737656101692,
+      "grad_norm": 7.541961383816745,
+      "learning_rate": 4.7009344354151424e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -808.0,
+      "logps/rejected": -924.0,
+      "loss": 0.4243,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.375,
+      "rewards/rejected": -7.6875,
+      "step": 6490
+    },
+    {
+      "epoch": 0.241945990210493,
+      "grad_norm": 7.430317560153076,
+      "learning_rate": 4.6993919250055557e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -812.0,
+      "logps/rejected": -928.0,
+      "loss": 0.4389,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.125,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -7.46875,
+      "step": 6500
+    },
+    {
+      "epoch": 0.24231821481081683,
+      "grad_norm": 6.407746800397953,
+      "learning_rate": 4.6978457012829174e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -836.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4015,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.46875,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -7.90625,
+      "step": 6510
+    },
+    {
+      "epoch": 0.24269043941114068,
+      "grad_norm": 6.252142015750858,
+      "learning_rate": 4.6962957668577736e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -844.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4302,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.46875,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -7.71875,
+      "step": 6520
+    },
+    {
+      "epoch": 0.2430626640114645,
+      "grad_norm": 5.497439476416673,
+      "learning_rate": 4.694742124346934e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -812.0,
+      "logps/rejected": -924.0,
+      "loss": 0.4382,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.28125,
+      "rewards/margins": 1.28125,
+      "rewards/rejected": -7.5625,
+      "step": 6530
+    },
+    {
+      "epoch": 0.24343488861178836,
+      "grad_norm": 6.490216840077626,
+      "learning_rate": 4.6931847763734703e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -796.0,
+      "logps/rejected": -920.0,
+      "loss": 0.4289,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.09375,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -7.40625,
+      "step": 6540
+    },
+    {
+      "epoch": 0.24380711321211218,
+      "grad_norm": 5.615847818685157,
+      "learning_rate": 4.6916237255667093e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -780.0,
+      "logps/rejected": -920.0,
+      "loss": 0.4257,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.96875,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -7.5,
+      "step": 6550
+    },
+    {
+      "epoch": 0.24417933781243603,
+      "grad_norm": 5.682486676946517,
+      "learning_rate": 4.6900589745622294e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -788.0,
+      "logps/rejected": -912.0,
+      "loss": 0.4311,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -6.09375,
+      "rewards/margins": 1.265625,
+      "rewards/rejected": -7.375,
+      "step": 6560
+    },
+    {
+      "epoch": 0.24455156241275985,
+      "grad_norm": 5.820232103868232,
+      "learning_rate": 4.688490526001856e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -848.0,
+      "logps/rejected": -960.0,
+      "loss": 0.4161,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.265625,
+      "rewards/rejected": -8.0625,
+      "step": 6570
+    },
+    {
+      "epoch": 0.2449237870130837,
+      "grad_norm": 8.169062220330268,
+      "learning_rate": 4.6869183825336587e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -800.0,
+      "logps/rejected": -952.0,
+      "loss": 0.4553,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -7.71875,
+      "step": 6580
+    },
+    {
+      "epoch": 0.24529601161340753,
+      "grad_norm": 5.721998116739456,
+      "learning_rate": 4.685342546811943e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -760.0,
+      "logps/rejected": -912.0,
+      "loss": 0.4278,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -5.96875,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -7.5,
+      "step": 6590
+    },
+    {
+      "epoch": 0.24566823621373138,
+      "grad_norm": 6.2761689310538555,
+      "learning_rate": 4.6837630214972514e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -844.0,
+      "logps/rejected": -932.0,
+      "loss": 0.4474,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 1.0625,
+      "rewards/rejected": -7.4375,
+      "step": 6600
+    },
+    {
+      "epoch": 0.2460404608140552,
+      "grad_norm": 5.900942155872346,
+      "learning_rate": 4.6821798092563514e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -816.0,
+      "logps/rejected": -908.0,
+      "loss": 0.4077,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.15625,
+      "rewards/margins": 1.203125,
+      "rewards/rejected": -7.375,
+      "step": 6610
+    },
+    {
+      "epoch": 0.24641268541437902,
+      "grad_norm": 6.641446753014602,
+      "learning_rate": 4.680592912762238e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.875,
+      "logps/chosen": -828.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4063,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -7.9375,
+      "step": 6620
+    },
+    {
+      "epoch": 0.24678491001470287,
+      "grad_norm": 9.449492598841816,
+      "learning_rate": 4.6790023346941274e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -844.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4193,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.46875,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -7.96875,
+      "step": 6630
+    },
+    {
+      "epoch": 0.2471571346150267,
+      "grad_norm": 7.714903830083636,
+      "learning_rate": 4.677408077737449e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.9375,
+      "logps/chosen": -808.0,
+      "logps/rejected": -932.0,
+      "loss": 0.441,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.28125,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -7.6875,
+      "step": 6640
+    },
+    {
+      "epoch": 0.24752935921535055,
+      "grad_norm": 5.4441759336675535,
+      "learning_rate": 4.6758101445838457e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -844.0,
+      "logps/rejected": -936.0,
+      "loss": 0.4336,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.2265625,
+      "rewards/rejected": -7.75,
+      "step": 6650
+    },
+    {
+      "epoch": 0.24790158381567437,
+      "grad_norm": 7.739944946158606,
+      "learning_rate": 4.6742085379311645e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -812.0,
+      "logps/rejected": -932.0,
+      "loss": 0.4067,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.28125,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -7.59375,
+      "step": 6660
+    },
+    {
+      "epoch": 0.24827380841599822,
+      "grad_norm": 5.9997287712072005,
+      "learning_rate": 4.6726032604834566e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -828.0,
+      "logps/rejected": -952.0,
+      "loss": 0.4163,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.40625,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -7.84375,
+      "step": 6670
+    },
+    {
+      "epoch": 0.24864603301632204,
+      "grad_norm": 5.798419256905425,
+      "learning_rate": 4.67099431495097e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -840.0,
+      "logps/rejected": -948.0,
+      "loss": 0.4266,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -6.5625,
+      "rewards/margins": 1.15625,
+      "rewards/rejected": -7.71875,
+      "step": 6680
+    },
+    {
+      "epoch": 0.2490182576166459,
+      "grad_norm": 8.196094364451225,
+      "learning_rate": 4.669381704050146e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.875,
+      "logps/chosen": -820.0,
+      "logps/rejected": -904.0,
+      "loss": 0.4254,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 1.1328125,
+      "rewards/rejected": -7.5625,
+      "step": 6690
+    },
+    {
+      "epoch": 0.24939048221696972,
+      "grad_norm": 5.894388057612442,
+      "learning_rate": 4.6677654305036136e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -780.0,
+      "logps/rejected": -920.0,
+      "loss": 0.4404,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.09375,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -7.5625,
+      "step": 6700
+    },
+    {
+      "epoch": 0.24976270681729357,
+      "grad_norm": 5.60483072522514,
+      "learning_rate": 4.666145497040186e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -804.0,
+      "logps/rejected": -936.0,
+      "loss": 0.4155,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -7.78125,
+      "step": 6710
+    },
+    {
+      "epoch": 0.2501349314176174,
+      "grad_norm": 5.961096567165508,
+      "learning_rate": 4.664521906394856e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -812.0,
+      "logps/rejected": -952.0,
+      "loss": 0.4256,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.21875,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -7.875,
+      "step": 6720
+    },
+    {
+      "epoch": 0.2505071560179412,
+      "grad_norm": 9.91923906368576,
+      "learning_rate": 4.662894661308789e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -796.0,
+      "logps/rejected": -892.0,
+      "loss": 0.4507,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 1.28125,
+      "rewards/rejected": -7.53125,
+      "step": 6730
+    },
+    {
+      "epoch": 0.25087938061826504,
+      "grad_norm": 5.83173760425227,
+      "learning_rate": 4.6612637645293243e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -788.0,
+      "logps/rejected": -928.0,
+      "loss": 0.4398,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -6.0625,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -7.4375,
+      "step": 6740
+    },
+    {
+      "epoch": 0.2512516052185889,
+      "grad_norm": 6.10559542732337,
+      "learning_rate": 4.659629218809963e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -788.0,
+      "logps/rejected": -928.0,
+      "loss": 0.4107,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.09375,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -7.6875,
+      "step": 6750
+    },
+    {
+      "epoch": 0.25162382981891274,
+      "grad_norm": 5.355304419131932,
+      "learning_rate": 4.657991026910366e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -824.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4118,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -7.875,
+      "step": 6760
+    },
+    {
+      "epoch": 0.25199605441923656,
+      "grad_norm": 6.250452222122328,
+      "learning_rate": 4.656349191596355e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -812.0,
+      "logps/rejected": -952.0,
+      "loss": 0.4031,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -7.84375,
+      "step": 6770
+    },
+    {
+      "epoch": 0.2523682790195604,
+      "grad_norm": 7.464677342725597,
+      "learning_rate": 4.6547037156398976e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.625,
+      "logps/chosen": -720.0,
+      "logps/rejected": -856.0,
+      "loss": 0.4148,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -5.59375,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -7.0625,
+      "step": 6780
+    },
+    {
+      "epoch": 0.25274050361988426,
+      "grad_norm": 6.004157006557244,
+      "learning_rate": 4.653054601819112e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -760.0,
+      "logps/rejected": -912.0,
+      "loss": 0.4222,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -5.78125,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -7.46875,
+      "step": 6790
+    },
+    {
+      "epoch": 0.2531127282202081,
+      "grad_norm": 6.554328420068891,
+      "learning_rate": 4.651401852918256e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -800.0,
+      "logps/rejected": -952.0,
+      "loss": 0.4463,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.125,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -7.75,
+      "step": 6800
+    },
+    {
+      "epoch": 0.2534849528205319,
+      "grad_norm": 6.002968003872758,
+      "learning_rate": 4.6497454717277253e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -788.0,
+      "logps/rejected": -904.0,
+      "loss": 0.4184,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.03125,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -7.375,
+      "step": 6810
+    },
+    {
+      "epoch": 0.25385717742085573,
+      "grad_norm": 5.912557991966657,
+      "learning_rate": 4.648085461044049e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -764.0,
+      "logps/rejected": -880.0,
+      "loss": 0.4264,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -5.65625,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -7.09375,
+      "step": 6820
+    },
+    {
+      "epoch": 0.2542294020211796,
+      "grad_norm": 5.879871351533836,
+      "learning_rate": 4.646421823669883e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -788.0,
+      "logps/rejected": -916.0,
+      "loss": 0.4379,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -5.9375,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -7.34375,
+      "step": 6830
+    },
+    {
+      "epoch": 0.25460162662150343,
+      "grad_norm": 5.8083438839498855,
+      "learning_rate": 4.644754562414006e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -764.0,
+      "logps/rejected": -888.0,
+      "loss": 0.4194,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -5.875,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -7.21875,
+      "step": 6840
+    },
+    {
+      "epoch": 0.25497385122182725,
+      "grad_norm": 4.823811006340166,
+      "learning_rate": 4.6430836800913167e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -800.0,
+      "logps/rejected": -912.0,
+      "loss": 0.3981,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -6.03125,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -7.59375,
+      "step": 6850
+    },
+    {
+      "epoch": 0.2553460758221511,
+      "grad_norm": 6.944388369911442,
+      "learning_rate": 4.6414091795228247e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -856.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4018,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.5625,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -8.0625,
+      "step": 6860
+    },
+    {
+      "epoch": 0.2557183004224749,
+      "grad_norm": 6.624621827733266,
+      "learning_rate": 4.6397310635356514e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -836.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4331,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.625,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -7.90625,
+      "step": 6870
+    },
+    {
+      "epoch": 0.2560905250227988,
+      "grad_norm": 5.695681707945807,
+      "learning_rate": 4.63804933496302e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -800.0,
+      "logps/rejected": -920.0,
+      "loss": 0.4283,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -7.71875,
+      "step": 6880
+    },
+    {
+      "epoch": 0.2564627496231226,
+      "grad_norm": 5.975492463825295,
+      "learning_rate": 4.6363639966442535e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -832.0,
+      "logps/rejected": -980.0,
+      "loss": 0.4295,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.28125,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -7.90625,
+      "step": 6890
+    },
+    {
+      "epoch": 0.2568349742234464,
+      "grad_norm": 7.9029141557386255,
+      "learning_rate": 4.634675051424771e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -820.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4352,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -7.78125,
+      "step": 6900
+    },
+    {
+      "epoch": 0.25720719882377024,
+      "grad_norm": 5.924450624299557,
+      "learning_rate": 4.632982502156078e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -816.0,
+      "logps/rejected": -928.0,
+      "loss": 0.4245,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -7.65625,
+      "step": 6910
+    },
+    {
+      "epoch": 0.2575794234240941,
+      "grad_norm": 6.920771352726799,
+      "learning_rate": 4.6312863516957686e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -848.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4197,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.3671875,
+      "rewards/rejected": -8.0,
+      "step": 6920
+    },
+    {
+      "epoch": 0.25795164802441795,
+      "grad_norm": 5.820297292357974,
+      "learning_rate": 4.629586602907514e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -824.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4506,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.5,
+      "rewards/margins": 1.3125,
+      "rewards/rejected": -7.8125,
+      "step": 6930
+    },
+    {
+      "epoch": 0.25832387262474177,
+      "grad_norm": 5.914879662411408,
+      "learning_rate": 4.627883258661061e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -800.0,
+      "logps/rejected": -936.0,
+      "loss": 0.4196,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.03125,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -7.625,
+      "step": 6940
+    },
+    {
+      "epoch": 0.2586960972250656,
+      "grad_norm": 6.198591450715814,
+      "learning_rate": 4.626176321832229e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -772.0,
+      "logps/rejected": -904.0,
+      "loss": 0.4192,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -5.96875,
+      "rewards/margins": 1.3125,
+      "rewards/rejected": -7.28125,
+      "step": 6950
+    },
+    {
+      "epoch": 0.2590683218253894,
+      "grad_norm": 6.382208272677864,
+      "learning_rate": 4.6244657953029005e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -816.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4161,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -6.28125,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -7.59375,
+      "step": 6960
+    },
+    {
+      "epoch": 0.2594405464257133,
+      "grad_norm": 6.195584631295477,
+      "learning_rate": 4.622751681961019e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -828.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4231,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.15625,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -7.71875,
+      "step": 6970
+    },
+    {
+      "epoch": 0.2598127710260371,
+      "grad_norm": 5.674286559755445,
+      "learning_rate": 4.621033984700585e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -796.0,
+      "logps/rejected": -920.0,
+      "loss": 0.4063,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.15625,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -7.59375,
+      "step": 6980
+    },
+    {
+      "epoch": 0.26018499562636094,
+      "grad_norm": 5.892757426878994,
+      "learning_rate": 4.6193127064216486e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -816.0,
+      "logps/rejected": -952.0,
+      "loss": 0.3907,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.28125,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -7.875,
+      "step": 6990
+    },
+    {
+      "epoch": 0.26055722022668476,
+      "grad_norm": 5.779461468889358,
+      "learning_rate": 4.6175878500303054e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -796.0,
+      "logps/rejected": -952.0,
+      "loss": 0.4012,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -7.9375,
+      "step": 7000
+    },
+    {
+      "epoch": 0.26092944482700864,
+      "grad_norm": 7.399670910318273,
+      "learning_rate": 4.615859418438695e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -840.0,
+      "logps/rejected": -948.0,
+      "loss": 0.4199,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -7.78125,
+      "step": 7010
+    },
+    {
+      "epoch": 0.26130166942733246,
+      "grad_norm": 5.579899430532945,
+      "learning_rate": 4.6141274145649876e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -808.0,
+      "logps/rejected": -920.0,
+      "loss": 0.4237,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -7.4375,
+      "step": 7020
+    },
+    {
+      "epoch": 0.2616738940276563,
+      "grad_norm": 5.861809476791946,
+      "learning_rate": 4.612391841333392e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -792.0,
+      "logps/rejected": -932.0,
+      "loss": 0.4062,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -5.875,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -7.53125,
+      "step": 7030
+    },
+    {
+      "epoch": 0.2620461186279801,
+      "grad_norm": 5.734970025363787,
+      "learning_rate": 4.6106527016741377e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -816.0,
+      "logps/rejected": -932.0,
+      "loss": 0.4113,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.09375,
+      "rewards/margins": 1.3671875,
+      "rewards/rejected": -7.46875,
+      "step": 7040
+    },
+    {
+      "epoch": 0.262418343228304,
+      "grad_norm": 7.515271231256475,
+      "learning_rate": 4.608909998523476e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -800.0,
+      "logps/rejected": -928.0,
+      "loss": 0.4251,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.1875,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -7.625,
+      "step": 7050
+    },
+    {
+      "epoch": 0.2627905678286278,
+      "grad_norm": 6.117346670109052,
+      "learning_rate": 4.607163734823678e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -788.0,
+      "logps/rejected": -916.0,
+      "loss": 0.4042,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -5.90625,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -7.4375,
+      "step": 7060
+    },
+    {
+      "epoch": 0.26316279242895163,
+      "grad_norm": 5.732134762321121,
+      "learning_rate": 4.605413913523022e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -808.0,
+      "logps/rejected": -928.0,
+      "loss": 0.4209,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.21875,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -7.5625,
+      "step": 7070
+    },
+    {
+      "epoch": 0.26353501702927545,
+      "grad_norm": 6.168376801567615,
+      "learning_rate": 4.603660537575796e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -792.0,
+      "logps/rejected": -900.0,
+      "loss": 0.4198,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.0,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -7.375,
+      "step": 7080
+    },
+    {
+      "epoch": 0.2639072416295993,
+      "grad_norm": 7.079605830294161,
+      "learning_rate": 4.601903609942287e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -808.0,
+      "logps/rejected": -952.0,
+      "loss": 0.4184,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.1875,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -7.625,
+      "step": 7090
+    },
+    {
+      "epoch": 0.26427946622992315,
+      "grad_norm": 6.1303983560448865,
+      "learning_rate": 4.600143133588781e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -772.0,
+      "logps/rejected": -916.0,
+      "loss": 0.3939,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -5.96875,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -7.46875,
+      "step": 7100
+    },
+    {
+      "epoch": 0.264651690830247,
+      "grad_norm": 7.257289544071512,
+      "learning_rate": 4.598379111487552e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -776.0,
+      "logps/rejected": -960.0,
+      "loss": 0.4291,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -5.96875,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -7.59375,
+      "step": 7110
+    },
+    {
+      "epoch": 0.2650239154305708,
+      "grad_norm": 6.629916399075003,
+      "learning_rate": 4.5966115466168645e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -836.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4251,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -7.65625,
+      "step": 7120
+    },
+    {
+      "epoch": 0.2653961400308946,
+      "grad_norm": 6.854369095431051,
+      "learning_rate": 4.594840441960961e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -800.0,
+      "logps/rejected": -936.0,
+      "loss": 0.423,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.125,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -7.625,
+      "step": 7130
+    },
+    {
+      "epoch": 0.2657683646312185,
+      "grad_norm": 5.452144756823122,
+      "learning_rate": 4.593065800510062e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -764.0,
+      "logps/rejected": -880.0,
+      "loss": 0.4232,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -5.96875,
+      "rewards/margins": 1.2421875,
+      "rewards/rejected": -7.1875,
+      "step": 7140
+    },
+    {
+      "epoch": 0.2661405892315423,
+      "grad_norm": 5.23057721543945,
+      "learning_rate": 4.5912876252603585e-07,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.5,
+      "logps/chosen": -800.0,
+      "logps/rejected": -928.0,
+      "loss": 0.4119,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -6.28125,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -7.59375,
+      "step": 7150
+    },
+    {
+      "epoch": 0.26651281383186615,
+      "grad_norm": 5.98550008036415,
+      "learning_rate": 4.5895059192140095e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -824.0,
+      "logps/rejected": -948.0,
+      "loss": 0.4293,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -7.84375,
+      "step": 7160
+    },
+    {
+      "epoch": 0.26688503843218997,
+      "grad_norm": 6.377827546099224,
+      "learning_rate": 4.587720685379132e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.75,
+      "logps/chosen": -816.0,
+      "logps/rejected": -932.0,
+      "loss": 0.4352,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.359375,
+      "rewards/rejected": -7.6875,
+      "step": 7170
+    },
+    {
+      "epoch": 0.2672572630325138,
+      "grad_norm": 6.759601967564584,
+      "learning_rate": 4.5859319267698025e-07,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -824.0,
+      "logps/rejected": -924.0,
+      "loss": 0.4364,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -7.71875,
+      "step": 7180
+    },
+    {
+      "epoch": 0.26762948763283767,
+      "grad_norm": 5.900896651278787,
+      "learning_rate": 4.584139646406047e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -776.0,
+      "logps/rejected": -916.0,
+      "loss": 0.4022,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.0,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -7.4375,
+      "step": 7190
+    },
+    {
+      "epoch": 0.2680017122331615,
+      "grad_norm": 5.263022048457993,
+      "learning_rate": 4.5823438473138353e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -792.0,
+      "logps/rejected": -924.0,
+      "loss": 0.421,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -7.59375,
+      "step": 7200
+    },
+    {
+      "epoch": 0.2683739368334853,
+      "grad_norm": 5.890167597830479,
+      "learning_rate": 4.5805445325250826e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -804.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4075,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.1875,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -7.875,
+      "step": 7210
+    },
+    {
+      "epoch": 0.26874616143380914,
+      "grad_norm": 8.762199756640179,
+      "learning_rate": 4.578741705077636e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -828.0,
+      "logps/rejected": -952.0,
+      "loss": 0.444,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -7.75,
+      "step": 7220
+    },
+    {
+      "epoch": 0.269118386034133,
+      "grad_norm": 5.461416990040634,
+      "learning_rate": 4.5769353680152735e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -792.0,
+      "logps/rejected": -920.0,
+      "loss": 0.4343,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -5.96875,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -7.46875,
+      "step": 7230
+    },
+    {
+      "epoch": 0.26949061063445684,
+      "grad_norm": 5.476272606008138,
+      "learning_rate": 4.575125524387701e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -792.0,
+      "logps/rejected": -908.0,
+      "loss": 0.4251,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.03125,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": -7.375,
+      "step": 7240
+    },
+    {
+      "epoch": 0.26986283523478066,
+      "grad_norm": 5.6661464976358875,
+      "learning_rate": 4.573312177250543e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -828.0,
+      "logps/rejected": -932.0,
+      "loss": 0.4147,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -7.75,
+      "step": 7250
+    },
+    {
+      "epoch": 0.2702350598351045,
+      "grad_norm": 6.188756853024002,
+      "learning_rate": 4.571495329665338e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -848.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4408,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -8.0625,
+      "step": 7260
+    },
+    {
+      "epoch": 0.27060728443542836,
+      "grad_norm": 5.603965350823858,
+      "learning_rate": 4.5696749846995365e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -856.0,
+      "logps/rejected": -956.0,
+      "loss": 0.438,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -6.59375,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -7.90625,
+      "step": 7270
+    },
+    {
+      "epoch": 0.2709795090357522,
+      "grad_norm": 5.993571491812543,
+      "learning_rate": 4.5678511454264924e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -772.0,
+      "logps/rejected": -924.0,
+      "loss": 0.4125,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -5.78125,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -7.375,
+      "step": 7280
+    },
+    {
+      "epoch": 0.271351733636076,
+      "grad_norm": 5.947469507645822,
+      "learning_rate": 4.566023814925459e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -780.0,
+      "logps/rejected": -900.0,
+      "loss": 0.4275,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -5.9375,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -7.34375,
+      "step": 7290
+    },
+    {
+      "epoch": 0.27172395823639983,
+      "grad_norm": 6.325841025077607,
+      "learning_rate": 4.5641929962815863e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -804.0,
+      "logps/rejected": -936.0,
+      "loss": 0.4413,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.28125,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -7.6875,
+      "step": 7300
+    },
+    {
+      "epoch": 0.27209618283672365,
+      "grad_norm": 5.18473163082829,
+      "learning_rate": 4.56235869258591e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -784.0,
+      "logps/rejected": -896.0,
+      "loss": 0.4344,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.03125,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -7.46875,
+      "step": 7310
+    },
+    {
+      "epoch": 0.27246840743704753,
+      "grad_norm": 5.609072281036662,
+      "learning_rate": 4.5605209069353525e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -808.0,
+      "logps/rejected": -972.0,
+      "loss": 0.3937,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.21875,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -7.875,
+      "step": 7320
+    },
+    {
+      "epoch": 0.27284063203737136,
+      "grad_norm": 5.880955789651384,
+      "learning_rate": 4.5586796424327135e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -852.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4412,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.59375,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -8.0,
+      "step": 7330
+    },
+    {
+      "epoch": 0.2732128566376952,
+      "grad_norm": 6.440537371603219,
+      "learning_rate": 4.5568349021866666e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.5,
+      "logps/chosen": -840.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4136,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -7.96875,
+      "step": 7340
+    },
+    {
+      "epoch": 0.273585081238019,
+      "grad_norm": 6.804135994492139,
+      "learning_rate": 4.554986689311754e-07,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -804.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4352,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.40625,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -7.96875,
+      "step": 7350
+    },
+    {
+      "epoch": 0.2739573058383429,
+      "grad_norm": 5.957121505319705,
+      "learning_rate": 4.553135006928379e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -824.0,
+      "logps/rejected": -996.0,
+      "loss": 0.4343,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.5,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -8.1875,
+      "step": 7360
+    },
+    {
+      "epoch": 0.2743295304386667,
+      "grad_norm": 8.04956443135275,
+      "learning_rate": 4.551279858162806e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -832.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4039,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.46875,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -8.125,
+      "step": 7370
+    },
+    {
+      "epoch": 0.2747017550389905,
+      "grad_norm": 6.3864781116711224,
+      "learning_rate": 4.549421246147149e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -828.0,
+      "logps/rejected": -948.0,
+      "loss": 0.4224,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.46875,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -7.90625,
+      "step": 7380
+    },
+    {
+      "epoch": 0.27507397963931435,
+      "grad_norm": 5.8097010692839905,
+      "learning_rate": 4.547559174019369e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -816.0,
+      "logps/rejected": -980.0,
+      "loss": 0.4075,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.0,
+      "step": 7390
+    },
+    {
+      "epoch": 0.27544620423963817,
+      "grad_norm": 5.9473524195336145,
+      "learning_rate": 4.5456936449232715e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -784.0,
+      "logps/rejected": -928.0,
+      "loss": 0.4173,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.21875,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -7.6875,
+      "step": 7400
+    },
+    {
+      "epoch": 0.27581842883996205,
+      "grad_norm": 5.9398143418043166,
+      "learning_rate": 4.543824662008496e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -812.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4105,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.1875,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -7.6875,
+      "step": 7410
+    },
+    {
+      "epoch": 0.27619065344028587,
+      "grad_norm": 6.763736215751725,
+      "learning_rate": 4.541952228430514e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -824.0,
+      "logps/rejected": -932.0,
+      "loss": 0.4308,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -7.78125,
+      "step": 7420
+    },
+    {
+      "epoch": 0.2765628780406097,
+      "grad_norm": 8.151576032792361,
+      "learning_rate": 4.540076347350623e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -840.0,
+      "logps/rejected": -968.0,
+      "loss": 0.3925,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -7.9375,
+      "step": 7430
+    },
+    {
+      "epoch": 0.2769351026409335,
+      "grad_norm": 11.288966043305832,
+      "learning_rate": 4.538197021935941e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -784.0,
+      "logps/rejected": -912.0,
+      "loss": 0.3969,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -5.78125,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -7.375,
+      "step": 7440
+    },
+    {
+      "epoch": 0.2773073272412574,
+      "grad_norm": 6.707145109970866,
+      "learning_rate": 4.5363142553594014e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -776.0,
+      "logps/rejected": -924.0,
+      "loss": 0.4053,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -5.78125,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -7.46875,
+      "step": 7450
+    },
+    {
+      "epoch": 0.2776795518415812,
+      "grad_norm": 6.905530201700849,
+      "learning_rate": 4.534428050799747e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -784.0,
+      "logps/rejected": -920.0,
+      "loss": 0.4164,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.0,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -7.5625,
+      "step": 7460
+    },
+    {
+      "epoch": 0.27805177644190504,
+      "grad_norm": 5.8253928740880365,
+      "learning_rate": 4.5325384114415254e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -780.0,
+      "logps/rejected": -916.0,
+      "loss": 0.4079,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -5.9375,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -7.59375,
+      "step": 7470
+    },
+    {
+      "epoch": 0.27842400104222886,
+      "grad_norm": 7.665848171883835,
+      "learning_rate": 4.530645340475083e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -776.0,
+      "logps/rejected": -912.0,
+      "loss": 0.4082,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -5.84375,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -7.40625,
+      "step": 7480
+    },
+    {
+      "epoch": 0.27879622564255274,
+      "grad_norm": 6.752541166954709,
+      "learning_rate": 4.52874884109656e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -748.0,
+      "logps/rejected": -912.0,
+      "loss": 0.4184,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.6875,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -7.28125,
+      "step": 7490
+    },
+    {
+      "epoch": 0.27916845024287656,
+      "grad_norm": 5.773009747462673,
+      "learning_rate": 4.5268489165078855e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -816.0,
+      "logps/rejected": -928.0,
+      "loss": 0.432,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.09375,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -7.53125,
+      "step": 7500
+    },
+    {
+      "epoch": 0.2795406748432004,
+      "grad_norm": 6.864929343234643,
+      "learning_rate": 4.5249455699167706e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -768.0,
+      "logps/rejected": -888.0,
+      "loss": 0.4167,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -5.9375,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -7.3125,
+      "step": 7510
+    },
+    {
+      "epoch": 0.2799128994435242,
+      "grad_norm": 6.632761532547937,
+      "learning_rate": 4.523038804536704e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -776.0,
+      "logps/rejected": -912.0,
+      "loss": 0.4157,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -5.9375,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -7.5,
+      "step": 7520
+    },
+    {
+      "epoch": 0.28028512404384803,
+      "grad_norm": 6.669079637823111,
+      "learning_rate": 4.521128623586947e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -812.0,
+      "logps/rejected": -948.0,
+      "loss": 0.434,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -7.78125,
+      "step": 7530
+    },
+    {
+      "epoch": 0.2806573486441719,
+      "grad_norm": 6.156049752852196,
+      "learning_rate": 4.519215030292527e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -816.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4368,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -7.8125,
+      "step": 7540
+    },
+    {
+      "epoch": 0.28102957324449573,
+      "grad_norm": 6.07035973255774,
+      "learning_rate": 4.517298027884234e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -840.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4299,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.21875,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -7.625,
+      "step": 7550
+    },
+    {
+      "epoch": 0.28140179784481956,
+      "grad_norm": 6.080414694651505,
+      "learning_rate": 4.5153776195986113e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -852.0,
+      "logps/rejected": -920.0,
+      "loss": 0.4358,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.46875,
+      "rewards/margins": 1.0546875,
+      "rewards/rejected": -7.5,
+      "step": 7560
+    },
+    {
+      "epoch": 0.2817740224451434,
+      "grad_norm": 5.605986319201334,
+      "learning_rate": 4.513453808677955e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -820.0,
+      "logps/rejected": -940.0,
+      "loss": 0.3868,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.1875,
+      "rewards/margins": 1.3671875,
+      "rewards/rejected": -7.5625,
+      "step": 7570
+    },
+    {
+      "epoch": 0.28214624704546726,
+      "grad_norm": 6.528616544992401,
+      "learning_rate": 4.5115265983703036e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -816.0,
+      "logps/rejected": -948.0,
+      "loss": 0.4088,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.46875,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -8.0,
+      "step": 7580
+    },
+    {
+      "epoch": 0.2825184716457911,
+      "grad_norm": 5.293613770865945,
+      "learning_rate": 4.5095959919294366e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -848.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4125,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.6875,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -8.125,
+      "step": 7590
+    },
+    {
+      "epoch": 0.2828906962461149,
+      "grad_norm": 8.308599384829984,
+      "learning_rate": 4.5076619926148673e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.4218,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -8.3125,
+      "step": 7600
+    },
+    {
+      "epoch": 0.2832629208464387,
+      "grad_norm": 6.89426563654055,
+      "learning_rate": 4.5057246036918357e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -856.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4079,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -8.1875,
+      "step": 7610
+    },
+    {
+      "epoch": 0.28363514544676255,
+      "grad_norm": 5.757382364852515,
+      "learning_rate": 4.503783828431306e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -820.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4148,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -7.84375,
+      "step": 7620
+    },
+    {
+      "epoch": 0.2840073700470864,
+      "grad_norm": 8.796876620057319,
+      "learning_rate": 4.50183967010996e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -816.0,
+      "logps/rejected": -936.0,
+      "loss": 0.4164,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -7.75,
+      "step": 7630
+    },
+    {
+      "epoch": 0.28437959464741025,
+      "grad_norm": 5.409682348117861,
+      "learning_rate": 4.49989213201019e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -788.0,
+      "logps/rejected": -912.0,
+      "loss": 0.4236,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.0625,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -7.53125,
+      "step": 7640
+    },
+    {
+      "epoch": 0.2847518192477341,
+      "grad_norm": 5.522829230003005,
+      "learning_rate": 4.497941217420095e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -816.0,
+      "logps/rejected": -936.0,
+      "loss": 0.4011,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.1875,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -7.71875,
+      "step": 7650
+    },
+    {
+      "epoch": 0.2851240438480579,
+      "grad_norm": 6.128079921566439,
+      "learning_rate": 4.495986929633476e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -816.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4334,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.40625,
+      "rewards/margins": 1.3125,
+      "rewards/rejected": -7.71875,
+      "step": 7660
+    },
+    {
+      "epoch": 0.2854962684483818,
+      "grad_norm": 7.737598587677687,
+      "learning_rate": 4.494029271949826e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -844.0,
+      "logps/rejected": -968.0,
+      "loss": 0.4078,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.625,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -8.0625,
+      "step": 7670
+    },
+    {
+      "epoch": 0.2858684930487056,
+      "grad_norm": 6.468631454744863,
+      "learning_rate": 4.492068247674331e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -848.0,
+      "logps/rejected": -980.0,
+      "loss": 0.4589,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.71875,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.3125,
+      "step": 7680
+    },
+    {
+      "epoch": 0.2862407176490294,
+      "grad_norm": 7.510640649966863,
+      "learning_rate": 4.490103860117858e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -816.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4346,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -7.8125,
+      "step": 7690
+    },
+    {
+      "epoch": 0.28661294224935324,
+      "grad_norm": 5.3436987473560125,
+      "learning_rate": 4.4881361125969546e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -812.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4136,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -6.21875,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -7.9375,
+      "step": 7700
+    },
+    {
+      "epoch": 0.2869851668496771,
+      "grad_norm": 5.84593282580263,
+      "learning_rate": 4.48616500843384e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -848.0,
+      "logps/rejected": -960.0,
+      "loss": 0.4235,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.5625,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -7.9375,
+      "step": 7710
+    },
+    {
+      "epoch": 0.28735739145000094,
+      "grad_norm": 6.060442318478308,
+      "learning_rate": 4.4841905509564e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -4.03125,
+      "logps/chosen": -820.0,
+      "logps/rejected": -928.0,
+      "loss": 0.3898,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -7.71875,
+      "step": 7720
+    },
+    {
+      "epoch": 0.28772961605032477,
+      "grad_norm": 6.194875526344922,
+      "learning_rate": 4.4822127434981845e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -824.0,
+      "logps/rejected": -972.0,
+      "loss": 0.4065,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.46875,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -8.0625,
+      "step": 7730
+    },
+    {
+      "epoch": 0.2881018406506486,
+      "grad_norm": 6.587423826797523,
+      "learning_rate": 4.4802315893983957e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -816.0,
+      "logps/rejected": -960.0,
+      "loss": 0.4107,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.28125,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -7.90625,
+      "step": 7740
+    },
+    {
+      "epoch": 0.2884740652509724,
+      "grad_norm": 5.579966834077149,
+      "learning_rate": 4.4782470920018875e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -828.0,
+      "logps/rejected": -968.0,
+      "loss": 0.4231,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.5,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -8.125,
+      "step": 7750
+    },
+    {
+      "epoch": 0.2888462898512963,
+      "grad_norm": 5.578942771970089,
+      "learning_rate": 4.4762592546591617e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -844.0,
+      "logps/rejected": -976.0,
+      "loss": 0.4297,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.71875,
+      "rewards/margins": 1.375,
+      "rewards/rejected": -8.0625,
+      "step": 7760
+    },
+    {
+      "epoch": 0.2892185144516201,
+      "grad_norm": 5.627177009698344,
+      "learning_rate": 4.4742680807263524e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -816.0,
+      "logps/rejected": -972.0,
+      "loss": 0.426,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -7.96875,
+      "step": 7770
+    },
+    {
+      "epoch": 0.28959073905194394,
+      "grad_norm": 6.211708773823986,
+      "learning_rate": 4.4722735735652327e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -832.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4449,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -8.0,
+      "step": 7780
+    },
+    {
+      "epoch": 0.28996296365226776,
+      "grad_norm": 4.965561637662461,
+      "learning_rate": 4.4702757365432007e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.875,
+      "logps/chosen": -816.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4155,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -7.625,
+      "step": 7790
+    },
+    {
+      "epoch": 0.29033518825259164,
+      "grad_norm": 5.633981216434436,
+      "learning_rate": 4.468274573033278e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -832.0,
+      "logps/rejected": -940.0,
+      "loss": 0.4245,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 1.359375,
+      "rewards/rejected": -7.71875,
+      "step": 7800
+    },
+    {
+      "epoch": 0.29070741285291546,
+      "grad_norm": 5.812551208964231,
+      "learning_rate": 4.4662700864141e-07,
+      "logits/chosen": -4.09375,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -836.0,
+      "logps/rejected": -952.0,
+      "loss": 0.418,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -7.6875,
+      "step": 7810
+    },
+    {
+      "epoch": 0.2910796374532393,
+      "grad_norm": 6.823075340921146,
+      "learning_rate": 4.4642622800699155e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -832.0,
+      "logps/rejected": -964.0,
+      "loss": 0.418,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -7.875,
+      "step": 7820
+    },
+    {
+      "epoch": 0.2914518620535631,
+      "grad_norm": 5.934821118318428,
+      "learning_rate": 4.4622511573905754e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -808.0,
+      "logps/rejected": -940.0,
+      "loss": 0.4082,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.28125,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -7.78125,
+      "step": 7830
+    },
+    {
+      "epoch": 0.291824086653887,
+      "grad_norm": 5.697784836555151,
+      "learning_rate": 4.460236721771531e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -820.0,
+      "logps/rejected": -932.0,
+      "loss": 0.4145,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -7.71875,
+      "step": 7840
+    },
+    {
+      "epoch": 0.2921963112542108,
+      "grad_norm": 8.194828445942946,
+      "learning_rate": 4.458218976613828e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -824.0,
+      "logps/rejected": -936.0,
+      "loss": 0.4249,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -7.71875,
+      "step": 7850
+    },
+    {
+      "epoch": 0.29256853585453463,
+      "grad_norm": 6.101392884473911,
+      "learning_rate": 4.456197925324098e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -824.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4257,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.46875,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -7.84375,
+      "step": 7860
+    },
+    {
+      "epoch": 0.29294076045485845,
+      "grad_norm": 5.775207827332482,
+      "learning_rate": 4.4541735713145546e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -816.0,
+      "logps/rejected": -960.0,
+      "loss": 0.4123,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.46875,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -8.0625,
+      "step": 7870
+    },
+    {
+      "epoch": 0.2933129850551823,
+      "grad_norm": 5.529179150621515,
+      "learning_rate": 4.4521459180029884e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -824.0,
+      "logps/rejected": -952.0,
+      "loss": 0.4271,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.40625,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -7.8125,
+      "step": 7880
+    },
+    {
+      "epoch": 0.29368520965550615,
+      "grad_norm": 5.629698643352411,
+      "learning_rate": 4.45011496881276e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -824.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4165,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.40625,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -7.90625,
+      "step": 7890
+    },
+    {
+      "epoch": 0.29405743425583,
+      "grad_norm": 5.858186522980214,
+      "learning_rate": 4.4480807271727954e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -840.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4144,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.40625,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -7.9375,
+      "step": 7900
+    },
+    {
+      "epoch": 0.2944296588561538,
+      "grad_norm": 6.51605195710432,
+      "learning_rate": 4.446043196517577e-07,
+      "logits/chosen": -4.15625,
+      "logits/rejected": -3.875,
+      "logps/chosen": -832.0,
+      "logps/rejected": -940.0,
+      "loss": 0.4436,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -6.625,
+      "rewards/margins": 1.1640625,
+      "rewards/rejected": -7.8125,
+      "step": 7910
+    },
+    {
+      "epoch": 0.2948018834564776,
+      "grad_norm": 7.191073113766226,
+      "learning_rate": 4.444002380287143e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -820.0,
+      "logps/rejected": -948.0,
+      "loss": 0.4176,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -7.84375,
+      "step": 7920
+    },
+    {
+      "epoch": 0.2951741080568015,
+      "grad_norm": 5.595229117567896,
+      "learning_rate": 4.441958281927075e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -816.0,
+      "logps/rejected": -936.0,
+      "loss": 0.4179,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -7.59375,
+      "step": 7930
+    },
+    {
+      "epoch": 0.2955463326571253,
+      "grad_norm": 6.527830177486686,
+      "learning_rate": 4.4399109048885006e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -856.0,
+      "logps/rejected": -992.0,
+      "loss": 0.4268,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -8.3125,
+      "step": 7940
+    },
+    {
+      "epoch": 0.29591855725744914,
+      "grad_norm": 6.079554429820177,
+      "learning_rate": 4.437860252628081e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -852.0,
+      "logps/rejected": -972.0,
+      "loss": 0.3929,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.375,
+      "rewards/rejected": -8.125,
+      "step": 7950
+    },
+    {
+      "epoch": 0.29629078185777297,
+      "grad_norm": 6.582564336121776,
+      "learning_rate": 4.435806328608004e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -844.0,
+      "logps/rejected": -992.0,
+      "loss": 0.3996,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -8.125,
+      "step": 7960
+    },
+    {
+      "epoch": 0.2966630064580968,
+      "grad_norm": 6.506949889046725,
+      "learning_rate": 4.4337491362959854e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.875,
+      "logps/chosen": -856.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.4074,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -8.5,
+      "step": 7970
+    },
+    {
+      "epoch": 0.29703523105842067,
+      "grad_norm": 7.100064140632613,
+      "learning_rate": 4.4316886791652584e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -872.0,
+      "logps/rejected": -988.0,
+      "loss": 0.415,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.234375,
+      "rewards/rejected": -8.125,
+      "step": 7980
+    },
+    {
+      "epoch": 0.2974074556587445,
+      "grad_norm": 7.183074534887785,
+      "learning_rate": 4.429624960694566e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4212,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -8.375,
+      "step": 7990
+    },
+    {
+      "epoch": 0.2977796802590683,
+      "grad_norm": 7.352505423981211,
+      "learning_rate": 4.42755798436816e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -844.0,
+      "logps/rejected": -976.0,
+      "loss": 0.4057,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -8.1875,
+      "step": 8000
+    },
+    {
+      "epoch": 0.29815190485939214,
+      "grad_norm": 5.877784277717734,
+      "learning_rate": 4.42548775367579e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -820.0,
+      "logps/rejected": -960.0,
+      "loss": 0.4085,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -8.0,
+      "step": 8010
+    },
+    {
+      "epoch": 0.298524129459716,
+      "grad_norm": 7.025513314364941,
+      "learning_rate": 4.4234142721127026e-07,
+      "logits/chosen": -4.1875,
+      "logits/rejected": -4.03125,
+      "logps/chosen": -860.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.4162,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.3125,
+      "step": 8020
+    },
+    {
+      "epoch": 0.29889635406003984,
+      "grad_norm": 5.092026385878604,
+      "learning_rate": 4.4213375431796316e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -832.0,
+      "logps/rejected": -968.0,
+      "loss": 0.4048,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.6875,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.1875,
+      "step": 8030
+    },
+    {
+      "epoch": 0.29926857866036366,
+      "grad_norm": 5.913584346369995,
+      "learning_rate": 4.419257570382793e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -840.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.4273,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.71875,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -8.4375,
+      "step": 8040
+    },
+    {
+      "epoch": 0.2996408032606875,
+      "grad_norm": 6.406795204967781,
+      "learning_rate": 4.4171743572338813e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.921875,
+      "logps/chosen": -860.0,
+      "logps/rejected": -972.0,
+      "loss": 0.4598,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.71875,
+      "rewards/margins": 1.28125,
+      "rewards/rejected": -8.0,
+      "step": 8050
+    },
+    {
+      "epoch": 0.30001302786101136,
+      "grad_norm": 5.856699305881422,
+      "learning_rate": 4.4150879072500604e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -840.0,
+      "logps/rejected": -952.0,
+      "loss": 0.3982,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.5,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -7.84375,
+      "step": 8060
+    },
+    {
+      "epoch": 0.3003852524613352,
+      "grad_norm": 6.001256951069163,
+      "learning_rate": 4.4129982239539594e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.9375,
+      "logps/chosen": -868.0,
+      "logps/rejected": -992.0,
+      "loss": 0.4172,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -8.25,
+      "step": 8070
+    },
+    {
+      "epoch": 0.300757477061659,
+      "grad_norm": 5.66074759558905,
+      "learning_rate": 4.410905310873665e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -824.0,
+      "logps/rejected": -940.0,
+      "loss": 0.4283,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -7.875,
+      "step": 8080
+    },
+    {
+      "epoch": 0.30112970166198283,
+      "grad_norm": 5.482429194736557,
+      "learning_rate": 4.40880917154272e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4026,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -8.3125,
+      "step": 8090
+    },
+    {
+      "epoch": 0.30150192626230665,
+      "grad_norm": 7.2184703453065655,
+      "learning_rate": 4.4067098095001113e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -840.0,
+      "logps/rejected": -972.0,
+      "loss": 0.4157,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.6875,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -8.1875,
+      "step": 8100
+    },
+    {
+      "epoch": 0.30187415086263053,
+      "grad_norm": 6.806034531524511,
+      "learning_rate": 4.4046072282902687e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -800.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4116,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.1875,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -7.96875,
+      "step": 8110
+    },
+    {
+      "epoch": 0.30224637546295435,
+      "grad_norm": 6.617690856299018,
+      "learning_rate": 4.402501431463055e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -840.0,
+      "logps/rejected": -936.0,
+      "loss": 0.4321,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.40625,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -7.71875,
+      "step": 8120
+    },
+    {
+      "epoch": 0.3026186000632782,
+      "grad_norm": 7.831250227203586,
+      "learning_rate": 4.4003924225737643e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -764.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4109,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.03125,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -7.84375,
+      "step": 8130
+    },
+    {
+      "epoch": 0.302990824663602,
+      "grad_norm": 5.535353433361087,
+      "learning_rate": 4.3982802051831127e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -832.0,
+      "logps/rejected": -960.0,
+      "loss": 0.3959,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.40625,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -7.9375,
+      "step": 8140
+    },
+    {
+      "epoch": 0.3033630492639259,
+      "grad_norm": 6.2529635387847895,
+      "learning_rate": 4.396164782857232e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4155,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.8125,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.375,
+      "step": 8150
+    },
+    {
+      "epoch": 0.3037352738642497,
+      "grad_norm": 5.425058988784764,
+      "learning_rate": 4.3940461591676683e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.4025,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -8.25,
+      "step": 8160
+    },
+    {
+      "epoch": 0.3041074984645735,
+      "grad_norm": 6.526733064420788,
+      "learning_rate": 4.391924337691368e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -836.0,
+      "logps/rejected": -972.0,
+      "loss": 0.3942,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.46875,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -8.0625,
+      "step": 8170
+    },
+    {
+      "epoch": 0.30447972306489735,
+      "grad_norm": 6.922891766106371,
+      "learning_rate": 4.3897993220106825e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -844.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.4086,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.625,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -8.3125,
+      "step": 8180
+    },
+    {
+      "epoch": 0.30485194766522117,
+      "grad_norm": 7.015051177524256,
+      "learning_rate": 4.3876711157133506e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -812.0,
+      "logps/rejected": -940.0,
+      "loss": 0.4089,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -7.90625,
+      "step": 8190
+    },
+    {
+      "epoch": 0.30522417226554505,
+      "grad_norm": 5.359070975311743,
+      "learning_rate": 4.385539722392501e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -840.0,
+      "logps/rejected": -980.0,
+      "loss": 0.4023,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.5,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -8.0625,
+      "step": 8200
+    },
+    {
+      "epoch": 0.30559639686586887,
+      "grad_norm": 6.1864107604357415,
+      "learning_rate": 4.3834051456466414e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -824.0,
+      "logps/rejected": -952.0,
+      "loss": 0.4138,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -7.8125,
+      "step": 8210
+    },
+    {
+      "epoch": 0.3059686214661927,
+      "grad_norm": 6.6538688532412404,
+      "learning_rate": 4.381267389079656e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -792.0,
+      "logps/rejected": -940.0,
+      "loss": 0.4024,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -8.0,
+      "step": 8220
+    },
+    {
+      "epoch": 0.3063408460665165,
+      "grad_norm": 7.855420982081355,
+      "learning_rate": 4.379126456300796e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -804.0,
+      "logps/rejected": -952.0,
+      "loss": 0.408,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -7.875,
+      "step": 8230
+    },
+    {
+      "epoch": 0.3067130706668404,
+      "grad_norm": 5.520161299022761,
+      "learning_rate": 4.3769823509246753e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -840.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4346,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.59375,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -8.0625,
+      "step": 8240
+    },
+    {
+      "epoch": 0.3070852952671642,
+      "grad_norm": 5.783006987164638,
+      "learning_rate": 4.374835076571265e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -828.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4127,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -7.75,
+      "step": 8250
+    },
+    {
+      "epoch": 0.30745751986748804,
+      "grad_norm": 7.194195524701205,
+      "learning_rate": 4.3726846368658874e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -820.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4255,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -7.78125,
+      "step": 8260
+    },
+    {
+      "epoch": 0.30782974446781186,
+      "grad_norm": 8.487284581586474,
+      "learning_rate": 4.370531035439206e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -776.0,
+      "logps/rejected": -920.0,
+      "loss": 0.4345,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.0,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -7.59375,
+      "step": 8270
+    },
+    {
+      "epoch": 0.30820196906813574,
+      "grad_norm": 5.503865115363084,
+      "learning_rate": 4.3683742759272255e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -808.0,
+      "logps/rejected": -948.0,
+      "loss": 0.4139,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -7.84375,
+      "step": 8280
+    },
+    {
+      "epoch": 0.30857419366845956,
+      "grad_norm": 5.4597357470381995,
+      "learning_rate": 4.36621436197128e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -860.0,
+      "logps/rejected": -992.0,
+      "loss": 0.4045,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -8.25,
+      "step": 8290
+    },
+    {
+      "epoch": 0.3089464182687834,
+      "grad_norm": 6.360812426641355,
+      "learning_rate": 4.364051297218031e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4133,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -8.5625,
+      "step": 8300
+    },
+    {
+      "epoch": 0.3093186428691072,
+      "grad_norm": 7.181286945346029,
+      "learning_rate": 4.361885085319459e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -816.0,
+      "logps/rejected": -948.0,
+      "loss": 0.4352,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -7.90625,
+      "step": 8310
+    },
+    {
+      "epoch": 0.30969086746943103,
+      "grad_norm": 5.758526971226857,
+      "learning_rate": 4.359715729932858e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -812.0,
+      "logps/rejected": -924.0,
+      "loss": 0.4186,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -7.6875,
+      "step": 8320
+    },
+    {
+      "epoch": 0.3100630920697549,
+      "grad_norm": 7.239872111687638,
+      "learning_rate": 4.357543234720829e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -820.0,
+      "logps/rejected": -948.0,
+      "loss": 0.426,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.265625,
+      "rewards/rejected": -7.625,
+      "step": 8330
+    },
+    {
+      "epoch": 0.31043531667007873,
+      "grad_norm": 6.238541324845665,
+      "learning_rate": 4.355367603351275e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -828.0,
+      "logps/rejected": -980.0,
+      "loss": 0.4064,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.1875,
+      "step": 8340
+    },
+    {
+      "epoch": 0.31080754127040255,
+      "grad_norm": 5.467215078581045,
+      "learning_rate": 4.353188839497393e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.984375,
+      "logps/chosen": -852.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4006,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -8.125,
+      "step": 8350
+    },
+    {
+      "epoch": 0.3111797658707264,
+      "grad_norm": 7.133710205092668,
+      "learning_rate": 4.3510069468376687e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -844.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4061,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.6875,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -8.25,
+      "step": 8360
+    },
+    {
+      "epoch": 0.31155199047105026,
+      "grad_norm": 6.78111846598109,
+      "learning_rate": 4.34882192905587e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -860.0,
+      "logps/rejected": -980.0,
+      "loss": 0.3985,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -8.3125,
+      "step": 8370
+    },
+    {
+      "epoch": 0.3119242150713741,
+      "grad_norm": 6.043776866561569,
+      "learning_rate": 4.3466337898410435e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.9375,
+      "logps/chosen": -872.0,
+      "logps/rejected": -996.0,
+      "loss": 0.4024,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.8125,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -8.3125,
+      "step": 8380
+    },
+    {
+      "epoch": 0.3122964396716979,
+      "grad_norm": 6.778031540663945,
+      "learning_rate": 4.3444425328875013e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4169,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.921875,
+      "rewards/rejected": -8.75,
+      "step": 8390
+    },
+    {
+      "epoch": 0.3126686642720217,
+      "grad_norm": 6.128481494621989,
+      "learning_rate": 4.3422481618948236e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.9375,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.4053,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -8.4375,
+      "step": 8400
+    },
+    {
+      "epoch": 0.31304088887234555,
+      "grad_norm": 6.150966985232953,
+      "learning_rate": 4.340050680567846e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -856.0,
+      "logps/rejected": -996.0,
+      "loss": 0.424,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.6875,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -8.375,
+      "step": 8410
+    },
+    {
+      "epoch": 0.3134131134726694,
+      "grad_norm": 7.923365210705741,
+      "learning_rate": 4.337850092616656e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -828.0,
+      "logps/rejected": -964.0,
+      "loss": 0.393,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -6.40625,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -8.125,
+      "step": 8420
+    },
+    {
+      "epoch": 0.31378533807299325,
+      "grad_norm": 5.894568629562948,
+      "learning_rate": 4.3356464017565856e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -888.0,
+      "logps/rejected": -996.0,
+      "loss": 0.4199,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -8.3125,
+      "step": 8430
+    },
+    {
+      "epoch": 0.31415756267331707,
+      "grad_norm": 6.13674719047374,
+      "learning_rate": 4.333439611708206e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -848.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.393,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -8.5,
+      "step": 8440
+    },
+    {
+      "epoch": 0.3145297872736409,
+      "grad_norm": 6.436436029161063,
+      "learning_rate": 4.3312297261973206e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -852.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4053,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.59375,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -8.0625,
+      "step": 8450
+    },
+    {
+      "epoch": 0.31490201187396477,
+      "grad_norm": 5.632353398865315,
+      "learning_rate": 4.32901674895496e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -804.0,
+      "logps/rejected": -932.0,
+      "loss": 0.4105,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -7.78125,
+      "step": 8460
+    },
+    {
+      "epoch": 0.3152742364742886,
+      "grad_norm": 6.818178988917069,
+      "learning_rate": 4.326800683717373e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -848.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.4177,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -8.375,
+      "step": 8470
+    },
+    {
+      "epoch": 0.3156464610746124,
+      "grad_norm": 5.785602357971954,
+      "learning_rate": 4.324581534226023e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.4283,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -8.375,
+      "step": 8480
+    },
+    {
+      "epoch": 0.31601868567493624,
+      "grad_norm": 6.860288559968709,
+      "learning_rate": 4.32235930422758e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.921875,
+      "logps/chosen": -872.0,
+      "logps/rejected": -984.0,
+      "loss": 0.3962,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.265625,
+      "rewards/rejected": -8.0625,
+      "step": 8490
+    },
+    {
+      "epoch": 0.3163909102752601,
+      "grad_norm": 6.6580316008849705,
+      "learning_rate": 4.3201339974739165e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -852.0,
+      "logps/rejected": -968.0,
+      "loss": 0.4244,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.625,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -8.0625,
+      "step": 8500
+    },
+    {
+      "epoch": 0.31676313487558394,
+      "grad_norm": 7.4233344738729485,
+      "learning_rate": 4.3179056177220976e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -832.0,
+      "logps/rejected": -968.0,
+      "loss": 0.4262,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.5625,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -8.0625,
+      "step": 8510
+    },
+    {
+      "epoch": 0.31713535947590776,
+      "grad_norm": 6.83893181284603,
+      "learning_rate": 4.3156741687343776e-07,
+      "logits/chosen": -4.09375,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -828.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4418,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -7.84375,
+      "step": 8520
+    },
+    {
+      "epoch": 0.3175075840762316,
+      "grad_norm": 6.014146334675527,
+      "learning_rate": 4.313439654278194e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.953125,
+      "logps/chosen": -848.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4105,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.5,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -7.84375,
+      "step": 8530
+    },
+    {
+      "epoch": 0.3178798086765554,
+      "grad_norm": 5.359151664689992,
+      "learning_rate": 4.311202078126156e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -868.0,
+      "logps/rejected": -972.0,
+      "loss": 0.4162,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.5625,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -7.96875,
+      "step": 8540
+    },
+    {
+      "epoch": 0.3182520332768793,
+      "grad_norm": 5.012945828943925,
+      "learning_rate": 4.308961444056046e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -828.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4209,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -7.8125,
+      "step": 8550
+    },
+    {
+      "epoch": 0.3186242578772031,
+      "grad_norm": 7.952124582877079,
+      "learning_rate": 4.306717755850808e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -828.0,
+      "logps/rejected": -968.0,
+      "loss": 0.4054,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -7.9375,
+      "step": 8560
+    },
+    {
+      "epoch": 0.31899648247752693,
+      "grad_norm": 5.187751163398861,
+      "learning_rate": 4.304471017298542e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -860.0,
+      "logps/rejected": -968.0,
+      "loss": 0.4079,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -8.0625,
+      "step": 8570
+    },
+    {
+      "epoch": 0.31936870707785076,
+      "grad_norm": 5.961024049387112,
+      "learning_rate": 4.302221232192497e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -856.0,
+      "logps/rejected": -992.0,
+      "loss": 0.4286,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -8.1875,
+      "step": 8580
+    },
+    {
+      "epoch": 0.31974093167817463,
+      "grad_norm": 5.7088461056840245,
+      "learning_rate": 4.2999684043310667e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -836.0,
+      "logps/rejected": -952.0,
+      "loss": 0.3969,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -7.875,
+      "step": 8590
+    },
+    {
+      "epoch": 0.32011315627849846,
+      "grad_norm": 5.619002792846216,
+      "learning_rate": 4.297712537517784e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -836.0,
+      "logps/rejected": -988.0,
+      "loss": 0.3823,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -6.5,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.0,
+      "step": 8600
+    },
+    {
+      "epoch": 0.3204853808788223,
+      "grad_norm": 8.099589563398085,
+      "learning_rate": 4.295453635561308e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.4243,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.625,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -8.4375,
+      "step": 8610
+    },
+    {
+      "epoch": 0.3208576054791461,
+      "grad_norm": 4.926184751372829,
+      "learning_rate": 4.293191702275426e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -848.0,
+      "logps/rejected": -976.0,
+      "loss": 0.3989,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -8.125,
+      "step": 8620
+    },
+    {
+      "epoch": 0.3212298300794699,
+      "grad_norm": 6.726433965744231,
+      "learning_rate": 4.290926741479043e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -824.0,
+      "logps/rejected": -940.0,
+      "loss": 0.4241,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -7.75,
+      "step": 8630
+    },
+    {
+      "epoch": 0.3216020546797938,
+      "grad_norm": 6.553744668409475,
+      "learning_rate": 4.288658756996172e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -836.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4081,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.46875,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -8.0625,
+      "step": 8640
+    },
+    {
+      "epoch": 0.3219742792801176,
+      "grad_norm": 6.188877117500308,
+      "learning_rate": 4.2863877526559344e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -856.0,
+      "logps/rejected": -980.0,
+      "loss": 0.4081,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.59375,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -8.0625,
+      "step": 8650
+    },
+    {
+      "epoch": 0.32234650388044145,
+      "grad_norm": 6.087998287649099,
+      "learning_rate": 4.2841137322925493e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -832.0,
+      "logps/rejected": -980.0,
+      "loss": 0.3958,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.25,
+      "step": 8660
+    },
+    {
+      "epoch": 0.32271872848076527,
+      "grad_norm": 6.23375850844606,
+      "learning_rate": 4.281836699745327e-07,
+      "logits/chosen": -4.125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -860.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.4059,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.8125,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.4375,
+      "step": 8670
+    },
+    {
+      "epoch": 0.32309095308108915,
+      "grad_norm": 6.648691687792136,
+      "learning_rate": 4.279556658858665e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.4363,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.25,
+      "step": 8680
+    },
+    {
+      "epoch": 0.32346317768141297,
+      "grad_norm": 5.860269394976744,
+      "learning_rate": 4.2772736134820385e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.4016,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.71875,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -8.3125,
+      "step": 8690
+    },
+    {
+      "epoch": 0.3238354022817368,
+      "grad_norm": 5.9892684094158035,
+      "learning_rate": 4.2749875674699954e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.875,
+      "logps/chosen": -860.0,
+      "logps/rejected": -992.0,
+      "loss": 0.4145,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.5625,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -8.0625,
+      "step": 8700
+    },
+    {
+      "epoch": 0.3242076268820606,
+      "grad_norm": 6.1944503642342195,
+      "learning_rate": 4.2726985246821507e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -820.0,
+      "logps/rejected": -984.0,
+      "loss": 0.3929,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -8.0,
+      "step": 8710
+    },
+    {
+      "epoch": 0.3245798514823845,
+      "grad_norm": 6.44235967655113,
+      "learning_rate": 4.270406488983177e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -832.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4196,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -8.25,
+      "step": 8720
+    },
+    {
+      "epoch": 0.3249520760827083,
+      "grad_norm": 7.6313692046854555,
+      "learning_rate": 4.268111464242803e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -840.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4136,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.46875,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -8.0625,
+      "step": 8730
+    },
+    {
+      "epoch": 0.32532430068303214,
+      "grad_norm": 7.331661181727046,
+      "learning_rate": 4.2658134543358e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -844.0,
+      "logps/rejected": -988.0,
+      "loss": 0.388,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.625,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -8.125,
+      "step": 8740
+    },
+    {
+      "epoch": 0.32569652528335596,
+      "grad_norm": 6.390430330476552,
+      "learning_rate": 4.2635124631419827e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -844.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4068,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -8.0625,
+      "step": 8750
+    },
+    {
+      "epoch": 0.3260687498836798,
+      "grad_norm": 6.260206823127818,
+      "learning_rate": 4.261208494546198e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -824.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4251,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.5625,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -8.0625,
+      "step": 8760
+    },
+    {
+      "epoch": 0.32644097448400367,
+      "grad_norm": 5.761717086475829,
+      "learning_rate": 4.2589015524383187e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -816.0,
+      "logps/rejected": -948.0,
+      "loss": 0.4155,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.28125,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -7.9375,
+      "step": 8770
+    },
+    {
+      "epoch": 0.3268131990843275,
+      "grad_norm": 6.3090046654095575,
+      "learning_rate": 4.2565916407132393e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -860.0,
+      "logps/rejected": -960.0,
+      "loss": 0.4102,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.59375,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -7.90625,
+      "step": 8780
+    },
+    {
+      "epoch": 0.3271854236846513,
+      "grad_norm": 6.223308737357057,
+      "learning_rate": 4.254278763270867e-07,
+      "logits/chosen": -4.21875,
+      "logits/rejected": -3.9375,
+      "logps/chosen": -844.0,
+      "logps/rejected": -976.0,
+      "loss": 0.4323,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -8.0625,
+      "step": 8790
+    },
+    {
+      "epoch": 0.32755764828497513,
+      "grad_norm": 7.208264103283307,
+      "learning_rate": 4.251962924016117e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -844.0,
+      "logps/rejected": -980.0,
+      "loss": 0.4141,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -8.0625,
+      "step": 8800
+    },
+    {
+      "epoch": 0.327929872885299,
+      "grad_norm": 8.576776788145699,
+      "learning_rate": 4.2496441268589047e-07,
+      "logits/chosen": -4.125,
+      "logits/rejected": -3.984375,
+      "logps/chosen": -864.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4005,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.125,
+      "step": 8810
+    },
+    {
+      "epoch": 0.32830209748562283,
+      "grad_norm": 6.5268376884150685,
+      "learning_rate": 4.2473223757141386e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.9375,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3926,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -8.625,
+      "step": 8820
+    },
+    {
+      "epoch": 0.32867432208594666,
+      "grad_norm": 6.870042330887718,
+      "learning_rate": 4.2449976745017157e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -860.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.4096,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.625,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -8.1875,
+      "step": 8830
+    },
+    {
+      "epoch": 0.3290465466862705,
+      "grad_norm": 6.745719900706627,
+      "learning_rate": 4.2426700271465134e-07,
+      "logits/chosen": -4.09375,
+      "logits/rejected": -4.0625,
+      "logps/chosen": -840.0,
+      "logps/rejected": -972.0,
+      "loss": 0.3994,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.0,
+      "step": 8840
+    },
+    {
+      "epoch": 0.32941877128659436,
+      "grad_norm": 6.902855629530909,
+      "learning_rate": 4.240339437578383e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -804.0,
+      "logps/rejected": -928.0,
+      "loss": 0.4104,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.09375,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -7.71875,
+      "step": 8850
+    },
+    {
+      "epoch": 0.3297909958869182,
+      "grad_norm": 5.8133105508535,
+      "learning_rate": 4.238005909732144e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -800.0,
+      "logps/rejected": -956.0,
+      "loss": 0.3944,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.0625,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -7.9375,
+      "step": 8860
+    },
+    {
+      "epoch": 0.330163220487242,
+      "grad_norm": 5.924084881415485,
+      "learning_rate": 4.235669447547574e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -816.0,
+      "logps/rejected": -972.0,
+      "loss": 0.413,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.40625,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -8.0625,
+      "step": 8870
+    },
+    {
+      "epoch": 0.3305354450875658,
+      "grad_norm": 6.033984333254001,
+      "learning_rate": 4.2333300549694085e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -828.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4177,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.6875,
+      "rewards/margins": 1.359375,
+      "rewards/rejected": -8.0625,
+      "step": 8880
+    },
+    {
+      "epoch": 0.33090766968788965,
+      "grad_norm": 5.071695094721961,
+      "learning_rate": 4.2309877359473277e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -796.0,
+      "logps/rejected": -900.0,
+      "loss": 0.4217,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -7.625,
+      "step": 8890
+    },
+    {
+      "epoch": 0.33127989428821353,
+      "grad_norm": 6.004609156857372,
+      "learning_rate": 4.2286424944359547e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.953125,
+      "logps/chosen": -820.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4247,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.28125,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -7.875,
+      "step": 8900
+    },
+    {
+      "epoch": 0.33165211888853735,
+      "grad_norm": 6.8374250518691415,
+      "learning_rate": 4.2262943343948445e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -852.0,
+      "logps/rejected": -976.0,
+      "loss": 0.4091,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.1953125,
+      "rewards/rejected": -7.9375,
+      "step": 8910
+    },
+    {
+      "epoch": 0.3320243434888612,
+      "grad_norm": 5.666561979657318,
+      "learning_rate": 4.223943259788481e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -840.0,
+      "logps/rejected": -988.0,
+      "loss": 0.3929,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.5,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -8.0625,
+      "step": 8920
+    },
+    {
+      "epoch": 0.332396568089185,
+      "grad_norm": 5.56437214907557,
+      "learning_rate": 4.22158927458627e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -840.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4217,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.5625,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -8.125,
+      "step": 8930
+    },
+    {
+      "epoch": 0.3327687926895089,
+      "grad_norm": 5.9009263619579215,
+      "learning_rate": 4.219232382762528e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -828.0,
+      "logps/rejected": -976.0,
+      "loss": 0.4046,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.46875,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -8.0,
+      "step": 8940
+    },
+    {
+      "epoch": 0.3331410172898327,
+      "grad_norm": 6.088222804779362,
+      "learning_rate": 4.2168725882964825e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -780.0,
+      "logps/rejected": -924.0,
+      "loss": 0.4241,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.09375,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -7.59375,
+      "step": 8950
+    },
+    {
+      "epoch": 0.3335132418901565,
+      "grad_norm": 5.890491853155072,
+      "learning_rate": 4.2145098951722584e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -824.0,
+      "logps/rejected": -984.0,
+      "loss": 0.428,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.40625,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -8.0625,
+      "step": 8960
+    },
+    {
+      "epoch": 0.33388546649048034,
+      "grad_norm": 5.790425010171351,
+      "learning_rate": 4.2121443073788775e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -816.0,
+      "logps/rejected": -980.0,
+      "loss": 0.3932,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.40625,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -8.0625,
+      "step": 8970
+    },
+    {
+      "epoch": 0.33425769109080417,
+      "grad_norm": 6.041405800344959,
+      "learning_rate": 4.209775828910247e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -828.0,
+      "logps/rejected": -980.0,
+      "loss": 0.4096,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -8.0,
+      "step": 8980
+    },
+    {
+      "epoch": 0.33462991569112804,
+      "grad_norm": 5.852364410055907,
+      "learning_rate": 4.207404463765154e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.96875,
+      "logps/chosen": -808.0,
+      "logps/rejected": -960.0,
+      "loss": 0.3975,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.34375,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -8.0625,
+      "step": 8990
+    },
+    {
+      "epoch": 0.33500214029145187,
+      "grad_norm": 6.165639730007472,
+      "learning_rate": 4.205030215947261e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -816.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4232,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.28125,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -7.84375,
+      "step": 9000
+    },
+    {
+      "epoch": 0.3353743648917757,
+      "grad_norm": 7.5286732545518245,
+      "learning_rate": 4.202653089465097e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -800.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4124,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.1875,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -7.75,
+      "step": 9010
+    },
+    {
+      "epoch": 0.3357465894920995,
+      "grad_norm": 6.041701210362503,
+      "learning_rate": 4.2002730883320493e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.96875,
+      "logps/chosen": -816.0,
+      "logps/rejected": -968.0,
+      "loss": 0.4149,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -7.90625,
+      "step": 9020
+    },
+    {
+      "epoch": 0.3361188140924234,
+      "grad_norm": 5.500827668613241,
+      "learning_rate": 4.1978902165663616e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -844.0,
+      "logps/rejected": -972.0,
+      "loss": 0.4225,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.59375,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -8.125,
+      "step": 9030
+    },
+    {
+      "epoch": 0.3364910386927472,
+      "grad_norm": 5.807049103425073,
+      "learning_rate": 4.1955044781911215e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.4421,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -8.375,
+      "step": 9040
+    },
+    {
+      "epoch": 0.33686326329307104,
+      "grad_norm": 7.980671508713517,
+      "learning_rate": 4.193115877234258e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.875,
+      "logps/chosen": -848.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4251,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -6.71875,
+      "rewards/margins": 1.28125,
+      "rewards/rejected": -8.0,
+      "step": 9050
+    },
+    {
+      "epoch": 0.33723548789339486,
+      "grad_norm": 5.885217286326728,
+      "learning_rate": 4.1907244177285326e-07,
+      "logits/chosen": -4.09375,
+      "logits/rejected": -3.953125,
+      "logps/chosen": -844.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4229,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -8.0625,
+      "step": 9060
+    },
+    {
+      "epoch": 0.33760771249371874,
+      "grad_norm": 5.777807317558145,
+      "learning_rate": 4.188330103711533e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -832.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4021,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -8.125,
+      "step": 9070
+    },
+    {
+      "epoch": 0.33797993709404256,
+      "grad_norm": 7.552208779159908,
+      "learning_rate": 4.185932939225666e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -880.0,
+      "logps/rejected": -996.0,
+      "loss": 0.4264,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.2421875,
+      "rewards/rejected": -8.1875,
+      "step": 9080
+    },
+    {
+      "epoch": 0.3383521616943664,
+      "grad_norm": 6.394021170531517,
+      "learning_rate": 4.1835329283181506e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -848.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4021,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.625,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -8.0625,
+      "step": 9090
+    },
+    {
+      "epoch": 0.3387243862946902,
+      "grad_norm": 8.363072057386074,
+      "learning_rate": 4.1811300750410137e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.875,
+      "logps/chosen": -848.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4163,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -8.1875,
+      "step": 9100
+    },
+    {
+      "epoch": 0.33909661089501403,
+      "grad_norm": 6.225915898034823,
+      "learning_rate": 4.1787243834510793e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -856.0,
+      "logps/rejected": -968.0,
+      "loss": 0.4194,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -6.5625,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -8.0,
+      "step": 9110
+    },
+    {
+      "epoch": 0.3394688354953379,
+      "grad_norm": 6.5489406352606165,
+      "learning_rate": 4.176315857609963e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.9375,
+      "logps/chosen": -904.0,
+      "logps/rejected": -980.0,
+      "loss": 0.4306,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.21875,
+      "rewards/rejected": -7.96875,
+      "step": 9120
+    },
+    {
+      "epoch": 0.33984106009566173,
+      "grad_norm": 6.370264442851537,
+      "learning_rate": 4.173904501584066e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.427,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.4375,
+      "step": 9130
+    },
+    {
+      "epoch": 0.34021328469598555,
+      "grad_norm": 6.239792194567684,
+      "learning_rate": 4.1714903194445686e-07,
+      "logits/chosen": -4.125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.4211,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.375,
+      "step": 9140
+    },
+    {
+      "epoch": 0.3405855092963094,
+      "grad_norm": 7.131153346474517,
+      "learning_rate": 4.169073315267421e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.96875,
+      "logps/chosen": -884.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4115,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.234375,
+      "rewards/rejected": -8.125,
+      "step": 9150
+    },
+    {
+      "epoch": 0.34095773389663325,
+      "grad_norm": 6.791941199489257,
+      "learning_rate": 4.1666534931333406e-07,
+      "logits/chosen": -4.125,
+      "logits/rejected": -3.96875,
+      "logps/chosen": -864.0,
+      "logps/rejected": -980.0,
+      "loss": 0.4201,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.6875,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -8.0625,
+      "step": 9160
+    },
+    {
+      "epoch": 0.3413299584969571,
+      "grad_norm": 6.861874229852902,
+      "learning_rate": 4.1642308571277996e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.875,
+      "logps/chosen": -852.0,
+      "logps/rejected": -992.0,
+      "loss": 0.4183,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -8.25,
+      "step": 9170
+    },
+    {
+      "epoch": 0.3417021830972809,
+      "grad_norm": 6.073376330025923,
+      "learning_rate": 4.1618054113410217e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -856.0,
+      "logps/rejected": -992.0,
+      "loss": 0.393,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.6875,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -8.125,
+      "step": 9180
+    },
+    {
+      "epoch": 0.3420744076976047,
+      "grad_norm": 6.766012141847919,
+      "learning_rate": 4.159377159867976e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -852.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.412,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -8.5,
+      "step": 9190
+    },
+    {
+      "epoch": 0.34244663229792854,
+      "grad_norm": 5.881485789495539,
+      "learning_rate": 4.1569461068083664e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.4273,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -8.375,
+      "step": 9200
+    },
+    {
+      "epoch": 0.3428188568982524,
+      "grad_norm": 6.149657891629919,
+      "learning_rate": 4.154512256266629e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -832.0,
+      "logps/rejected": -948.0,
+      "loss": 0.4181,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": -7.8125,
+      "step": 9210
+    },
+    {
+      "epoch": 0.34319108149857624,
+      "grad_norm": 6.103269128159296,
+      "learning_rate": 4.152075612351921e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -852.0,
+      "logps/rejected": -980.0,
+      "loss": 0.3943,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -6.625,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -8.125,
+      "step": 9220
+    },
+    {
+      "epoch": 0.34356330609890007,
+      "grad_norm": 6.8721754063831035,
+      "learning_rate": 4.149636179178117e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.875,
+      "logps/chosen": -880.0,
+      "logps/rejected": -992.0,
+      "loss": 0.4249,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -8.3125,
+      "step": 9230
+    },
+    {
+      "epoch": 0.3439355306992239,
+      "grad_norm": 6.921635966074735,
+      "learning_rate": 4.1471939608637997e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -860.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.4273,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.8125,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.4375,
+      "step": 9240
+    },
+    {
+      "epoch": 0.34430775529954777,
+      "grad_norm": 6.068303934933823,
+      "learning_rate": 4.144748961532255e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -848.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4055,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.59375,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.125,
+      "step": 9250
+    },
+    {
+      "epoch": 0.3446799798998716,
+      "grad_norm": 6.6380746622565,
+      "learning_rate": 4.1423011853114644e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.921875,
+      "logps/chosen": -856.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4191,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.5,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -7.90625,
+      "step": 9260
+    },
+    {
+      "epoch": 0.3450522045001954,
+      "grad_norm": 5.882505378159859,
+      "learning_rate": 4.1398506363340965e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -832.0,
+      "logps/rejected": -960.0,
+      "loss": 0.401,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.5,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -7.90625,
+      "step": 9270
+    },
+    {
+      "epoch": 0.34542442910051924,
+      "grad_norm": 6.426011847463431,
+      "learning_rate": 4.137397318737502e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -836.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4015,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.5625,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.1875,
+      "step": 9280
+    },
+    {
+      "epoch": 0.3457966537008431,
+      "grad_norm": 7.2104806165875726,
+      "learning_rate": 4.134941236663706e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -820.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.4209,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.46875,
+      "rewards/margins": 1.984375,
+      "rewards/rejected": -8.5,
+      "step": 9290
+    },
+    {
+      "epoch": 0.34616887830116694,
+      "grad_norm": 7.35171826367392,
+      "learning_rate": 4.132482394259401e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -800.0,
+      "logps/rejected": -948.0,
+      "loss": 0.4099,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -7.875,
+      "step": 9300
+    },
+    {
+      "epoch": 0.34654110290149076,
+      "grad_norm": 6.141606417222074,
+      "learning_rate": 4.1300207956759395e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -836.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4253,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.59375,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -8.0,
+      "step": 9310
+    },
+    {
+      "epoch": 0.3469133275018146,
+      "grad_norm": 7.164569397555365,
+      "learning_rate": 4.127556445069328e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -800.0,
+      "logps/rejected": -948.0,
+      "loss": 0.4077,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.21875,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -7.84375,
+      "step": 9320
+    },
+    {
+      "epoch": 0.3472855521021384,
+      "grad_norm": 7.137950738605295,
+      "learning_rate": 4.125089346600218e-07,
+      "logits/chosen": -4.09375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -860.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.4082,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -8.375,
+      "step": 9330
+    },
+    {
+      "epoch": 0.3476577767024623,
+      "grad_norm": 7.051170004750458,
+      "learning_rate": 4.122619504433902e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -852.0,
+      "logps/rejected": -976.0,
+      "loss": 0.4213,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.5625,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -7.9375,
+      "step": 9340
+    },
+    {
+      "epoch": 0.3480300013027861,
+      "grad_norm": 5.536612127727956,
+      "learning_rate": 4.120146922740303e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -828.0,
+      "logps/rejected": -976.0,
+      "loss": 0.4164,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.40625,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -7.96875,
+      "step": 9350
+    },
+    {
+      "epoch": 0.34840222590310993,
+      "grad_norm": 5.569949171199698,
+      "learning_rate": 4.1176716056939715e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -852.0,
+      "logps/rejected": -980.0,
+      "loss": 0.3928,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.5,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -7.96875,
+      "step": 9360
+    },
+    {
+      "epoch": 0.34877445050343375,
+      "grad_norm": 6.482695065678995,
+      "learning_rate": 4.115193557474074e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -804.0,
+      "logps/rejected": -948.0,
+      "loss": 0.4086,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -7.875,
+      "step": 9370
+    },
+    {
+      "epoch": 0.34914667510375763,
+      "grad_norm": 6.889701996051546,
+      "learning_rate": 4.1127127822643894e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -836.0,
+      "logps/rejected": -972.0,
+      "loss": 0.4133,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.59375,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -8.0,
+      "step": 9380
+    },
+    {
+      "epoch": 0.34951889970408145,
+      "grad_norm": 6.720031347736244,
+      "learning_rate": 4.1102292842533004e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.4125,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -8.4375,
+      "step": 9390
+    },
+    {
+      "epoch": 0.3498911243044053,
+      "grad_norm": 4.933620099747848,
+      "learning_rate": 4.1077430676337867e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -832.0,
+      "logps/rejected": -992.0,
+      "loss": 0.4053,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -8.25,
+      "step": 9400
+    },
+    {
+      "epoch": 0.3502633489047291,
+      "grad_norm": 7.255235719368424,
+      "learning_rate": 4.1052541366034174e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -800.0,
+      "logps/rejected": -944.0,
+      "loss": 0.4025,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -7.9375,
+      "step": 9410
+    },
+    {
+      "epoch": 0.3506355735050529,
+      "grad_norm": 7.6839958173409055,
+      "learning_rate": 4.102762495364346e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -824.0,
+      "logps/rejected": -972.0,
+      "loss": 0.4309,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.5625,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -8.125,
+      "step": 9420
+    },
+    {
+      "epoch": 0.3510077981053768,
+      "grad_norm": 5.841110947628403,
+      "learning_rate": 4.100268148123299e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -864.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4058,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.625,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -8.0625,
+      "step": 9430
+    },
+    {
+      "epoch": 0.3513800227057006,
+      "grad_norm": 5.481293636604824,
+      "learning_rate": 4.0977710990915747e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -836.0,
+      "logps/rejected": -964.0,
+      "loss": 0.3984,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.71875,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -8.1875,
+      "step": 9440
+    },
+    {
+      "epoch": 0.35175224730602445,
+      "grad_norm": 7.690837125250735,
+      "learning_rate": 4.0952713524850313e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -852.0,
+      "logps/rejected": -980.0,
+      "loss": 0.4106,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.0625,
+      "step": 9450
+    },
+    {
+      "epoch": 0.35212447190634827,
+      "grad_norm": 7.095617053942233,
+      "learning_rate": 4.0927689125240806e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -832.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4277,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.21875,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -7.71875,
+      "step": 9460
+    },
+    {
+      "epoch": 0.35249669650667215,
+      "grad_norm": 5.665000442318552,
+      "learning_rate": 4.090263783433683e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -836.0,
+      "logps/rejected": -968.0,
+      "loss": 0.4353,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -8.0,
+      "step": 9470
+    },
+    {
+      "epoch": 0.35286892110699597,
+      "grad_norm": 6.6790604780930005,
+      "learning_rate": 4.0877559694433384e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -844.0,
+      "logps/rejected": -972.0,
+      "loss": 0.3913,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.59375,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -8.125,
+      "step": 9480
+    },
+    {
+      "epoch": 0.3532411457073198,
+      "grad_norm": 7.265344769439008,
+      "learning_rate": 4.0852454747870807e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -852.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4349,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.1796875,
+      "rewards/rejected": -7.90625,
+      "step": 9490
+    },
+    {
+      "epoch": 0.3536133703076436,
+      "grad_norm": 6.656558714753696,
+      "learning_rate": 4.082732303703469e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -804.0,
+      "logps/rejected": -920.0,
+      "loss": 0.4135,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.15625,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -7.4375,
+      "step": 9500
+    },
+    {
+      "epoch": 0.3539855949079675,
+      "grad_norm": 5.906756468220538,
+      "learning_rate": 4.0802164604355805e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -848.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4097,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -7.9375,
+      "step": 9510
+    },
+    {
+      "epoch": 0.3543578195082913,
+      "grad_norm": 6.394831334208994,
+      "learning_rate": 4.077697949231005e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -844.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4172,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.71875,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -8.1875,
+      "step": 9520
+    },
+    {
+      "epoch": 0.35473004410861514,
+      "grad_norm": 6.700397024654636,
+      "learning_rate": 4.075176774341835e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -848.0,
+      "logps/rejected": -972.0,
+      "loss": 0.4128,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -6.71875,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -8.0,
+      "step": 9530
+    },
+    {
+      "epoch": 0.35510226870893896,
+      "grad_norm": 6.421194975246333,
+      "learning_rate": 4.0726529400246634e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -788.0,
+      "logps/rejected": -952.0,
+      "loss": 0.4137,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -7.96875,
+      "step": 9540
+    },
+    {
+      "epoch": 0.3554744933092628,
+      "grad_norm": 6.507157642196768,
+      "learning_rate": 4.070126450540569e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.4091,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.8125,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -8.1875,
+      "step": 9550
+    },
+    {
+      "epoch": 0.35584671790958666,
+      "grad_norm": 5.867565943804244,
+      "learning_rate": 4.0675973101551177e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -808.0,
+      "logps/rejected": -976.0,
+      "loss": 0.4041,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -8.0,
+      "step": 9560
+    },
+    {
+      "epoch": 0.3562189425099105,
+      "grad_norm": 6.552527694586657,
+      "learning_rate": 4.065065523138347e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -820.0,
+      "logps/rejected": -972.0,
+      "loss": 0.4143,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -8.125,
+      "step": 9570
+    },
+    {
+      "epoch": 0.3565911671102343,
+      "grad_norm": 6.0695908704161505,
+      "learning_rate": 4.062531093764764e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -832.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4128,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -8.0625,
+      "step": 9580
+    },
+    {
+      "epoch": 0.35696339171055813,
+      "grad_norm": 6.038046759978597,
+      "learning_rate": 4.05999402631334e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.413,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -8.25,
+      "step": 9590
+    },
+    {
+      "epoch": 0.357335616310882,
+      "grad_norm": 5.787236047699248,
+      "learning_rate": 4.0574543250674973e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -868.0,
+      "logps/rejected": -996.0,
+      "loss": 0.4198,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -8.25,
+      "step": 9600
+    },
+    {
+      "epoch": 0.35770784091120583,
+      "grad_norm": 6.4799435976085915,
+      "learning_rate": 4.054911994315104e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3904,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.3125,
+      "step": 9610
+    },
+    {
+      "epoch": 0.35808006551152965,
+      "grad_norm": 7.598979397793083,
+      "learning_rate": 4.052367038348471e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.3771,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -8.5,
+      "step": 9620
+    },
+    {
+      "epoch": 0.3584522901118535,
+      "grad_norm": 6.462650275311091,
+      "learning_rate": 4.049819461464338e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -856.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4052,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -8.5,
+      "step": 9630
+    },
+    {
+      "epoch": 0.3588245147121773,
+      "grad_norm": 7.127969543442139,
+      "learning_rate": 4.0472692679638733e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -840.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.3992,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.6875,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -8.5625,
+      "step": 9640
+    },
+    {
+      "epoch": 0.3591967393125012,
+      "grad_norm": 7.264604843154183,
+      "learning_rate": 4.0447164621526586e-07,
+      "logits/chosen": -4.125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3966,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -8.6875,
+      "step": 9650
+    },
+    {
+      "epoch": 0.359568963912825,
+      "grad_norm": 6.041655908210674,
+      "learning_rate": 4.04216104834069e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.411,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -8.6875,
+      "step": 9660
+    },
+    {
+      "epoch": 0.3599411885131488,
+      "grad_norm": 7.118305214555132,
+      "learning_rate": 4.0396030308423643e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4317,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -8.5625,
+      "step": 9670
+    },
+    {
+      "epoch": 0.36031341311347265,
+      "grad_norm": 5.868559425967568,
+      "learning_rate": 4.037042413976476e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.3892,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.8125,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -8.375,
+      "step": 9680
+    },
+    {
+      "epoch": 0.3606856377137965,
+      "grad_norm": 5.408074753055558,
+      "learning_rate": 4.0344792020662067e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -836.0,
+      "logps/rejected": -976.0,
+      "loss": 0.3867,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -8.3125,
+      "step": 9690
+    },
+    {
+      "epoch": 0.36105786231412035,
+      "grad_norm": 7.219656421844504,
+      "learning_rate": 4.0319133994391206e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4187,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -8.4375,
+      "step": 9700
+    },
+    {
+      "epoch": 0.36143008691444417,
+      "grad_norm": 6.40071269787148,
+      "learning_rate": 4.0293450104271544e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -856.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4109,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -8.3125,
+      "step": 9710
+    },
+    {
+      "epoch": 0.361802311514768,
+      "grad_norm": 6.562244074494197,
+      "learning_rate": 4.026774039366612e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -856.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4042,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.8125,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -8.375,
+      "step": 9720
+    },
+    {
+      "epoch": 0.36217453611509187,
+      "grad_norm": 6.822427416072704,
+      "learning_rate": 4.0242004905981587e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -880.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4295,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -8.4375,
+      "step": 9730
+    },
+    {
+      "epoch": 0.3625467607154157,
+      "grad_norm": 5.471950994450153,
+      "learning_rate": 4.0216243684668073e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -848.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4041,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.375,
+      "step": 9740
+    },
+    {
+      "epoch": 0.3629189853157395,
+      "grad_norm": 7.590254576888013,
+      "learning_rate": 4.019045677321921e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -864.0,
+      "logps/rejected": -980.0,
+      "loss": 0.4069,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -8.1875,
+      "step": 9750
+    },
+    {
+      "epoch": 0.36329120991606334,
+      "grad_norm": 6.184365090268907,
+      "learning_rate": 4.016464421517196e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -844.0,
+      "logps/rejected": -980.0,
+      "loss": 0.4048,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.125,
+      "step": 9760
+    },
+    {
+      "epoch": 0.36366343451638716,
+      "grad_norm": 6.879495523485304,
+      "learning_rate": 4.0138806054106604e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4106,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -8.375,
+      "step": 9770
+    },
+    {
+      "epoch": 0.36403565911671104,
+      "grad_norm": 7.191175493183634,
+      "learning_rate": 4.011294233364664e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3942,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -8.6875,
+      "step": 9780
+    },
+    {
+      "epoch": 0.36440788371703486,
+      "grad_norm": 6.081213370303092,
+      "learning_rate": 4.0087053097458735e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3756,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.875,
+      "step": 9790
+    },
+    {
+      "epoch": 0.3647801083173587,
+      "grad_norm": 7.29763235176727,
+      "learning_rate": 4.00611383892526e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -832.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4464,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -8.4375,
+      "step": 9800
+    },
+    {
+      "epoch": 0.3651523329176825,
+      "grad_norm": 6.738921621615778,
+      "learning_rate": 4.003519825278101e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -832.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4262,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.40625,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -8.0,
+      "step": 9810
+    },
+    {
+      "epoch": 0.3655245575180064,
+      "grad_norm": 7.030815345440591,
+      "learning_rate": 4.000923273183961e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -792.0,
+      "logps/rejected": -948.0,
+      "loss": 0.4004,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -8.0,
+      "step": 9820
+    },
+    {
+      "epoch": 0.3658967821183302,
+      "grad_norm": 6.000374566301237,
+      "learning_rate": 3.9983241870266935e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -860.0,
+      "logps/rejected": -996.0,
+      "loss": 0.3927,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -8.25,
+      "step": 9830
+    },
+    {
+      "epoch": 0.36626900671865403,
+      "grad_norm": 6.583633173464554,
+      "learning_rate": 3.995722571194431e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.4069,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.3671875,
+      "rewards/rejected": -8.3125,
+      "step": 9840
+    },
+    {
+      "epoch": 0.36664123131897786,
+      "grad_norm": 7.813685347445612,
+      "learning_rate": 3.9931184300795746e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.413,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -8.5,
+      "step": 9850
+    },
+    {
+      "epoch": 0.36701345591930173,
+      "grad_norm": 6.3535686751235785,
+      "learning_rate": 3.9905117680787906e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4234,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -8.375,
+      "step": 9860
+    },
+    {
+      "epoch": 0.36738568051962556,
+      "grad_norm": 5.8115365905731435,
+      "learning_rate": 3.987902589593e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4168,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.3125,
+      "step": 9870
+    },
+    {
+      "epoch": 0.3677579051199494,
+      "grad_norm": 5.460402405508471,
+      "learning_rate": 3.9852908990273737e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.4049,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.5,
+      "step": 9880
+    },
+    {
+      "epoch": 0.3681301297202732,
+      "grad_norm": 5.7438331257859705,
+      "learning_rate": 3.9826767007913246e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -844.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.4075,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.6875,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -8.375,
+      "step": 9890
+    },
+    {
+      "epoch": 0.368502354320597,
+      "grad_norm": 5.844322694867869,
+      "learning_rate": 3.9800599992984973e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -832.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4148,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.5,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -8.1875,
+      "step": 9900
+    },
+    {
+      "epoch": 0.3688745789209209,
+      "grad_norm": 6.806914404761199,
+      "learning_rate": 3.9774407989667637e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -844.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4066,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -6.5,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.0625,
+      "step": 9910
+    },
+    {
+      "epoch": 0.3692468035212447,
+      "grad_norm": 6.048428105563145,
+      "learning_rate": 3.9748191042182143e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3822,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -8.625,
+      "step": 9920
+    },
+    {
+      "epoch": 0.36961902812156855,
+      "grad_norm": 7.008262693146331,
+      "learning_rate": 3.9721949194791527e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3994,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -8.8125,
+      "step": 9930
+    },
+    {
+      "epoch": 0.36999125272189237,
+      "grad_norm": 5.779755677283361,
+      "learning_rate": 3.969568249180083e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -880.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4346,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -8.1875,
+      "step": 9940
+    },
+    {
+      "epoch": 0.37036347732221625,
+      "grad_norm": 5.858468681242547,
+      "learning_rate": 3.96693909775571e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -824.0,
+      "logps/rejected": -956.0,
+      "loss": 0.3998,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -6.375,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -7.96875,
+      "step": 9950
+    },
+    {
+      "epoch": 0.3707357019225401,
+      "grad_norm": 6.793562522275051,
+      "learning_rate": 3.9643074696449235e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -856.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4056,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -8.375,
+      "step": 9960
+    },
+    {
+      "epoch": 0.3711079265228639,
+      "grad_norm": 6.317373520282721,
+      "learning_rate": 3.961673369290798e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.4035,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -8.25,
+      "step": 9970
+    },
+    {
+      "epoch": 0.3714801511231877,
+      "grad_norm": 6.501821857889046,
+      "learning_rate": 3.9590368011405786e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -856.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.388,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.71875,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -8.375,
+      "step": 9980
+    },
+    {
+      "epoch": 0.37185237572351154,
+      "grad_norm": 6.609022314785501,
+      "learning_rate": 3.956397769645681e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -856.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.3976,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -8.5,
+      "step": 9990
+    },
+    {
+      "epoch": 0.3722246003238354,
+      "grad_norm": 7.090127236159045,
+      "learning_rate": 3.9537562792616753e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.4132,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.5625,
+      "step": 10000
+    },
+    {
+      "epoch": 0.3722246003238354,
+      "eval_logits/chosen": -3.90625,
+      "eval_logits/rejected": -3.71875,
+      "eval_logps/chosen": -912.0,
+      "eval_logps/rejected": -1024.0,
+      "eval_loss": 0.4587233364582062,
+      "eval_rewards/accuracies": 0.755190908908844,
+      "eval_rewards/chosen": -7.125,
+      "eval_rewards/margins": 1.40625,
+      "eval_rewards/rejected": -8.5,
+      "eval_runtime": 6269.7,
+      "eval_samples_per_second": 30.473,
+      "eval_steps_per_second": 0.476,
+      "step": 10000
+    },
+    {
+      "epoch": 0.37259682492415924,
+      "grad_norm": 6.779952102054952,
+      "learning_rate": 3.951112334448288e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.4014,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -8.5,
+      "step": 10010
+    },
+    {
+      "epoch": 0.37296904952448307,
+      "grad_norm": 7.527072668096298,
+      "learning_rate": 3.948465939669385e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -856.0,
+      "logps/rejected": -980.0,
+      "loss": 0.4091,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.71875,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -8.125,
+      "step": 10020
+    },
+    {
+      "epoch": 0.3733412741248069,
+      "grad_norm": 6.6682252692638695,
+      "learning_rate": 3.9458170993929705e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.5,
+      "logps/chosen": -820.0,
+      "logps/rejected": -960.0,
+      "loss": 0.4461,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.3125,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -7.84375,
+      "step": 10030
+    },
+    {
+      "epoch": 0.37371349872513077,
+      "grad_norm": 5.3998914224314145,
+      "learning_rate": 3.94316581809118e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -812.0,
+      "logps/rejected": -928.0,
+      "loss": 0.4136,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.25,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -7.65625,
+      "step": 10040
+    },
+    {
+      "epoch": 0.3740857233254546,
+      "grad_norm": 6.0041789113424215,
+      "learning_rate": 3.940512100240264e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -4.0,
+      "logps/chosen": -864.0,
+      "logps/rejected": -964.0,
+      "loss": 0.4009,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -7.96875,
+      "step": 10050
+    },
+    {
+      "epoch": 0.3744579479257784,
+      "grad_norm": 6.39066986559693,
+      "learning_rate": 3.9378559503205934e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -820.0,
+      "logps/rejected": -976.0,
+      "loss": 0.3916,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.5,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -8.25,
+      "step": 10060
+    },
+    {
+      "epoch": 0.37483017252610223,
+      "grad_norm": 7.1807120166639615,
+      "learning_rate": 3.9351973728166407e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -816.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4098,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -8.125,
+      "step": 10070
+    },
+    {
+      "epoch": 0.3752023971264261,
+      "grad_norm": 5.4249408438593205,
+      "learning_rate": 3.9325363722169787e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4058,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -8.625,
+      "step": 10080
+    },
+    {
+      "epoch": 0.37557462172674994,
+      "grad_norm": 5.431048967355182,
+      "learning_rate": 3.9298729530142716e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -840.0,
+      "logps/rejected": -992.0,
+      "loss": 0.3902,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.6875,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.25,
+      "step": 10090
+    },
+    {
+      "epoch": 0.37594684632707376,
+      "grad_norm": 7.851965006541211,
+      "learning_rate": 3.927207119705267e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.4178,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -8.375,
+      "step": 10100
+    },
+    {
+      "epoch": 0.3763190709273976,
+      "grad_norm": 6.690727711927166,
+      "learning_rate": 3.924538876790787e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3824,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -8.75,
+      "step": 10110
+    },
+    {
+      "epoch": 0.3766912955277214,
+      "grad_norm": 7.0085641591138845,
+      "learning_rate": 3.9218682287757233e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4086,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -8.5,
+      "step": 10120
+    },
+    {
+      "epoch": 0.3770635201280453,
+      "grad_norm": 7.1961509641723636,
+      "learning_rate": 3.9191951801690273e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.4032,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -8.125,
+      "step": 10130
+    },
+    {
+      "epoch": 0.3774357447283691,
+      "grad_norm": 6.592048642055363,
+      "learning_rate": 3.916519735483703e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -824.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.4032,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -8.4375,
+      "step": 10140
+    },
+    {
+      "epoch": 0.3778079693286929,
+      "grad_norm": 7.014300630950333,
+      "learning_rate": 3.913841899236803e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -852.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4033,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.625,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.125,
+      "step": 10150
+    },
+    {
+      "epoch": 0.37818019392901675,
+      "grad_norm": 6.059582641874987,
+      "learning_rate": 3.911161675949412e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -832.0,
+      "logps/rejected": -972.0,
+      "loss": 0.4061,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -6.5625,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -8.125,
+      "step": 10160
+    },
+    {
+      "epoch": 0.37855241852934063,
+      "grad_norm": 6.625511979713202,
+      "learning_rate": 3.90847907014665e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -856.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.4139,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.4375,
+      "step": 10170
+    },
+    {
+      "epoch": 0.37892464312966445,
+      "grad_norm": 6.85028678316669,
+      "learning_rate": 3.905794086357658e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4179,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -8.6875,
+      "step": 10180
+    },
+    {
+      "epoch": 0.3792968677299883,
+      "grad_norm": 7.014468726531082,
+      "learning_rate": 3.903106729115591e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.4115,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.8125,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -8.5625,
+      "step": 10190
+    },
+    {
+      "epoch": 0.3796690923303121,
+      "grad_norm": 7.019860578223725,
+      "learning_rate": 3.90041700295761e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -856.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4033,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -8.3125,
+      "step": 10200
+    },
+    {
+      "epoch": 0.3800413169306359,
+      "grad_norm": 6.013423111216983,
+      "learning_rate": 3.897724912424879e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -832.0,
+      "logps/rejected": -972.0,
+      "loss": 0.4031,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.0625,
+      "step": 10210
+    },
+    {
+      "epoch": 0.3804135415309598,
+      "grad_norm": 5.008164120417849,
+      "learning_rate": 3.8950304620625514e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -836.0,
+      "logps/rejected": -968.0,
+      "loss": 0.4133,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -8.25,
+      "step": 10220
+    },
+    {
+      "epoch": 0.3807857661312836,
+      "grad_norm": 7.008040983170978,
+      "learning_rate": 3.892333656419765e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4279,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -8.5625,
+      "step": 10230
+    },
+    {
+      "epoch": 0.38115799073160744,
+      "grad_norm": 5.54756499531245,
+      "learning_rate": 3.8896345000496343e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -876.0,
+      "logps/rejected": -984.0,
+      "loss": 0.3984,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -8.3125,
+      "step": 10240
+    },
+    {
+      "epoch": 0.38153021533193127,
+      "grad_norm": 5.910114821266497,
+      "learning_rate": 3.886932997509244e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3724,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -8.6875,
+      "step": 10250
+    },
+    {
+      "epoch": 0.38190243993225514,
+      "grad_norm": 9.517838636654623,
+      "learning_rate": 3.884229153359637e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -872.0,
+      "logps/rejected": -992.0,
+      "loss": 0.4233,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.359375,
+      "rewards/rejected": -8.25,
+      "step": 10260
+    },
+    {
+      "epoch": 0.38227466453257897,
+      "grad_norm": 5.6218446486798594,
+      "learning_rate": 3.881522972165812e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.625,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.392,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -8.3125,
+      "step": 10270
+    },
+    {
+      "epoch": 0.3826468891329028,
+      "grad_norm": 6.465995179565068,
+      "learning_rate": 3.8788144584967135e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3857,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.984375,
+      "rewards/rejected": -9.125,
+      "step": 10280
+    },
+    {
+      "epoch": 0.3830191137332266,
+      "grad_norm": 7.416653607793235,
+      "learning_rate": 3.876103616925223e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4063,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.625,
+      "step": 10290
+    },
+    {
+      "epoch": 0.3833913383335505,
+      "grad_norm": 7.149011353925458,
+      "learning_rate": 3.873390452028151e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.393,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -8.875,
+      "step": 10300
+    },
+    {
+      "epoch": 0.3837635629338743,
+      "grad_norm": 5.911789008322568,
+      "learning_rate": 3.870674968386234e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3878,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -8.9375,
+      "step": 10310
+    },
+    {
+      "epoch": 0.38413578753419814,
+      "grad_norm": 6.4930095025304215,
+      "learning_rate": 3.86795717058412e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3981,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -8.6875,
+      "step": 10320
+    },
+    {
+      "epoch": 0.38450801213452196,
+      "grad_norm": 7.509556789613068,
+      "learning_rate": 3.8652370632103665e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.75,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4102,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.9609375,
+      "rewards/rejected": -8.875,
+      "step": 10330
+    },
+    {
+      "epoch": 0.3848802367348458,
+      "grad_norm": 6.394014077669874,
+      "learning_rate": 3.862514650857428e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4372,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.234375,
+      "rewards/rejected": -8.625,
+      "step": 10340
+    },
+    {
+      "epoch": 0.38525246133516966,
+      "grad_norm": 5.755408873492995,
+      "learning_rate": 3.859789938121654e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4127,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -8.5,
+      "step": 10350
+    },
+    {
+      "epoch": 0.3856246859354935,
+      "grad_norm": 7.318901102559693,
+      "learning_rate": 3.8570629296032734e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -860.0,
+      "logps/rejected": -976.0,
+      "loss": 0.4003,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -8.0625,
+      "step": 10360
+    },
+    {
+      "epoch": 0.3859969105358173,
+      "grad_norm": 6.835292810820879,
+      "learning_rate": 3.854333629906395e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -880.0,
+      "logps/rejected": -976.0,
+      "loss": 0.434,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -8.1875,
+      "step": 10370
+    },
+    {
+      "epoch": 0.38636913513614113,
+      "grad_norm": 6.625679992018827,
+      "learning_rate": 3.851602043638994e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3991,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -8.5,
+      "step": 10380
+    },
+    {
+      "epoch": 0.386741359736465,
+      "grad_norm": 5.791124585353487,
+      "learning_rate": 3.848868175412906e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.399,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -8.625,
+      "step": 10390
+    },
+    {
+      "epoch": 0.38711358433678883,
+      "grad_norm": 5.887120632025183,
+      "learning_rate": 3.8461320298438206e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.4177,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -8.4375,
+      "step": 10400
+    },
+    {
+      "epoch": 0.38748580893711265,
+      "grad_norm": 6.091872714444972,
+      "learning_rate": 3.84339361155127e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.4048,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -8.5,
+      "step": 10410
+    },
+    {
+      "epoch": 0.3878580335374365,
+      "grad_norm": 7.134751282084675,
+      "learning_rate": 3.8406529251586254e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.408,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -8.75,
+      "step": 10420
+    },
+    {
+      "epoch": 0.3882302581377603,
+      "grad_norm": 7.076803676298143,
+      "learning_rate": 3.837909975293088e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -860.0,
+      "logps/rejected": -992.0,
+      "loss": 0.4066,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.71875,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -8.375,
+      "step": 10430
+    },
+    {
+      "epoch": 0.3886024827380842,
+      "grad_norm": 7.654813186699686,
+      "learning_rate": 3.835164766585679e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4047,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -8.4375,
+      "step": 10440
+    },
+    {
+      "epoch": 0.388974707338408,
+      "grad_norm": 6.954083232717519,
+      "learning_rate": 3.8324173036712336e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -860.0,
+      "logps/rejected": -996.0,
+      "loss": 0.4087,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -8.3125,
+      "step": 10450
+    },
+    {
+      "epoch": 0.3893469319387318,
+      "grad_norm": 6.3829840903074375,
+      "learning_rate": 3.829667591188393e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -852.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.3889,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -8.5,
+      "step": 10460
+    },
+    {
+      "epoch": 0.38971915653905564,
+      "grad_norm": 5.7369481688581745,
+      "learning_rate": 3.826915633779596e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4083,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -8.8125,
+      "step": 10470
+    },
+    {
+      "epoch": 0.3900913811393795,
+      "grad_norm": 6.881795636096312,
+      "learning_rate": 3.8241614360910726e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3991,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.625,
+      "step": 10480
+    },
+    {
+      "epoch": 0.39046360573970335,
+      "grad_norm": 12.19675475207699,
+      "learning_rate": 3.8214050027728335e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4019,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -8.3125,
+      "step": 10490
+    },
+    {
+      "epoch": 0.39083583034002717,
+      "grad_norm": 6.160453179830614,
+      "learning_rate": 3.818646338478665e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.4182,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -8.125,
+      "step": 10500
+    },
+    {
+      "epoch": 0.391208054940351,
+      "grad_norm": 6.482497191122117,
+      "learning_rate": 3.815885447866121e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.75,
+      "logps/chosen": -896.0,
+      "logps/rejected": -992.0,
+      "loss": 0.4389,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.3125,
+      "rewards/rejected": -8.25,
+      "step": 10510
+    },
+    {
+      "epoch": 0.39158027954067487,
+      "grad_norm": 6.62221266883809,
+      "learning_rate": 3.813122335596513e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.4275,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.375,
+      "rewards/rejected": -8.375,
+      "step": 10520
+    },
+    {
+      "epoch": 0.3919525041409987,
+      "grad_norm": 6.5872740656990105,
+      "learning_rate": 3.8103570063349027e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -872.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4077,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -8.0625,
+      "step": 10530
+    },
+    {
+      "epoch": 0.3923247287413225,
+      "grad_norm": 5.803158514867611,
+      "learning_rate": 3.807589464750097e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.3975,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -8.4375,
+      "step": 10540
+    },
+    {
+      "epoch": 0.39269695334164634,
+      "grad_norm": 6.6768162088198615,
+      "learning_rate": 3.8048197155146343e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3902,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -8.6875,
+      "step": 10550
+    },
+    {
+      "epoch": 0.39306917794197016,
+      "grad_norm": 6.0859183911204395,
+      "learning_rate": 3.8020477633047847e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -888.0,
+      "logps/rejected": -992.0,
+      "loss": 0.4338,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.1875,
+      "rewards/rejected": -8.25,
+      "step": 10560
+    },
+    {
+      "epoch": 0.39344140254229404,
+      "grad_norm": 6.697776696519422,
+      "learning_rate": 3.7992736128005344e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.9375,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.4491,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.8125,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -8.375,
+      "step": 10570
+    },
+    {
+      "epoch": 0.39381362714261786,
+      "grad_norm": 6.444776844580686,
+      "learning_rate": 3.7964972686855833e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -888.0,
+      "logps/rejected": -996.0,
+      "loss": 0.4133,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.234375,
+      "rewards/rejected": -8.25,
+      "step": 10580
+    },
+    {
+      "epoch": 0.3941858517429417,
+      "grad_norm": 6.808662867286216,
+      "learning_rate": 3.7937187356473323e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3935,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -8.5,
+      "step": 10590
+    },
+    {
+      "epoch": 0.3945580763432655,
+      "grad_norm": 6.612542544177442,
+      "learning_rate": 3.7909380183768796e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -852.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3928,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 2.0,
+      "rewards/rejected": -8.625,
+      "step": 10600
+    },
+    {
+      "epoch": 0.3949303009435894,
+      "grad_norm": 6.718268064693689,
+      "learning_rate": 3.7881551215690123e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4276,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.625,
+      "step": 10610
+    },
+    {
+      "epoch": 0.3953025255439132,
+      "grad_norm": 7.142340690198998,
+      "learning_rate": 3.785370049922194e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3911,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -8.625,
+      "step": 10620
+    },
+    {
+      "epoch": 0.39567475014423703,
+      "grad_norm": 5.887605153552629,
+      "learning_rate": 3.782582808138564e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.4152,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -6.8125,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -8.25,
+      "step": 10630
+    },
+    {
+      "epoch": 0.39604697474456085,
+      "grad_norm": 6.5068864417704,
+      "learning_rate": 3.7797934009239217e-07,
+      "logits/chosen": -4.09375,
+      "logits/rejected": -3.953125,
+      "logps/chosen": -864.0,
+      "logps/rejected": -972.0,
+      "loss": 0.3934,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.25,
+      "step": 10640
+    },
+    {
+      "epoch": 0.3964191993448847,
+      "grad_norm": 7.631467171481536,
+      "learning_rate": 3.777001832987726e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4275,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -8.3125,
+      "step": 10650
+    },
+    {
+      "epoch": 0.39679142394520855,
+      "grad_norm": 6.667257571188621,
+      "learning_rate": 3.77420810904308e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.4139,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -8.5,
+      "step": 10660
+    },
+    {
+      "epoch": 0.3971636485455324,
+      "grad_norm": 7.122972460286967,
+      "learning_rate": 3.771412233806731e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.96875,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4038,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -8.4375,
+      "step": 10670
+    },
+    {
+      "epoch": 0.3975358731458562,
+      "grad_norm": 6.286075833117893,
+      "learning_rate": 3.768614211999056e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.3894,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -8.4375,
+      "step": 10680
+    },
+    {
+      "epoch": 0.39790809774618,
+      "grad_norm": 6.3483774252560075,
+      "learning_rate": 3.765814048344056e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.4098,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -8.5,
+      "step": 10690
+    },
+    {
+      "epoch": 0.3982803223465039,
+      "grad_norm": 7.015847191948804,
+      "learning_rate": 3.76301174756935e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4033,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -8.6875,
+      "step": 10700
+    },
+    {
+      "epoch": 0.3986525469468277,
+      "grad_norm": 5.953411572352932,
+      "learning_rate": 3.7602073144061617e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -848.0,
+      "logps/rejected": -996.0,
+      "loss": 0.3957,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.625,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -8.375,
+      "step": 10710
+    },
+    {
+      "epoch": 0.39902477154715155,
+      "grad_norm": 6.576501786367329,
+      "learning_rate": 3.757400753589319e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -848.0,
+      "logps/rejected": -992.0,
+      "loss": 0.4021,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -6.53125,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -8.25,
+      "step": 10720
+    },
+    {
+      "epoch": 0.39939699614747537,
+      "grad_norm": 6.361995504737115,
+      "learning_rate": 3.7545920698572393e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -852.0,
+      "logps/rejected": -988.0,
+      "loss": 0.3903,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.625,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -8.125,
+      "step": 10730
+    },
+    {
+      "epoch": 0.39976922074779925,
+      "grad_norm": 6.7723565173790865,
+      "learning_rate": 3.7517812679519255e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4104,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -8.4375,
+      "step": 10740
+    },
+    {
+      "epoch": 0.40014144534812307,
+      "grad_norm": 6.419614099192593,
+      "learning_rate": 3.748968352618957e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.3821,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -8.4375,
+      "step": 10750
+    },
+    {
+      "epoch": 0.4005136699484469,
+      "grad_norm": 6.5055714975763115,
+      "learning_rate": 3.7461533286074785e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3985,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.8125,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -8.5625,
+      "step": 10760
+    },
+    {
+      "epoch": 0.4008858945487707,
+      "grad_norm": 7.786187395627907,
+      "learning_rate": 3.743336200670199e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.376,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -8.8125,
+      "step": 10770
+    },
+    {
+      "epoch": 0.40125811914909454,
+      "grad_norm": 5.821122867200847,
+      "learning_rate": 3.7405169735633766e-07,
+      "logits/chosen": -4.125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4128,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.5625,
+      "step": 10780
+    },
+    {
+      "epoch": 0.4016303437494184,
+      "grad_norm": 6.679817159363863,
+      "learning_rate": 3.7376956520468154e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4086,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -8.6875,
+      "step": 10790
+    },
+    {
+      "epoch": 0.40200256834974224,
+      "grad_norm": 6.188724194323378,
+      "learning_rate": 3.734872240883854e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -852.0,
+      "logps/rejected": -956.0,
+      "loss": 0.4089,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.65625,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -8.125,
+      "step": 10800
+    },
+    {
+      "epoch": 0.40237479295006606,
+      "grad_norm": 6.616489539438808,
+      "learning_rate": 3.73204674484136e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -844.0,
+      "logps/rejected": -984.0,
+      "loss": 0.4272,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.625,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -8.125,
+      "step": 10810
+    },
+    {
+      "epoch": 0.4027470175503899,
+      "grad_norm": 6.738522652674378,
+      "learning_rate": 3.729219168689721e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -840.0,
+      "logps/rejected": -960.0,
+      "loss": 0.4123,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.4375,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -7.875,
+      "step": 10820
+    },
+    {
+      "epoch": 0.40311924215071376,
+      "grad_norm": 6.299065511718082,
+      "learning_rate": 3.7263895172028345e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.4024,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -8.4375,
+      "step": 10830
+    },
+    {
+      "epoch": 0.4034914667510376,
+      "grad_norm": 7.637405274053756,
+      "learning_rate": 3.7235577951581063e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -852.0,
+      "logps/rejected": -976.0,
+      "loss": 0.4245,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -8.3125,
+      "step": 10840
+    },
+    {
+      "epoch": 0.4038636913513614,
+      "grad_norm": 6.6084839877862365,
+      "learning_rate": 3.7207240073364334e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -836.0,
+      "logps/rejected": -968.0,
+      "loss": 0.4122,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.6875,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -8.25,
+      "step": 10850
+    },
+    {
+      "epoch": 0.40423591595168523,
+      "grad_norm": 7.943948882425527,
+      "learning_rate": 3.7178881585222035e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.3997,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.25,
+      "step": 10860
+    },
+    {
+      "epoch": 0.40460814055200905,
+      "grad_norm": 7.236277675557204,
+      "learning_rate": 3.7150502535032825e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.4237,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.21875,
+      "rewards/rejected": -8.375,
+      "step": 10870
+    },
+    {
+      "epoch": 0.40498036515233293,
+      "grad_norm": 6.420221226781848,
+      "learning_rate": 3.7122102970710065e-07,
+      "logits/chosen": -4.125,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4163,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -8.875,
+      "step": 10880
+    },
+    {
+      "epoch": 0.40535258975265676,
+      "grad_norm": 6.498767483538408,
+      "learning_rate": 3.7093682940201803e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.921875,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3978,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -8.6875,
+      "step": 10890
+    },
+    {
+      "epoch": 0.4057248143529806,
+      "grad_norm": 6.591848975257687,
+      "learning_rate": 3.7065242491490576e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4087,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -8.8125,
+      "step": 10900
+    },
+    {
+      "epoch": 0.4060970389533044,
+      "grad_norm": 7.232364710768949,
+      "learning_rate": 3.703678167259344e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4099,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.125,
+      "step": 10910
+    },
+    {
+      "epoch": 0.4064692635536283,
+      "grad_norm": 6.3469364138211715,
+      "learning_rate": 3.700830053156182e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.921875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.408,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -8.875,
+      "step": 10920
+    },
+    {
+      "epoch": 0.4068414881539521,
+      "grad_norm": 6.818244911795219,
+      "learning_rate": 3.6979799116481457e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3782,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.9453125,
+      "rewards/rejected": -9.0625,
+      "step": 10930
+    },
+    {
+      "epoch": 0.4072137127542759,
+      "grad_norm": 5.573239320530639,
+      "learning_rate": 3.6951277475472333e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4023,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -8.8125,
+      "step": 10940
+    },
+    {
+      "epoch": 0.40758593735459975,
+      "grad_norm": 8.264525110485382,
+      "learning_rate": 3.6922735656688545e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3815,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -8.8125,
+      "step": 10950
+    },
+    {
+      "epoch": 0.4079581619549236,
+      "grad_norm": 5.763152866962167,
+      "learning_rate": 3.6894173708318305e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -832.0,
+      "logps/rejected": -976.0,
+      "loss": 0.4002,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.5625,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -8.125,
+      "step": 10960
+    },
+    {
+      "epoch": 0.40833038655524745,
+      "grad_norm": 6.475956590857558,
+      "learning_rate": 3.686559167858377e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3992,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -8.5,
+      "step": 10970
+    },
+    {
+      "epoch": 0.40870261115557127,
+      "grad_norm": 5.942462445160721,
+      "learning_rate": 3.683698961574102e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4012,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -8.75,
+      "step": 10980
+    },
+    {
+      "epoch": 0.4090748357558951,
+      "grad_norm": 6.662979366187995,
+      "learning_rate": 3.680836756807995e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4168,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -8.6875,
+      "step": 10990
+    },
+    {
+      "epoch": 0.4094470603562189,
+      "grad_norm": 7.7282084896525856,
+      "learning_rate": 3.677972558392421e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3928,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -8.8125,
+      "step": 11000
+    },
+    {
+      "epoch": 0.4098192849565428,
+      "grad_norm": 6.433159281650206,
+      "learning_rate": 3.675106371163109e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4313,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -8.75,
+      "step": 11010
+    },
+    {
+      "epoch": 0.4101915095568666,
+      "grad_norm": 5.969507914737734,
+      "learning_rate": 3.672238199959146e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -836.0,
+      "logps/rejected": -984.0,
+      "loss": 0.3947,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -8.375,
+      "step": 11020
+    },
+    {
+      "epoch": 0.41056373415719044,
+      "grad_norm": 6.113990119094154,
+      "learning_rate": 3.6693680496229717e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -876.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4159,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -8.3125,
+      "step": 11030
+    },
+    {
+      "epoch": 0.41093595875751426,
+      "grad_norm": 6.5558802486834935,
+      "learning_rate": 3.6664959250003623e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.4079,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -8.5625,
+      "step": 11040
+    },
+    {
+      "epoch": 0.41130818335783814,
+      "grad_norm": 6.238805184196473,
+      "learning_rate": 3.663621830940432e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4254,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -8.75,
+      "step": 11050
+    },
+    {
+      "epoch": 0.41168040795816196,
+      "grad_norm": 6.635715430655651,
+      "learning_rate": 3.6607457722956163e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.4111,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.2265625,
+      "rewards/rejected": -8.4375,
+      "step": 11060
+    },
+    {
+      "epoch": 0.4120526325584858,
+      "grad_norm": 6.8325176519490025,
+      "learning_rate": 3.6578677539216704e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.953125,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4103,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -8.5,
+      "step": 11070
+    },
+    {
+      "epoch": 0.4124248571588096,
+      "grad_norm": 8.600177716278951,
+      "learning_rate": 3.6549877806776555e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -856.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.4152,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -8.4375,
+      "step": 11080
+    },
+    {
+      "epoch": 0.4127970817591335,
+      "grad_norm": 7.092758630206045,
+      "learning_rate": 3.6521058574259355e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4342,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -8.5625,
+      "step": 11090
+    },
+    {
+      "epoch": 0.4131693063594573,
+      "grad_norm": 6.3969931507011,
+      "learning_rate": 3.6492219890321674e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.4268,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -8.3125,
+      "step": 11100
+    },
+    {
+      "epoch": 0.41354153095978113,
+      "grad_norm": 5.658395164903331,
+      "learning_rate": 3.6463361803652877e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -860.0,
+      "logps/rejected": -976.0,
+      "loss": 0.3833,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -8.0625,
+      "step": 11110
+    },
+    {
+      "epoch": 0.41391375556010496,
+      "grad_norm": 7.08682175998033,
+      "learning_rate": 3.6434484362975135e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3998,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.265625,
+      "rewards/rejected": -8.625,
+      "step": 11120
+    },
+    {
+      "epoch": 0.4142859801604288,
+      "grad_norm": 7.435242637533934,
+      "learning_rate": 3.640558761704328e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.4218,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.6875,
+      "step": 11130
+    },
+    {
+      "epoch": 0.41465820476075266,
+      "grad_norm": 7.875089854942056,
+      "learning_rate": 3.6376671614644726e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -856.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.408,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -8.4375,
+      "step": 11140
+    },
+    {
+      "epoch": 0.4150304293610765,
+      "grad_norm": 6.559910759253231,
+      "learning_rate": 3.6347736404599404e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4253,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -8.5625,
+      "step": 11150
+    },
+    {
+      "epoch": 0.4154026539614003,
+      "grad_norm": 7.696509083770465,
+      "learning_rate": 3.631878203575969e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3837,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -8.6875,
+      "step": 11160
+    },
+    {
+      "epoch": 0.4157748785617241,
+      "grad_norm": 6.488778256005278,
+      "learning_rate": 3.628980855701028e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3932,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -8.5625,
+      "step": 11170
+    },
+    {
+      "epoch": 0.416147103162048,
+      "grad_norm": 5.512923982266312,
+      "learning_rate": 3.626081601726816e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4059,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -8.5625,
+      "step": 11180
+    },
+    {
+      "epoch": 0.4165193277623718,
+      "grad_norm": 6.854984497702953,
+      "learning_rate": 3.623180446548248e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4101,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -8.4375,
+      "step": 11190
+    },
+    {
+      "epoch": 0.41689155236269565,
+      "grad_norm": 6.547269073537186,
+      "learning_rate": 3.620277395063449e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -844.0,
+      "logps/rejected": -996.0,
+      "loss": 0.3848,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.6875,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -8.375,
+      "step": 11200
+    },
+    {
+      "epoch": 0.4172637769630195,
+      "grad_norm": 6.3116315324294865,
+      "learning_rate": 3.6173724521737467e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3913,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.75,
+      "step": 11210
+    },
+    {
+      "epoch": 0.4176360015633433,
+      "grad_norm": 6.280073811486498,
+      "learning_rate": 3.61446562278366e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4195,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.875,
+      "step": 11220
+    },
+    {
+      "epoch": 0.4180082261636672,
+      "grad_norm": 5.729524434327027,
+      "learning_rate": 3.6115569118008966e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3909,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -8.875,
+      "step": 11230
+    },
+    {
+      "epoch": 0.418380450763991,
+      "grad_norm": 6.663094828679094,
+      "learning_rate": 3.6086463241363365e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4134,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -8.875,
+      "step": 11240
+    },
+    {
+      "epoch": 0.4187526753643148,
+      "grad_norm": 6.335296904615711,
+      "learning_rate": 3.605733864704031e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.4177,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.1875,
+      "step": 11250
+    },
+    {
+      "epoch": 0.41912489996463864,
+      "grad_norm": 6.4824982688731,
+      "learning_rate": 3.602819538421191e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4008,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -8.5625,
+      "step": 11260
+    },
+    {
+      "epoch": 0.4194971245649625,
+      "grad_norm": 6.43990169822606,
+      "learning_rate": 3.5999033502081783e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -860.0,
+      "logps/rejected": -980.0,
+      "loss": 0.4169,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.375,
+      "rewards/rejected": -8.1875,
+      "step": 11270
+    },
+    {
+      "epoch": 0.41986934916528634,
+      "grad_norm": 5.902525567368692,
+      "learning_rate": 3.596985304988501e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4011,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -8.5625,
+      "step": 11280
+    },
+    {
+      "epoch": 0.42024157376561017,
+      "grad_norm": 7.472901600971799,
+      "learning_rate": 3.5940654076887977e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.4213,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -8.5625,
+      "step": 11290
+    },
+    {
+      "epoch": 0.420613798365934,
+      "grad_norm": 6.8218428066398875,
+      "learning_rate": 3.5911436632388396e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.4016,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -8.5,
+      "step": 11300
+    },
+    {
+      "epoch": 0.42098602296625787,
+      "grad_norm": 8.301590494880626,
+      "learning_rate": 3.588220076571513e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4188,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -8.75,
+      "step": 11310
+    },
+    {
+      "epoch": 0.4213582475665817,
+      "grad_norm": 6.2147088452492545,
+      "learning_rate": 3.5852946526228136e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.96875,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.3812,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -8.375,
+      "step": 11320
+    },
+    {
+      "epoch": 0.4217304721669055,
+      "grad_norm": 7.2286454887347125,
+      "learning_rate": 3.582367396331842e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4022,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -8.6875,
+      "step": 11330
+    },
+    {
+      "epoch": 0.42210269676722934,
+      "grad_norm": 6.510837401631301,
+      "learning_rate": 3.5794383126407905e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3872,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -8.75,
+      "step": 11340
+    },
+    {
+      "epoch": 0.42247492136755316,
+      "grad_norm": 6.730071761397383,
+      "learning_rate": 3.5765074064949376e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3794,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -8.9375,
+      "step": 11350
+    },
+    {
+      "epoch": 0.42284714596787704,
+      "grad_norm": 8.268960838735984,
+      "learning_rate": 3.573574682842637e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3927,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.1875,
+      "step": 11360
+    },
+    {
+      "epoch": 0.42321937056820086,
+      "grad_norm": 8.242338172819547,
+      "learning_rate": 3.5706401466353146e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.418,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.5,
+      "step": 11370
+    },
+    {
+      "epoch": 0.4235915951685247,
+      "grad_norm": 5.98485116947097,
+      "learning_rate": 3.567703802827453e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4131,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -8.5625,
+      "step": 11380
+    },
+    {
+      "epoch": 0.4239638197688485,
+      "grad_norm": 7.085582904588452,
+      "learning_rate": 3.564765656376587e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4028,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -8.625,
+      "step": 11390
+    },
+    {
+      "epoch": 0.4243360443691724,
+      "grad_norm": 7.086842233489581,
+      "learning_rate": 3.561825712243297e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3838,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -8.75,
+      "step": 11400
+    },
+    {
+      "epoch": 0.4247082689694962,
+      "grad_norm": 7.772866376047485,
+      "learning_rate": 3.5588839753911964e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3828,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.9375,
+      "step": 11410
+    },
+    {
+      "epoch": 0.42508049356982003,
+      "grad_norm": 6.025987701400948,
+      "learning_rate": 3.555940450786926e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4163,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -9.1875,
+      "step": 11420
+    },
+    {
+      "epoch": 0.42545271817014385,
+      "grad_norm": 6.52332487394145,
+      "learning_rate": 3.5529951434001453e-07,
+      "logits/chosen": -4.09375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.4098,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -8.875,
+      "step": 11430
+    },
+    {
+      "epoch": 0.4258249427704677,
+      "grad_norm": 6.391070174335603,
+      "learning_rate": 3.5500480582035236e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.405,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -8.5625,
+      "step": 11440
+    },
+    {
+      "epoch": 0.42619716737079155,
+      "grad_norm": 6.794901532120129,
+      "learning_rate": 3.547099200172731e-07,
+      "logits/chosen": -4.09375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3967,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -8.75,
+      "step": 11450
+    },
+    {
+      "epoch": 0.4265693919711154,
+      "grad_norm": 7.070180251968737,
+      "learning_rate": 3.544148574286432e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3961,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -8.625,
+      "step": 11460
+    },
+    {
+      "epoch": 0.4269416165714392,
+      "grad_norm": 6.888236186071729,
+      "learning_rate": 3.541196185526275e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3888,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -8.6875,
+      "step": 11470
+    },
+    {
+      "epoch": 0.427313841171763,
+      "grad_norm": 6.815452822485508,
+      "learning_rate": 3.538242038876885e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3992,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.0625,
+      "step": 11480
+    },
+    {
+      "epoch": 0.4276860657720869,
+      "grad_norm": 6.007329183570532,
+      "learning_rate": 3.5352861393258535e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4028,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.75,
+      "step": 11490
+    },
+    {
+      "epoch": 0.4280582903724107,
+      "grad_norm": 7.180646487065023,
+      "learning_rate": 3.5323284918637346e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3968,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -8.5625,
+      "step": 11500
+    },
+    {
+      "epoch": 0.42843051497273454,
+      "grad_norm": 7.710644278996495,
+      "learning_rate": 3.529369101484031e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -852.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4339,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -8.5625,
+      "step": 11510
+    },
+    {
+      "epoch": 0.42880273957305837,
+      "grad_norm": 6.710524331773978,
+      "learning_rate": 3.526407973183188e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3814,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -8.8125,
+      "step": 11520
+    },
+    {
+      "epoch": 0.42917496417338225,
+      "grad_norm": 7.344507172595124,
+      "learning_rate": 3.5234451119605864e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3969,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.25,
+      "step": 11530
+    },
+    {
+      "epoch": 0.42954718877370607,
+      "grad_norm": 6.458039134295703,
+      "learning_rate": 3.520480522818532e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4044,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.0625,
+      "step": 11540
+    },
+    {
+      "epoch": 0.4299194133740299,
+      "grad_norm": 6.366652978265848,
+      "learning_rate": 3.517514210762248e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.427,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -8.6875,
+      "step": 11550
+    },
+    {
+      "epoch": 0.4302916379743537,
+      "grad_norm": 6.0544512660302985,
+      "learning_rate": 3.514546180799867e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.4197,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -8.625,
+      "step": 11560
+    },
+    {
+      "epoch": 0.43066386257467754,
+      "grad_norm": 6.021327109199788,
+      "learning_rate": 3.5115764379424204e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.4022,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -8.5,
+      "step": 11570
+    },
+    {
+      "epoch": 0.4310360871750014,
+      "grad_norm": 6.986428522047901,
+      "learning_rate": 3.508604987203834e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4192,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -8.75,
+      "step": 11580
+    },
+    {
+      "epoch": 0.43140831177532524,
+      "grad_norm": 7.596201585770056,
+      "learning_rate": 3.505631833600914e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4065,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -8.8125,
+      "step": 11590
+    },
+    {
+      "epoch": 0.43178053637564906,
+      "grad_norm": 6.635880522010002,
+      "learning_rate": 3.5026569821533445e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3653,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.25,
+      "step": 11600
+    },
+    {
+      "epoch": 0.4321527609759729,
+      "grad_norm": 6.337035010996294,
+      "learning_rate": 3.4996804378836755e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3944,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 2.03125,
+      "rewards/rejected": -9.1875,
+      "step": 11610
+    },
+    {
+      "epoch": 0.43252498557629676,
+      "grad_norm": 7.555615364918259,
+      "learning_rate": 3.496702205817313e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3801,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -8.6875,
+      "step": 11620
+    },
+    {
+      "epoch": 0.4328972101766206,
+      "grad_norm": 6.718830120269611,
+      "learning_rate": 3.4937222909825153e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4043,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.96875,
+      "rewards/rejected": -9.0,
+      "step": 11630
+    },
+    {
+      "epoch": 0.4332694347769444,
+      "grad_norm": 6.741132147475561,
+      "learning_rate": 3.4907406984103794e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3933,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.828125,
+      "rewards/rejected": -9.0625,
+      "step": 11640
+    },
+    {
+      "epoch": 0.43364165937726823,
+      "grad_norm": 6.687695972980779,
+      "learning_rate": 3.4877574331348376e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3755,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.0,
+      "step": 11650
+    },
+    {
+      "epoch": 0.43401388397759205,
+      "grad_norm": 6.5494745816039,
+      "learning_rate": 3.484772500192643e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4192,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -8.875,
+      "step": 11660
+    },
+    {
+      "epoch": 0.43438610857791593,
+      "grad_norm": 6.055053654173566,
+      "learning_rate": 3.481785904623368e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -852.0,
+      "logps/rejected": -976.0,
+      "loss": 0.4159,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -6.625,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -8.3125,
+      "step": 11670
+    },
+    {
+      "epoch": 0.43475833317823975,
+      "grad_norm": 6.312866434204354,
+      "learning_rate": 3.4787976514693873e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -864.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4042,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -8.1875,
+      "step": 11680
+    },
+    {
+      "epoch": 0.4351305577785636,
+      "grad_norm": 6.79282235431904,
+      "learning_rate": 3.475807745775879e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.4117,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": -8.375,
+      "step": 11690
+    },
+    {
+      "epoch": 0.4355027823788874,
+      "grad_norm": 5.734823622141035,
+      "learning_rate": 3.4728161925908093e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3929,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -8.8125,
+      "step": 11700
+    },
+    {
+      "epoch": 0.4358750069792113,
+      "grad_norm": 6.955294931778068,
+      "learning_rate": 3.4698229969649246e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -872.0,
+      "logps/rejected": -996.0,
+      "loss": 0.4077,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -8.3125,
+      "step": 11710
+    },
+    {
+      "epoch": 0.4362472315795351,
+      "grad_norm": 6.732014289225819,
+      "learning_rate": 3.466828163951747e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.4203,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -8.375,
+      "step": 11720
+    },
+    {
+      "epoch": 0.4366194561798589,
+      "grad_norm": 6.593255125872756,
+      "learning_rate": 3.46383169860756e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.3862,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -8.4375,
+      "step": 11730
+    },
+    {
+      "epoch": 0.43699168078018275,
+      "grad_norm": 6.4191259891609205,
+      "learning_rate": 3.460833605991405e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3994,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -8.8125,
+      "step": 11740
+    },
+    {
+      "epoch": 0.4373639053805066,
+      "grad_norm": 8.81076635751982,
+      "learning_rate": 3.4578338911650713e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4539,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.125,
+      "step": 11750
+    },
+    {
+      "epoch": 0.43773612998083045,
+      "grad_norm": 7.2599110967793,
+      "learning_rate": 3.454832559193085e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3888,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -9.375,
+      "step": 11760
+    },
+    {
+      "epoch": 0.43810835458115427,
+      "grad_norm": 7.298180359939871,
+      "learning_rate": 3.4518296151427036e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3824,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -8.875,
+      "step": 11770
+    },
+    {
+      "epoch": 0.4384805791814781,
+      "grad_norm": 8.408466939444446,
+      "learning_rate": 3.448825064083907e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3995,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -8.625,
+      "step": 11780
+    },
+    {
+      "epoch": 0.4388528037818019,
+      "grad_norm": 7.267261745415541,
+      "learning_rate": 3.445818911089387e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3814,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -8.75,
+      "step": 11790
+    },
+    {
+      "epoch": 0.4392250283821258,
+      "grad_norm": 5.746071784931887,
+      "learning_rate": 3.4428111612345413e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4103,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -8.8125,
+      "step": 11800
+    },
+    {
+      "epoch": 0.4395972529824496,
+      "grad_norm": 5.86383513948349,
+      "learning_rate": 3.439801819597462e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.984375,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4222,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -8.8125,
+      "step": 11810
+    },
+    {
+      "epoch": 0.43996947758277344,
+      "grad_norm": 6.772936580351455,
+      "learning_rate": 3.4367908912589307e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -860.0,
+      "logps/rejected": -988.0,
+      "loss": 0.4007,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -8.375,
+      "step": 11820
+    },
+    {
+      "epoch": 0.44034170218309726,
+      "grad_norm": 6.1945131757780105,
+      "learning_rate": 3.4337783813024057e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4001,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -8.5,
+      "step": 11830
+    },
+    {
+      "epoch": 0.44071392678342114,
+      "grad_norm": 6.862471970226016,
+      "learning_rate": 3.430764294814017e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3806,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -8.9375,
+      "step": 11840
+    },
+    {
+      "epoch": 0.44108615138374496,
+      "grad_norm": 6.509274860036443,
+      "learning_rate": 3.4277486368825557e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4051,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.0,
+      "step": 11850
+    },
+    {
+      "epoch": 0.4414583759840688,
+      "grad_norm": 7.389703915353271,
+      "learning_rate": 3.4247314125994677e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3721,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -8.875,
+      "step": 11860
+    },
+    {
+      "epoch": 0.4418306005843926,
+      "grad_norm": 6.764597605776528,
+      "learning_rate": 3.421712627058841e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.3963,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -8.5,
+      "step": 11870
+    },
+    {
+      "epoch": 0.44220282518471643,
+      "grad_norm": 7.350574701122705,
+      "learning_rate": 3.4186922853573996e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -860.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.3817,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -8.5625,
+      "step": 11880
+    },
+    {
+      "epoch": 0.4425750497850403,
+      "grad_norm": 6.904535173446593,
+      "learning_rate": 3.415670392594498e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.399,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -9.0625,
+      "step": 11890
+    },
+    {
+      "epoch": 0.44294727438536413,
+      "grad_norm": 6.771487081602254,
+      "learning_rate": 3.4126469538721053e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4147,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -8.6875,
+      "step": 11900
+    },
+    {
+      "epoch": 0.44331949898568795,
+      "grad_norm": 5.470812682079132,
+      "learning_rate": 3.409621974294804e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3776,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.921875,
+      "rewards/rejected": -9.0625,
+      "step": 11910
+    },
+    {
+      "epoch": 0.4436917235860118,
+      "grad_norm": 5.431784429199158,
+      "learning_rate": 3.4065954589697753e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.4329,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -8.5,
+      "step": 11920
+    },
+    {
+      "epoch": 0.44406394818633566,
+      "grad_norm": 5.338494629109279,
+      "learning_rate": 3.403567413006796e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4113,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -8.4375,
+      "step": 11930
+    },
+    {
+      "epoch": 0.4444361727866595,
+      "grad_norm": 6.759455649052316,
+      "learning_rate": 3.400537841518224e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3947,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.5,
+      "step": 11940
+    },
+    {
+      "epoch": 0.4448083973869833,
+      "grad_norm": 6.271101849982418,
+      "learning_rate": 3.3975067496189963e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4002,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -8.6875,
+      "step": 11950
+    },
+    {
+      "epoch": 0.4451806219873071,
+      "grad_norm": 6.80850833405418,
+      "learning_rate": 3.394474142426615e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.9375,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3824,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -8.625,
+      "step": 11960
+    },
+    {
+      "epoch": 0.445552846587631,
+      "grad_norm": 7.235157172512589,
+      "learning_rate": 3.39144002506114e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4182,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -8.6875,
+      "step": 11970
+    },
+    {
+      "epoch": 0.4459250711879548,
+      "grad_norm": 7.541911784147671,
+      "learning_rate": 3.388404402645182e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3932,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -6.8125,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -8.5625,
+      "step": 11980
+    },
+    {
+      "epoch": 0.44629729578827865,
+      "grad_norm": 6.838470908312442,
+      "learning_rate": 3.3853672803038916e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.4229,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.5625,
+      "step": 11990
+    },
+    {
+      "epoch": 0.44666952038860247,
+      "grad_norm": 7.647284370875841,
+      "learning_rate": 3.382328663164954e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4084,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.6875,
+      "step": 12000
+    },
+    {
+      "epoch": 0.4470417449889263,
+      "grad_norm": 6.083463324058337,
+      "learning_rate": 3.379288556358574e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.411,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.625,
+      "step": 12010
+    },
+    {
+      "epoch": 0.44741396958925017,
+      "grad_norm": 6.37327251697913,
+      "learning_rate": 3.376246965017476e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.953125,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.4039,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -8.5625,
+      "step": 12020
+    },
+    {
+      "epoch": 0.447786194189574,
+      "grad_norm": 6.626369617816579,
+      "learning_rate": 3.3732038942768895e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.3928,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.5625,
+      "step": 12030
+    },
+    {
+      "epoch": 0.4481584187898978,
+      "grad_norm": 8.323270022542616,
+      "learning_rate": 3.3701593492745386e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -848.0,
+      "logps/rejected": -992.0,
+      "loss": 0.4257,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.5,
+      "step": 12040
+    },
+    {
+      "epoch": 0.44853064339022164,
+      "grad_norm": 6.454485607014557,
+      "learning_rate": 3.36711333515064e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3841,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -8.6875,
+      "step": 12050
+    },
+    {
+      "epoch": 0.4489028679905455,
+      "grad_norm": 5.553920606747376,
+      "learning_rate": 3.364065857047891e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3976,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -8.6875,
+      "step": 12060
+    },
+    {
+      "epoch": 0.44927509259086934,
+      "grad_norm": 7.324551460649139,
+      "learning_rate": 3.361016920111458e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3907,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -9.0625,
+      "step": 12070
+    },
+    {
+      "epoch": 0.44964731719119316,
+      "grad_norm": 7.364201298314512,
+      "learning_rate": 3.3579665294889714e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4086,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.125,
+      "step": 12080
+    },
+    {
+      "epoch": 0.450019541791517,
+      "grad_norm": 6.033128198507223,
+      "learning_rate": 3.354914690330517e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3998,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.0625,
+      "step": 12090
+    },
+    {
+      "epoch": 0.45039176639184086,
+      "grad_norm": 6.265204334716435,
+      "learning_rate": 3.3518614077886246e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.3849,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -8.625,
+      "step": 12100
+    },
+    {
+      "epoch": 0.4507639909921647,
+      "grad_norm": 6.405911221847536,
+      "learning_rate": 3.348806687018262e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3928,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -8.5,
+      "step": 12110
+    },
+    {
+      "epoch": 0.4511362155924885,
+      "grad_norm": 8.220475361194373,
+      "learning_rate": 3.3457505331768254e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4217,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -8.8125,
+      "step": 12120
+    },
+    {
+      "epoch": 0.45150844019281233,
+      "grad_norm": 7.212318421612806,
+      "learning_rate": 3.3426929514241294e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.875,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3888,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.1875,
+      "step": 12130
+    },
+    {
+      "epoch": 0.45188066479313616,
+      "grad_norm": 6.609095144366113,
+      "learning_rate": 3.3396339469224e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4271,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.0,
+      "step": 12140
+    },
+    {
+      "epoch": 0.45225288939346003,
+      "grad_norm": 6.736685892609171,
+      "learning_rate": 3.336573524836266e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4035,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.3125,
+      "step": 12150
+    },
+    {
+      "epoch": 0.45262511399378386,
+      "grad_norm": 6.212656206276573,
+      "learning_rate": 3.3335116903327485e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3933,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -8.75,
+      "step": 12160
+    },
+    {
+      "epoch": 0.4529973385941077,
+      "grad_norm": 6.5944710708386,
+      "learning_rate": 3.330448448581254e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.4112,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.5,
+      "step": 12170
+    },
+    {
+      "epoch": 0.4533695631944315,
+      "grad_norm": 8.380177448591247,
+      "learning_rate": 3.327383804753563e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3953,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -8.625,
+      "step": 12180
+    },
+    {
+      "epoch": 0.4537417877947554,
+      "grad_norm": 6.595114343936548,
+      "learning_rate": 3.3243177640238263e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.391,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 2.078125,
+      "rewards/rejected": -9.1875,
+      "step": 12190
+    },
+    {
+      "epoch": 0.4541140123950792,
+      "grad_norm": 8.15009807353792,
+      "learning_rate": 3.32125033156855e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3962,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -9.0625,
+      "step": 12200
+    },
+    {
+      "epoch": 0.454486236995403,
+      "grad_norm": 8.100630262206934,
+      "learning_rate": 3.3181815125665943e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3972,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.984375,
+      "rewards/rejected": -9.25,
+      "step": 12210
+    },
+    {
+      "epoch": 0.45485846159572685,
+      "grad_norm": 6.605985772792216,
+      "learning_rate": 3.315111312199154e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4019,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -8.875,
+      "step": 12220
+    },
+    {
+      "epoch": 0.45523068619605067,
+      "grad_norm": 6.483013658820195,
+      "learning_rate": 3.312039735649761e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -860.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3961,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -6.71875,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": -8.6875,
+      "step": 12230
+    },
+    {
+      "epoch": 0.45560291079637455,
+      "grad_norm": 7.2980888956191405,
+      "learning_rate": 3.308966788104271e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4055,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -8.875,
+      "step": 12240
+    },
+    {
+      "epoch": 0.45597513539669837,
+      "grad_norm": 8.01813755943812,
+      "learning_rate": 3.305892474750849e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3954,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -8.75,
+      "step": 12250
+    },
+    {
+      "epoch": 0.4563473599970222,
+      "grad_norm": 7.782975124175755,
+      "learning_rate": 3.3028168007799736e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4155,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -8.875,
+      "step": 12260
+    },
+    {
+      "epoch": 0.456719584597346,
+      "grad_norm": 6.814143219773761,
+      "learning_rate": 3.299739771384413e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4141,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -8.75,
+      "step": 12270
+    },
+    {
+      "epoch": 0.4570918091976699,
+      "grad_norm": 6.4585312813292735,
+      "learning_rate": 3.296661391759229e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4109,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -8.625,
+      "step": 12280
+    },
+    {
+      "epoch": 0.4574640337979937,
+      "grad_norm": 6.518093987936251,
+      "learning_rate": 3.2935816671017624e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3736,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -8.875,
+      "step": 12290
+    },
+    {
+      "epoch": 0.45783625839831754,
+      "grad_norm": 9.594075542197025,
+      "learning_rate": 3.290500602611622e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3968,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.0625,
+      "step": 12300
+    },
+    {
+      "epoch": 0.45820848299864136,
+      "grad_norm": 6.579180903035412,
+      "learning_rate": 3.2874182034906816e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4095,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -8.9375,
+      "step": 12310
+    },
+    {
+      "epoch": 0.45858070759896524,
+      "grad_norm": 5.776181335611089,
+      "learning_rate": 3.2843344749430666e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.4018,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.5,
+      "step": 12320
+    },
+    {
+      "epoch": 0.45895293219928907,
+      "grad_norm": 7.145279810794003,
+      "learning_rate": 3.28124942217515e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.4137,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -8.625,
+      "step": 12330
+    },
+    {
+      "epoch": 0.4593251567996129,
+      "grad_norm": 6.571820871824894,
+      "learning_rate": 3.2781630503955344e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4007,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -8.9375,
+      "step": 12340
+    },
+    {
+      "epoch": 0.4596973813999367,
+      "grad_norm": 7.288027414885038,
+      "learning_rate": 3.275075364815056e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.391,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -9.0,
+      "step": 12350
+    },
+    {
+      "epoch": 0.46006960600026053,
+      "grad_norm": 6.211743239121556,
+      "learning_rate": 3.271986370646764e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3823,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.0,
+      "step": 12360
+    },
+    {
+      "epoch": 0.4604418306005844,
+      "grad_norm": 7.189661480883692,
+      "learning_rate": 3.2688960731059204e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.90625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3859,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.75,
+      "step": 12370
+    },
+    {
+      "epoch": 0.46081405520090823,
+      "grad_norm": 7.7777417284402075,
+      "learning_rate": 3.2658044774099875e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3962,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.0625,
+      "step": 12380
+    },
+    {
+      "epoch": 0.46118627980123206,
+      "grad_norm": 7.651220886293237,
+      "learning_rate": 3.262711588778616e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.389,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.8125,
+      "step": 12390
+    },
+    {
+      "epoch": 0.4615585044015559,
+      "grad_norm": 7.627509775274965,
+      "learning_rate": 3.259617412433644e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4007,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -9.0,
+      "step": 12400
+    },
+    {
+      "epoch": 0.46193072900187976,
+      "grad_norm": 7.43723128599684,
+      "learning_rate": 3.25652195359908e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3949,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -8.9375,
+      "step": 12410
+    },
+    {
+      "epoch": 0.4623029536022036,
+      "grad_norm": 7.458451346683611,
+      "learning_rate": 3.2534252175010994e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.4019,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -8.5625,
+      "step": 12420
+    },
+    {
+      "epoch": 0.4626751782025274,
+      "grad_norm": 5.765995983801743,
+      "learning_rate": 3.2503272093680347e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.4091,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.5,
+      "step": 12430
+    },
+    {
+      "epoch": 0.4630474028028512,
+      "grad_norm": 6.298829065535838,
+      "learning_rate": 3.247227934430364e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3657,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -8.6875,
+      "step": 12440
+    },
+    {
+      "epoch": 0.46341962740317505,
+      "grad_norm": 6.431434763296798,
+      "learning_rate": 3.2441273979207074e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3822,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -8.8125,
+      "step": 12450
+    },
+    {
+      "epoch": 0.46379185200349893,
+      "grad_norm": 8.507177938511202,
+      "learning_rate": 3.2410256050738107e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.395,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.125,
+      "step": 12460
+    },
+    {
+      "epoch": 0.46416407660382275,
+      "grad_norm": 6.640655048480872,
+      "learning_rate": 3.237922561126545e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3876,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.0625,
+      "step": 12470
+    },
+    {
+      "epoch": 0.4645363012041466,
+      "grad_norm": 7.602003158744994,
+      "learning_rate": 3.23481827131789e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.875,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4214,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.125,
+      "step": 12480
+    },
+    {
+      "epoch": 0.4649085258044704,
+      "grad_norm": 5.8056812809450555,
+      "learning_rate": 3.2317127408889327e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3971,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -8.875,
+      "step": 12490
+    },
+    {
+      "epoch": 0.4652807504047943,
+      "grad_norm": 6.203526634243301,
+      "learning_rate": 3.228605975082851e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.394,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -8.5,
+      "step": 12500
+    },
+    {
+      "epoch": 0.4656529750051181,
+      "grad_norm": 7.085750730027124,
+      "learning_rate": 3.225497979144911e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4059,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -9.0625,
+      "step": 12510
+    },
+    {
+      "epoch": 0.4660251996054419,
+      "grad_norm": 6.819002495727156,
+      "learning_rate": 3.222388758322454e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.9375,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4248,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.1875,
+      "step": 12520
+    },
+    {
+      "epoch": 0.46639742420576574,
+      "grad_norm": 7.365070395464945,
+      "learning_rate": 3.219278317864891e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4127,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -8.875,
+      "step": 12530
+    },
+    {
+      "epoch": 0.4667696488060896,
+      "grad_norm": 7.3870441506528755,
+      "learning_rate": 3.2161666630236913e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3765,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -8.8125,
+      "step": 12540
+    },
+    {
+      "epoch": 0.46714187340641344,
+      "grad_norm": 7.185447895897448,
+      "learning_rate": 3.213053799052373e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3796,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -8.9375,
+      "step": 12550
+    },
+    {
+      "epoch": 0.46751409800673727,
+      "grad_norm": 5.68511396521476,
+      "learning_rate": 3.209939731206499e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3984,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -8.875,
+      "step": 12560
+    },
+    {
+      "epoch": 0.4678863226070611,
+      "grad_norm": 7.198198273069706,
+      "learning_rate": 3.206824464743662e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3886,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -8.9375,
+      "step": 12570
+    },
+    {
+      "epoch": 0.4682585472073849,
+      "grad_norm": 8.054355419588779,
+      "learning_rate": 3.2037080049234795e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4082,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -8.8125,
+      "step": 12580
+    },
+    {
+      "epoch": 0.4686307718077088,
+      "grad_norm": 5.892042449947636,
+      "learning_rate": 3.2005903570075826e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.984375,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3999,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.125,
+      "step": 12590
+    },
+    {
+      "epoch": 0.4690029964080326,
+      "grad_norm": 5.865260354898958,
+      "learning_rate": 3.197471526259611e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4088,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -8.9375,
+      "step": 12600
+    },
+    {
+      "epoch": 0.46937522100835644,
+      "grad_norm": 6.594156402463973,
+      "learning_rate": 3.194351517945197e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -4.09375,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4075,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.265625,
+      "rewards/rejected": -8.6875,
+      "step": 12610
+    },
+    {
+      "epoch": 0.46974744560868026,
+      "grad_norm": 6.512954220848175,
+      "learning_rate": 3.1912303373319654e-07,
+      "logits/chosen": -4.09375,
+      "logits/rejected": -3.9375,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3934,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -8.8125,
+      "step": 12620
+    },
+    {
+      "epoch": 0.47011967020900414,
+      "grad_norm": 6.224345093076355,
+      "learning_rate": 3.1881079896895187e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3809,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -8.6875,
+      "step": 12630
+    },
+    {
+      "epoch": 0.47049189480932796,
+      "grad_norm": 6.782096908382096,
+      "learning_rate": 3.184984480289429e-07,
+      "logits/chosen": -4.15625,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3706,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -8.875,
+      "step": 12640
+    },
+    {
+      "epoch": 0.4708641194096518,
+      "grad_norm": 8.51401362820416,
+      "learning_rate": 3.18185981440523e-07,
+      "logits/chosen": -4.15625,
+      "logits/rejected": -4.03125,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4015,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -8.9375,
+      "step": 12650
+    },
+    {
+      "epoch": 0.4712363440099756,
+      "grad_norm": 9.35535760938938,
+      "learning_rate": 3.1787339973124087e-07,
+      "logits/chosen": -4.09375,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3888,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -8.875,
+      "step": 12660
+    },
+    {
+      "epoch": 0.47160856861029943,
+      "grad_norm": 9.039959859179886,
+      "learning_rate": 3.1756070342883955e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4058,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.875,
+      "step": 12670
+    },
+    {
+      "epoch": 0.4719807932106233,
+      "grad_norm": 6.893091282299248,
+      "learning_rate": 3.1724789306125556e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3879,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -8.9375,
+      "step": 12680
+    },
+    {
+      "epoch": 0.47235301781094713,
+      "grad_norm": 6.949622802357949,
+      "learning_rate": 3.1693496915661776e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3914,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -8.8125,
+      "step": 12690
+    },
+    {
+      "epoch": 0.47272524241127095,
+      "grad_norm": 6.600419829101733,
+      "learning_rate": 3.1662193224324717e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.4084,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.75,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -8.375,
+      "step": 12700
+    },
+    {
+      "epoch": 0.4730974670115948,
+      "grad_norm": 6.168669759880037,
+      "learning_rate": 3.163087828496553e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.4109,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.375,
+      "rewards/rejected": -8.375,
+      "step": 12710
+    },
+    {
+      "epoch": 0.47346969161191865,
+      "grad_norm": 6.599623140371136,
+      "learning_rate": 3.1599552150454357e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3816,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -8.4375,
+      "step": 12720
+    },
+    {
+      "epoch": 0.4738419162122425,
+      "grad_norm": 6.855131597969373,
+      "learning_rate": 3.1568214873680245e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4093,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -8.8125,
+      "step": 12730
+    },
+    {
+      "epoch": 0.4742141408125663,
+      "grad_norm": 6.511510080991066,
+      "learning_rate": 3.1536866507551047e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3817,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -9.125,
+      "step": 12740
+    },
+    {
+      "epoch": 0.4745863654128901,
+      "grad_norm": 6.08175048096123,
+      "learning_rate": 3.150550710499336e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3863,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -9.0,
+      "step": 12750
+    },
+    {
+      "epoch": 0.474958590013214,
+      "grad_norm": 6.399816140501925,
+      "learning_rate": 3.1474136718952384e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.396,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.125,
+      "step": 12760
+    },
+    {
+      "epoch": 0.4753308146135378,
+      "grad_norm": 7.214214506783356,
+      "learning_rate": 3.144275540239189e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.398,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.28125,
+      "rewards/rejected": -8.75,
+      "step": 12770
+    },
+    {
+      "epoch": 0.47570303921386164,
+      "grad_norm": 7.559991830832495,
+      "learning_rate": 3.1411363208294077e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.3952,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -8.4375,
+      "step": 12780
+    },
+    {
+      "epoch": 0.47607526381418547,
+      "grad_norm": 8.52872459705108,
+      "learning_rate": 3.137996018965955e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4308,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -8.6875,
+      "step": 12790
+    },
+    {
+      "epoch": 0.4764474884145093,
+      "grad_norm": 6.985948781124637,
+      "learning_rate": 3.1348546399507137e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3996,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -8.625,
+      "step": 12800
+    },
+    {
+      "epoch": 0.47681971301483317,
+      "grad_norm": 6.729873673956851,
+      "learning_rate": 3.131712189087389e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.3944,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.5625,
+      "step": 12810
+    },
+    {
+      "epoch": 0.477191937615157,
+      "grad_norm": 7.109909968980753,
+      "learning_rate": 3.128568671681494e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -852.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4045,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.71875,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -8.4375,
+      "step": 12820
+    },
+    {
+      "epoch": 0.4775641622154808,
+      "grad_norm": 7.183690820278653,
+      "learning_rate": 3.1254240930403424e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3894,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -8.9375,
+      "step": 12830
+    },
+    {
+      "epoch": 0.47793638681580464,
+      "grad_norm": 7.651837683202167,
+      "learning_rate": 3.122278458473042e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -860.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.3919,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.8125,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -8.625,
+      "step": 12840
+    },
+    {
+      "epoch": 0.4783086114161285,
+      "grad_norm": 6.72730358426319,
+      "learning_rate": 3.1191317732904803e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -860.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4001,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.375,
+      "step": 12850
+    },
+    {
+      "epoch": 0.47868083601645234,
+      "grad_norm": 5.562202844199672,
+      "learning_rate": 3.11598404280532e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.953125,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.3917,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -8.625,
+      "step": 12860
+    },
+    {
+      "epoch": 0.47905306061677616,
+      "grad_norm": 7.398567449737131,
+      "learning_rate": 3.112835272331989e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3866,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.0,
+      "step": 12870
+    },
+    {
+      "epoch": 0.4794252852171,
+      "grad_norm": 7.467933641221229,
+      "learning_rate": 3.1096854671866696e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4153,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -8.75,
+      "step": 12880
+    },
+    {
+      "epoch": 0.4797975098174238,
+      "grad_norm": 7.473429311962256,
+      "learning_rate": 3.106534632687293e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4111,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.359375,
+      "rewards/rejected": -8.8125,
+      "step": 12890
+    },
+    {
+      "epoch": 0.4801697344177477,
+      "grad_norm": 5.046857360975577,
+      "learning_rate": 3.103382774153526e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3775,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.3671875,
+      "rewards/rejected": -8.5625,
+      "step": 12900
+    },
+    {
+      "epoch": 0.4805419590180715,
+      "grad_norm": 7.2772556946750075,
+      "learning_rate": 3.1002298969067676e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4166,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -8.625,
+      "step": 12910
+    },
+    {
+      "epoch": 0.48091418361839533,
+      "grad_norm": 6.828368371772935,
+      "learning_rate": 3.097076006270132e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3963,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -8.6875,
+      "step": 12920
+    },
+    {
+      "epoch": 0.48128640821871915,
+      "grad_norm": 7.400053186729568,
+      "learning_rate": 3.093921107568449e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4117,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -8.6875,
+      "step": 12930
+    },
+    {
+      "epoch": 0.48165863281904303,
+      "grad_norm": 5.577829235822006,
+      "learning_rate": 3.0907652061282467e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.4077,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -8.625,
+      "step": 12940
+    },
+    {
+      "epoch": 0.48203085741936685,
+      "grad_norm": 6.709279093390351,
+      "learning_rate": 3.087608307277749e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -832.0,
+      "logps/rejected": -976.0,
+      "loss": 0.4001,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.5,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -8.1875,
+      "step": 12950
+    },
+    {
+      "epoch": 0.4824030820196907,
+      "grad_norm": 6.251984450490257,
+      "learning_rate": 3.0844504163468634e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3964,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.625,
+      "step": 12960
+    },
+    {
+      "epoch": 0.4827753066200145,
+      "grad_norm": 7.062457537454555,
+      "learning_rate": 3.081291538667169e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4011,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -9.0625,
+      "step": 12970
+    },
+    {
+      "epoch": 0.4831475312203384,
+      "grad_norm": 7.072167019965112,
+      "learning_rate": 3.0781316795719165e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4044,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -8.8125,
+      "step": 12980
+    },
+    {
+      "epoch": 0.4835197558206622,
+      "grad_norm": 7.04881342348994,
+      "learning_rate": 3.0749708443960076e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4092,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -8.9375,
+      "step": 12990
+    },
+    {
+      "epoch": 0.483891980420986,
+      "grad_norm": 7.355018445604345,
+      "learning_rate": 3.071809038475997e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -8.8125,
+      "step": 13000
+    },
+    {
+      "epoch": 0.48426420502130985,
+      "grad_norm": 6.132355405641413,
+      "learning_rate": 3.0686462671500744e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4345,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -8.9375,
+      "step": 13010
+    },
+    {
+      "epoch": 0.48463642962163367,
+      "grad_norm": 6.8525123112205195,
+      "learning_rate": 3.065482535758063e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.4048,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.625,
+      "step": 13020
+    },
+    {
+      "epoch": 0.48500865422195755,
+      "grad_norm": 7.430971869898991,
+      "learning_rate": 3.0623178496414025e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4009,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -8.75,
+      "step": 13030
+    },
+    {
+      "epoch": 0.48538087882228137,
+      "grad_norm": 7.862159378352481,
+      "learning_rate": 3.0591522141431495e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4142,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.8671875,
+      "rewards/rejected": -8.8125,
+      "step": 13040
+    },
+    {
+      "epoch": 0.4857531034226052,
+      "grad_norm": 6.546177146850429,
+      "learning_rate": 3.0559856346079605e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4038,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -8.6875,
+      "step": 13050
+    },
+    {
+      "epoch": 0.486125328022929,
+      "grad_norm": 5.589274012809965,
+      "learning_rate": 3.052818116382086e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.398,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -8.625,
+      "step": 13060
+    },
+    {
+      "epoch": 0.4864975526232529,
+      "grad_norm": 6.084558369243841,
+      "learning_rate": 3.0496496648133623e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3808,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -8.875,
+      "step": 13070
+    },
+    {
+      "epoch": 0.4868697772235767,
+      "grad_norm": 6.017961614205846,
+      "learning_rate": 3.0464802852512e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3852,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -8.9375,
+      "step": 13080
+    },
+    {
+      "epoch": 0.48724200182390054,
+      "grad_norm": 7.874544081895002,
+      "learning_rate": 3.0433099830465784e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3997,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.6875,
+      "step": 13090
+    },
+    {
+      "epoch": 0.48761422642422436,
+      "grad_norm": 6.0594709895588545,
+      "learning_rate": 3.040138763552034e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3695,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.0625,
+      "step": 13100
+    },
+    {
+      "epoch": 0.48798645102454824,
+      "grad_norm": 6.694824975453483,
+      "learning_rate": 3.0369666321216494e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3871,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -9.25,
+      "step": 13110
+    },
+    {
+      "epoch": 0.48835867562487206,
+      "grad_norm": 7.274905715251884,
+      "learning_rate": 3.03379359411105e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4026,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -8.9375,
+      "step": 13120
+    },
+    {
+      "epoch": 0.4887309002251959,
+      "grad_norm": 7.0889423315338425,
+      "learning_rate": 3.0306196548773914e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4004,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -8.875,
+      "step": 13130
+    },
+    {
+      "epoch": 0.4891031248255197,
+      "grad_norm": 6.390432931779669,
+      "learning_rate": 3.027444819779351e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3944,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -8.8125,
+      "step": 13140
+    },
+    {
+      "epoch": 0.48947534942584353,
+      "grad_norm": 6.273661359354513,
+      "learning_rate": 3.024269094177116e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3718,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -8.6875,
+      "step": 13150
+    },
+    {
+      "epoch": 0.4898475740261674,
+      "grad_norm": 6.9996439549691525,
+      "learning_rate": 3.0210924834323794e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3957,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.125,
+      "step": 13160
+    },
+    {
+      "epoch": 0.49021979862649123,
+      "grad_norm": 8.921952239176647,
+      "learning_rate": 3.0179149929083293e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4279,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.0,
+      "step": 13170
+    },
+    {
+      "epoch": 0.49059202322681505,
+      "grad_norm": 6.833806593735758,
+      "learning_rate": 3.014736627969636e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4003,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -8.875,
+      "step": 13180
+    },
+    {
+      "epoch": 0.4909642478271389,
+      "grad_norm": 7.335370522611424,
+      "learning_rate": 3.0115573939824507e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4014,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -8.8125,
+      "step": 13190
+    },
+    {
+      "epoch": 0.49133647242746276,
+      "grad_norm": 6.7882540774683715,
+      "learning_rate": 3.0083772963143866e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4128,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -8.6875,
+      "step": 13200
+    },
+    {
+      "epoch": 0.4917086970277866,
+      "grad_norm": 6.7188755571681975,
+      "learning_rate": 3.00519634033452e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3761,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -8.8125,
+      "step": 13210
+    },
+    {
+      "epoch": 0.4920809216281104,
+      "grad_norm": 7.117010040223568,
+      "learning_rate": 3.002014531413374e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4084,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -8.9375,
+      "step": 13220
+    },
+    {
+      "epoch": 0.4924531462284342,
+      "grad_norm": 6.165706369471509,
+      "learning_rate": 2.99883187492291e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3972,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.0,
+      "step": 13230
+    },
+    {
+      "epoch": 0.49282537082875805,
+      "grad_norm": 6.751520608576958,
+      "learning_rate": 2.995648376236524e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4107,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -8.6875,
+      "step": 13240
+    },
+    {
+      "epoch": 0.4931975954290819,
+      "grad_norm": 6.286419334736631,
+      "learning_rate": 2.992464040729031e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3953,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -8.875,
+      "step": 13250
+    },
+    {
+      "epoch": 0.49356982002940575,
+      "grad_norm": 6.229940154358909,
+      "learning_rate": 2.989278873776661e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4029,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.0,
+      "step": 13260
+    },
+    {
+      "epoch": 0.49394204462972957,
+      "grad_norm": 6.66179650506066,
+      "learning_rate": 2.986092880757046e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3891,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -8.75,
+      "step": 13270
+    },
+    {
+      "epoch": 0.4943142692300534,
+      "grad_norm": 5.597643114110202,
+      "learning_rate": 2.9829060670492134e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3903,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -8.9375,
+      "step": 13280
+    },
+    {
+      "epoch": 0.49468649383037727,
+      "grad_norm": 7.326201588013259,
+      "learning_rate": 2.9797184380335763e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3821,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.953125,
+      "rewards/rejected": -9.3125,
+      "step": 13290
+    },
+    {
+      "epoch": 0.4950587184307011,
+      "grad_norm": 7.111472483298179,
+      "learning_rate": 2.976529999091924e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3797,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.9453125,
+      "rewards/rejected": -9.3125,
+      "step": 13300
+    },
+    {
+      "epoch": 0.4954309430310249,
+      "grad_norm": 7.716132583708576,
+      "learning_rate": 2.9733407556074146e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4005,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.3125,
+      "step": 13310
+    },
+    {
+      "epoch": 0.49580316763134874,
+      "grad_norm": 7.291029333117921,
+      "learning_rate": 2.9701507129645625e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4051,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -9.1875,
+      "step": 13320
+    },
+    {
+      "epoch": 0.4961753922316726,
+      "grad_norm": 6.975640447117475,
+      "learning_rate": 2.966959876549233e-07,
+      "logits/chosen": -4.0625,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3779,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -8.9375,
+      "step": 13330
+    },
+    {
+      "epoch": 0.49654761683199644,
+      "grad_norm": 5.811556544381152,
+      "learning_rate": 2.9637682517486284e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3969,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -8.9375,
+      "step": 13340
+    },
+    {
+      "epoch": 0.49691984143232026,
+      "grad_norm": 6.389579764880233,
+      "learning_rate": 2.9605758439512874e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4088,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -8.9375,
+      "step": 13350
+    },
+    {
+      "epoch": 0.4972920660326441,
+      "grad_norm": 7.639908634178015,
+      "learning_rate": 2.957382658547066e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3881,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -8.9375,
+      "step": 13360
+    },
+    {
+      "epoch": 0.4976642906329679,
+      "grad_norm": 7.9037749476669985,
+      "learning_rate": 2.9541887009271354e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3944,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.0,
+      "step": 13370
+    },
+    {
+      "epoch": 0.4980365152332918,
+      "grad_norm": 8.274766209947646,
+      "learning_rate": 2.95099397648397e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4256,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -8.75,
+      "step": 13380
+    },
+    {
+      "epoch": 0.4984087398336156,
+      "grad_norm": 6.939253435817808,
+      "learning_rate": 2.947798490611339e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4107,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -8.6875,
+      "step": 13390
+    },
+    {
+      "epoch": 0.49878096443393943,
+      "grad_norm": 7.110764194936422,
+      "learning_rate": 2.9446022487042966e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3995,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.3671875,
+      "rewards/rejected": -8.875,
+      "step": 13400
+    },
+    {
+      "epoch": 0.49915318903426326,
+      "grad_norm": 7.2359431302636485,
+      "learning_rate": 2.9414052561591727e-07,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3813,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.25,
+      "step": 13410
+    },
+    {
+      "epoch": 0.49952541363458713,
+      "grad_norm": 5.577183399555977,
+      "learning_rate": 2.9382075183735675e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3942,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.0,
+      "step": 13420
+    },
+    {
+      "epoch": 0.49989763823491096,
+      "grad_norm": 6.764218319626137,
+      "learning_rate": 2.9350090407463363e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3847,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -8.9375,
+      "step": 13430
+    },
+    {
+      "epoch": 0.5002698628352348,
+      "grad_norm": 6.672285878207189,
+      "learning_rate": 2.9318098286775857e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.3969,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -8.4375,
+      "step": 13440
+    },
+    {
+      "epoch": 0.5006420874355586,
+      "grad_norm": 7.754931542748165,
+      "learning_rate": 2.92860988756866e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3878,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -8.4375,
+      "step": 13450
+    },
+    {
+      "epoch": 0.5010143120358824,
+      "grad_norm": 6.2994203496128405,
+      "learning_rate": 2.9254092228221365e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.375,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.0,
+      "step": 13460
+    },
+    {
+      "epoch": 0.5013865366362062,
+      "grad_norm": 7.464695502491214,
+      "learning_rate": 2.922207839841814e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3758,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -8.6875,
+      "step": 13470
+    },
+    {
+      "epoch": 0.5017587612365301,
+      "grad_norm": 6.747544237608699,
+      "learning_rate": 2.919005744032702e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3827,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.25,
+      "step": 13480
+    },
+    {
+      "epoch": 0.502130985836854,
+      "grad_norm": 6.497671086988043,
+      "learning_rate": 2.915802940801016e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4102,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -8.8125,
+      "step": 13490
+    },
+    {
+      "epoch": 0.5025032104371778,
+      "grad_norm": 6.026109437129896,
+      "learning_rate": 2.9125994355541637e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3702,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": -9.0625,
+      "step": 13500
+    },
+    {
+      "epoch": 0.5028754350375017,
+      "grad_norm": 6.475623166606656,
+      "learning_rate": 2.9093952337007404e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3845,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -8.8125,
+      "step": 13510
+    },
+    {
+      "epoch": 0.5032476596378255,
+      "grad_norm": 7.344987462213634,
+      "learning_rate": 2.9061903406505153e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.398,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -8.6875,
+      "step": 13520
+    },
+    {
+      "epoch": 0.5036198842381493,
+      "grad_norm": 7.085970155412669,
+      "learning_rate": 2.9029847618144243e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -856.0,
+      "logps/rejected": -992.0,
+      "loss": 0.4165,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -6.71875,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.25,
+      "step": 13530
+    },
+    {
+      "epoch": 0.5039921088384731,
+      "grad_norm": 7.363738387936339,
+      "learning_rate": 2.8997785026045643e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.421875,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.4056,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -8.5,
+      "step": 13540
+    },
+    {
+      "epoch": 0.5043643334387969,
+      "grad_norm": 6.998877344146602,
+      "learning_rate": 2.8965715684341767e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4015,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.90625,
+      "rewards/rejected": -9.25,
+      "step": 13550
+    },
+    {
+      "epoch": 0.5047365580391208,
+      "grad_norm": 7.143534472914478,
+      "learning_rate": 2.893363964717647e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3999,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.25,
+      "step": 13560
+    },
+    {
+      "epoch": 0.5051087826394446,
+      "grad_norm": 6.323383197595003,
+      "learning_rate": 2.8901556968704875e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4073,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.0546875,
+      "rewards/rejected": -8.75,
+      "step": 13570
+    },
+    {
+      "epoch": 0.5054810072397685,
+      "grad_norm": 6.080059376626755,
+      "learning_rate": 2.886946770309333e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.379,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -8.8125,
+      "step": 13580
+    },
+    {
+      "epoch": 0.5058532318400923,
+      "grad_norm": 6.556190332809324,
+      "learning_rate": 2.8837371904519315e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.376,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.125,
+      "step": 13590
+    },
+    {
+      "epoch": 0.5062254564404162,
+      "grad_norm": 6.567243992544531,
+      "learning_rate": 2.88052696271713e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4225,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -9.0625,
+      "step": 13600
+    },
+    {
+      "epoch": 0.50659768104074,
+      "grad_norm": 8.000388311643157,
+      "learning_rate": 2.877316092524876e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4103,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -8.8125,
+      "step": 13610
+    },
+    {
+      "epoch": 0.5069699056410638,
+      "grad_norm": 6.555055678154845,
+      "learning_rate": 2.8741045852961955e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.3952,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -8.4375,
+      "step": 13620
+    },
+    {
+      "epoch": 0.5073421302413876,
+      "grad_norm": 6.9170995820241465,
+      "learning_rate": 2.870892446453193e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -852.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.3729,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -8.5,
+      "step": 13630
+    },
+    {
+      "epoch": 0.5077143548417115,
+      "grad_norm": 5.813734355044847,
+      "learning_rate": 2.867679681419039e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3689,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -8.8125,
+      "step": 13640
+    },
+    {
+      "epoch": 0.5080865794420353,
+      "grad_norm": 9.141899225782703,
+      "learning_rate": 2.8644662956179617e-07,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4083,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.6875,
+      "step": 13650
+    },
+    {
+      "epoch": 0.5084588040423592,
+      "grad_norm": 8.631118529723661,
+      "learning_rate": 2.861252294475236e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4017,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.1875,
+      "step": 13660
+    },
+    {
+      "epoch": 0.508831028642683,
+      "grad_norm": 6.521938112568747,
+      "learning_rate": 2.858037683417176e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3999,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -8.9375,
+      "step": 13670
+    },
+    {
+      "epoch": 0.5092032532430069,
+      "grad_norm": 6.549383508279262,
+      "learning_rate": 2.8548224678711276e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4082,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -8.625,
+      "step": 13680
+    },
+    {
+      "epoch": 0.5095754778433307,
+      "grad_norm": 8.841371418894543,
+      "learning_rate": 2.8516066532654536e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.423,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -8.9375,
+      "step": 13690
+    },
+    {
+      "epoch": 0.5099477024436545,
+      "grad_norm": 5.83511935808375,
+      "learning_rate": 2.8483902450295324e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3874,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -8.9375,
+      "step": 13700
+    },
+    {
+      "epoch": 0.5103199270439783,
+      "grad_norm": 6.619263578889869,
+      "learning_rate": 2.8451732485937405e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -888.0,
+      "logps/rejected": -996.0,
+      "loss": 0.3844,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.8125,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.375,
+      "step": 13710
+    },
+    {
+      "epoch": 0.5106921516443022,
+      "grad_norm": 7.267685266782704,
+      "learning_rate": 2.841955669389451e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.4171,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -8.5625,
+      "step": 13720
+    },
+    {
+      "epoch": 0.511064376244626,
+      "grad_norm": 5.437362953991143,
+      "learning_rate": 2.838737512849019e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.96875,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.397,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -8.5625,
+      "step": 13730
+    },
+    {
+      "epoch": 0.5114366008449498,
+      "grad_norm": 7.514609538332915,
+      "learning_rate": 2.8355187844057736e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3845,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -8.8125,
+      "step": 13740
+    },
+    {
+      "epoch": 0.5118088254452737,
+      "grad_norm": 6.166823259623254,
+      "learning_rate": 2.8322994894940127e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1000.0,
+      "loss": 0.4283,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -8.5625,
+      "step": 13750
+    },
+    {
+      "epoch": 0.5121810500455976,
+      "grad_norm": 6.962333570731038,
+      "learning_rate": 2.829079633548987e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4026,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.0,
+      "step": 13760
+    },
+    {
+      "epoch": 0.5125532746459214,
+      "grad_norm": 7.679134970085725,
+      "learning_rate": 2.8258592220068966e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4042,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -8.6875,
+      "step": 13770
+    },
+    {
+      "epoch": 0.5129254992462452,
+      "grad_norm": 6.419079021618556,
+      "learning_rate": 2.8226382603048784e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.399,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -8.5625,
+      "step": 13780
+    },
+    {
+      "epoch": 0.513297723846569,
+      "grad_norm": 6.76090948052491,
+      "learning_rate": 2.819416753880999e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.3973,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.375,
+      "step": 13790
+    },
+    {
+      "epoch": 0.5136699484468928,
+      "grad_norm": 7.3501492048347,
+      "learning_rate": 2.8161947081742447e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.5,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4017,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -8.5625,
+      "step": 13800
+    },
+    {
+      "epoch": 0.5140421730472167,
+      "grad_norm": 7.603046836679688,
+      "learning_rate": 2.812972128624511e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3738,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -8.75,
+      "step": 13810
+    },
+    {
+      "epoch": 0.5144143976475405,
+      "grad_norm": 6.17781555589395,
+      "learning_rate": 2.8097490206725976e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3854,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -8.9375,
+      "step": 13820
+    },
+    {
+      "epoch": 0.5147866222478643,
+      "grad_norm": 7.931441540053952,
+      "learning_rate": 2.806525389760192e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4179,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.1875,
+      "step": 13830
+    },
+    {
+      "epoch": 0.5151588468481882,
+      "grad_norm": 7.252242636046333,
+      "learning_rate": 2.803301241329869e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4094,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.5,
+      "step": 13840
+    },
+    {
+      "epoch": 0.5155310714485121,
+      "grad_norm": 7.276005920488118,
+      "learning_rate": 2.800076580825074e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -840.0,
+      "logps/rejected": -980.0,
+      "loss": 0.4071,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -6.71875,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -8.1875,
+      "step": 13850
+    },
+    {
+      "epoch": 0.5159032960488359,
+      "grad_norm": 6.935979321801387,
+      "learning_rate": 2.796851413690119e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.4094,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -6.8125,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -8.5,
+      "step": 13860
+    },
+    {
+      "epoch": 0.5162755206491597,
+      "grad_norm": 9.22004815539808,
+      "learning_rate": 2.7936257453701693e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3944,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -8.9375,
+      "step": 13870
+    },
+    {
+      "epoch": 0.5166477452494835,
+      "grad_norm": 7.192174398609411,
+      "learning_rate": 2.790399581311238e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3952,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -9.0,
+      "step": 13880
+    },
+    {
+      "epoch": 0.5170199698498074,
+      "grad_norm": 7.6085407913319845,
+      "learning_rate": 2.7871729269601763e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3909,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -8.8125,
+      "step": 13890
+    },
+    {
+      "epoch": 0.5173921944501312,
+      "grad_norm": 6.786052907009634,
+      "learning_rate": 2.7839457877646587e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3901,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -8.625,
+      "step": 13900
+    },
+    {
+      "epoch": 0.517764419050455,
+      "grad_norm": 9.97123525062437,
+      "learning_rate": 2.780718169173184e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3835,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.8671875,
+      "rewards/rejected": -9.0625,
+      "step": 13910
+    },
+    {
+      "epoch": 0.5181366436507788,
+      "grad_norm": 7.024367355973595,
+      "learning_rate": 2.7774900766350564e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4169,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.390625,
+      "rewards/rejected": -8.8125,
+      "step": 13920
+    },
+    {
+      "epoch": 0.5185088682511028,
+      "grad_norm": 6.701742032772339,
+      "learning_rate": 2.7742615156003805e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3884,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.1875,
+      "step": 13930
+    },
+    {
+      "epoch": 0.5188810928514266,
+      "grad_norm": 6.610007938253779,
+      "learning_rate": 2.7710324915200546e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3912,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.84375,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -8.625,
+      "step": 13940
+    },
+    {
+      "epoch": 0.5192533174517504,
+      "grad_norm": 6.9465246907194516,
+      "learning_rate": 2.767803009845756e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4003,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -8.625,
+      "step": 13950
+    },
+    {
+      "epoch": 0.5196255420520742,
+      "grad_norm": 6.787787473874666,
+      "learning_rate": 2.764573076029935e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3908,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -8.8125,
+      "step": 13960
+    },
+    {
+      "epoch": 0.519997766652398,
+      "grad_norm": 7.8614911088252,
+      "learning_rate": 2.7613426955258074e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3956,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -8.875,
+      "step": 13970
+    },
+    {
+      "epoch": 0.5203699912527219,
+      "grad_norm": 6.420281750004171,
+      "learning_rate": 2.758111873787341e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.96875,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4271,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -8.875,
+      "step": 13980
+    },
+    {
+      "epoch": 0.5207422158530457,
+      "grad_norm": 7.298458475561115,
+      "learning_rate": 2.754880616269248e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.3945,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": -8.5,
+      "step": 13990
+    },
+    {
+      "epoch": 0.5211144404533695,
+      "grad_norm": 7.895141105525205,
+      "learning_rate": 2.75164892842698e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3745,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.125,
+      "step": 14000
+    },
+    {
+      "epoch": 0.5214866650536933,
+      "grad_norm": 7.61218935106479,
+      "learning_rate": 2.7484168157167095e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3988,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.1875,
+      "step": 14010
+    },
+    {
+      "epoch": 0.5218588896540173,
+      "grad_norm": 6.390207414536727,
+      "learning_rate": 2.7451842835953326e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.4212,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.0625,
+      "step": 14020
+    },
+    {
+      "epoch": 0.5222311142543411,
+      "grad_norm": 6.238814572122423,
+      "learning_rate": 2.7419513375204485e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3782,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -9.0,
+      "step": 14030
+    },
+    {
+      "epoch": 0.5226033388546649,
+      "grad_norm": 11.746337063902551,
+      "learning_rate": 2.738717982950357e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.4074,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -8.6875,
+      "step": 14040
+    },
+    {
+      "epoch": 0.5229755634549887,
+      "grad_norm": 6.590535935014694,
+      "learning_rate": 2.7354842253440484e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3678,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.0,
+      "step": 14050
+    },
+    {
+      "epoch": 0.5233477880553126,
+      "grad_norm": 7.903293723031568,
+      "learning_rate": 2.7322500701611926e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.398,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -8.9375,
+      "step": 14060
+    },
+    {
+      "epoch": 0.5237200126556364,
+      "grad_norm": 7.700506217889851,
+      "learning_rate": 2.7290155228621315e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3864,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -9.0625,
+      "step": 14070
+    },
+    {
+      "epoch": 0.5240922372559602,
+      "grad_norm": 7.866654860741371,
+      "learning_rate": 2.7257805889078677e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.4052,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -8.9375,
+      "step": 14080
+    },
+    {
+      "epoch": 0.524464461856284,
+      "grad_norm": 7.50839285832117,
+      "learning_rate": 2.7225452737600574e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4138,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -8.5,
+      "step": 14090
+    },
+    {
+      "epoch": 0.524836686456608,
+      "grad_norm": 7.659388191111035,
+      "learning_rate": 2.7193095828810016e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4012,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -8.9375,
+      "step": 14100
+    },
+    {
+      "epoch": 0.5252089110569318,
+      "grad_norm": 8.55681528191728,
+      "learning_rate": 2.7160735217336314e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4024,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.9921875,
+      "rewards/rejected": -8.9375,
+      "step": 14110
+    },
+    {
+      "epoch": 0.5255811356572556,
+      "grad_norm": 7.16727339555467,
+      "learning_rate": 2.71283709578151e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3792,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.0,
+      "step": 14120
+    },
+    {
+      "epoch": 0.5259533602575794,
+      "grad_norm": 11.73206706711725,
+      "learning_rate": 2.709600310488809e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3863,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.0625,
+      "step": 14130
+    },
+    {
+      "epoch": 0.5263255848579033,
+      "grad_norm": 6.823026753277971,
+      "learning_rate": 2.706363171320313e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3886,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.125,
+      "step": 14140
+    },
+    {
+      "epoch": 0.5266978094582271,
+      "grad_norm": 8.110746857819453,
+      "learning_rate": 2.7031256837414e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.4039,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.4375,
+      "step": 14150
+    },
+    {
+      "epoch": 0.5270700340585509,
+      "grad_norm": 6.902529471191516,
+      "learning_rate": 2.6998878532180374e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3914,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -9.0,
+      "step": 14160
+    },
+    {
+      "epoch": 0.5274422586588747,
+      "grad_norm": 7.585718456674298,
+      "learning_rate": 2.6966496852167723e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.4357,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.9375,
+      "step": 14170
+    },
+    {
+      "epoch": 0.5278144832591986,
+      "grad_norm": 7.11490323887478,
+      "learning_rate": 2.693411185204721e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3987,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -9.0,
+      "step": 14180
+    },
+    {
+      "epoch": 0.5281867078595225,
+      "grad_norm": 6.41907278214237,
+      "learning_rate": 2.69017235864956e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.398,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -8.6875,
+      "step": 14190
+    },
+    {
+      "epoch": 0.5285589324598463,
+      "grad_norm": 6.479034925223726,
+      "learning_rate": 2.686933211019517e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3901,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -8.6875,
+      "step": 14200
+    },
+    {
+      "epoch": 0.5289311570601701,
+      "grad_norm": 7.087539680153311,
+      "learning_rate": 2.6836937477833637e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3933,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -8.875,
+      "step": 14210
+    },
+    {
+      "epoch": 0.529303381660494,
+      "grad_norm": 8.862543061376018,
+      "learning_rate": 2.680453974410402e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4195,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.8125,
+      "step": 14220
+    },
+    {
+      "epoch": 0.5296756062608178,
+      "grad_norm": 8.11738752886338,
+      "learning_rate": 2.677213896370459e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4032,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -8.8125,
+      "step": 14230
+    },
+    {
+      "epoch": 0.5300478308611416,
+      "grad_norm": 7.567257769775458,
+      "learning_rate": 2.6739735191338765e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.953125,
+      "logps/chosen": -880.0,
+      "logps/rejected": -996.0,
+      "loss": 0.3969,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -6.90625,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -8.3125,
+      "step": 14240
+    },
+    {
+      "epoch": 0.5304200554614654,
+      "grad_norm": 6.837647398955726,
+      "learning_rate": 2.6707328481714997e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4026,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.0,
+      "step": 14250
+    },
+    {
+      "epoch": 0.5307922800617892,
+      "grad_norm": 6.894529418949448,
+      "learning_rate": 2.6674918889546713e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3788,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -8.875,
+      "step": 14260
+    },
+    {
+      "epoch": 0.5311645046621131,
+      "grad_norm": 6.998955700140186,
+      "learning_rate": 2.6642506469552197e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.372,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.3125,
+      "step": 14270
+    },
+    {
+      "epoch": 0.531536729262437,
+      "grad_norm": 8.340806207523931,
+      "learning_rate": 2.6610091276454523e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3983,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.0625,
+      "step": 14280
+    },
+    {
+      "epoch": 0.5319089538627608,
+      "grad_norm": 6.546477513724081,
+      "learning_rate": 2.657767336498142e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3887,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.4375,
+      "step": 14290
+    },
+    {
+      "epoch": 0.5322811784630846,
+      "grad_norm": 6.664434790918801,
+      "learning_rate": 2.654525278986523e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3967,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -8.8125,
+      "step": 14300
+    },
+    {
+      "epoch": 0.5326534030634085,
+      "grad_norm": 6.176735499675511,
+      "learning_rate": 2.651282960584279e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4031,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.6875,
+      "step": 14310
+    },
+    {
+      "epoch": 0.5330256276637323,
+      "grad_norm": 6.652636948850704,
+      "learning_rate": 2.6480403867655325e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3914,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.90625,
+      "rewards/rejected": -9.0,
+      "step": 14320
+    },
+    {
+      "epoch": 0.5333978522640561,
+      "grad_norm": 6.720265649775818,
+      "learning_rate": 2.644797563004839e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.395,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.8125,
+      "step": 14330
+    },
+    {
+      "epoch": 0.5337700768643799,
+      "grad_norm": 7.103271810767403,
+      "learning_rate": 2.641554494777175e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3828,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -9.0625,
+      "step": 14340
+    },
+    {
+      "epoch": 0.5341423014647038,
+      "grad_norm": 7.4953286855316135,
+      "learning_rate": 2.6383111875579313e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3985,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -8.9375,
+      "step": 14350
+    },
+    {
+      "epoch": 0.5345145260650276,
+      "grad_norm": 6.831448675863622,
+      "learning_rate": 2.635067646822898e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.40625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3956,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.25,
+      "step": 14360
+    },
+    {
+      "epoch": 0.5348867506653515,
+      "grad_norm": 7.596768554508921,
+      "learning_rate": 2.631823878048266e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3993,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.75,
+      "step": 14370
+    },
+    {
+      "epoch": 0.5352589752656753,
+      "grad_norm": 6.564407084319504,
+      "learning_rate": 2.628579886710605e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4009,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.75,
+      "step": 14380
+    },
+    {
+      "epoch": 0.5356311998659992,
+      "grad_norm": 8.432970770474082,
+      "learning_rate": 2.625335678286864e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4021,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -8.9375,
+      "step": 14390
+    },
+    {
+      "epoch": 0.536003424466323,
+      "grad_norm": 5.972038101446805,
+      "learning_rate": 2.622091258254358e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3808,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.0625,
+      "step": 14400
+    },
+    {
+      "epoch": 0.5363756490666468,
+      "grad_norm": 6.282918379790417,
+      "learning_rate": 2.618846632090758e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.5,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3872,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.9375,
+      "step": 14410
+    },
+    {
+      "epoch": 0.5367478736669706,
+      "grad_norm": 6.482026013361085,
+      "learning_rate": 2.6156018052740844e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4114,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -8.875,
+      "step": 14420
+    },
+    {
+      "epoch": 0.5371200982672945,
+      "grad_norm": 6.533841135317475,
+      "learning_rate": 2.6123567832826955e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -876.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3907,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -6.875,
+      "rewards/margins": 1.828125,
+      "rewards/rejected": -8.6875,
+      "step": 14430
+    },
+    {
+      "epoch": 0.5374923228676183,
+      "grad_norm": 9.343072999522697,
+      "learning_rate": 2.60911157159528e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -856.0,
+      "logps/rejected": -996.0,
+      "loss": 0.4061,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -8.5,
+      "step": 14440
+    },
+    {
+      "epoch": 0.5378645474679422,
+      "grad_norm": 6.4917376309419685,
+      "learning_rate": 2.6058661756908463e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3727,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.5625,
+      "step": 14450
+    },
+    {
+      "epoch": 0.538236772068266,
+      "grad_norm": 6.606339290189657,
+      "learning_rate": 2.6026206010487135e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3986,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -8.625,
+      "step": 14460
+    },
+    {
+      "epoch": 0.5386089966685899,
+      "grad_norm": 7.242716998437837,
+      "learning_rate": 2.599374853148502e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3865,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -8.875,
+      "step": 14470
+    },
+    {
+      "epoch": 0.5389812212689137,
+      "grad_norm": 6.750422533669666,
+      "learning_rate": 2.596128937470127e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3882,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -8.9375,
+      "step": 14480
+    },
+    {
+      "epoch": 0.5393534458692375,
+      "grad_norm": 7.378663444926177,
+      "learning_rate": 2.592882859493785e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3964,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -8.875,
+      "step": 14490
+    },
+    {
+      "epoch": 0.5397256704695613,
+      "grad_norm": 6.180996814914712,
+      "learning_rate": 2.5896366246999474e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3816,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -8.875,
+      "step": 14500
+    },
+    {
+      "epoch": 0.5400978950698851,
+      "grad_norm": 7.312809225518716,
+      "learning_rate": 2.586390238569349e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3892,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -8.8125,
+      "step": 14510
+    },
+    {
+      "epoch": 0.540470119670209,
+      "grad_norm": 7.735931403716923,
+      "learning_rate": 2.583143706582983e-07,
+      "logits/chosen": -3.640625,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.4042,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -8.75,
+      "step": 14520
+    },
+    {
+      "epoch": 0.5408423442705328,
+      "grad_norm": 7.386550142635529,
+      "learning_rate": 2.579897034222084e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3923,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.0,
+      "step": 14530
+    },
+    {
+      "epoch": 0.5412145688708567,
+      "grad_norm": 7.801685311116698,
+      "learning_rate": 2.57665022696813e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.372,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.0,
+      "step": 14540
+    },
+    {
+      "epoch": 0.5415867934711805,
+      "grad_norm": 7.950857417470352,
+      "learning_rate": 2.573403290302819e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3697,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -8.6875,
+      "step": 14550
+    },
+    {
+      "epoch": 0.5419590180715044,
+      "grad_norm": 6.054736592005576,
+      "learning_rate": 2.5701562297080755e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3788,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.953125,
+      "rewards/rejected": -8.9375,
+      "step": 14560
+    },
+    {
+      "epoch": 0.5423312426718282,
+      "grad_norm": 8.32299315563001,
+      "learning_rate": 2.5669090506660284e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.4043,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -9.0625,
+      "step": 14570
+    },
+    {
+      "epoch": 0.542703467272152,
+      "grad_norm": 7.457508532114577,
+      "learning_rate": 2.563661758659007e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4103,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -9.0,
+      "step": 14580
+    },
+    {
+      "epoch": 0.5430756918724758,
+      "grad_norm": 7.923365062566301,
+      "learning_rate": 2.560414359169533e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.381,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -8.9375,
+      "step": 14590
+    },
+    {
+      "epoch": 0.5434479164727997,
+      "grad_norm": 7.612168570836311,
+      "learning_rate": 2.5571668576803083e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3862,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -8.875,
+      "step": 14600
+    },
+    {
+      "epoch": 0.5438201410731235,
+      "grad_norm": 6.4994103046870135,
+      "learning_rate": 2.553919259674207e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3893,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.0625,
+      "step": 14610
+    },
+    {
+      "epoch": 0.5441923656734473,
+      "grad_norm": 6.326789197760104,
+      "learning_rate": 2.5506715706342665e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4096,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.75,
+      "step": 14620
+    },
+    {
+      "epoch": 0.5445645902737712,
+      "grad_norm": 6.648732167687645,
+      "learning_rate": 2.547423796043678e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.625,
+      "logps/chosen": -868.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.374,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.8671875,
+      "rewards/rejected": -8.875,
+      "step": 14630
+    },
+    {
+      "epoch": 0.5449368148740951,
+      "grad_norm": 8.329275976348509,
+      "learning_rate": 2.5441759413857764e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4001,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -8.8125,
+      "step": 14640
+    },
+    {
+      "epoch": 0.5453090394744189,
+      "grad_norm": 6.639693503014401,
+      "learning_rate": 2.540928012144033e-07,
+      "logits/chosen": -3.5625,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3906,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -8.75,
+      "step": 14650
+    },
+    {
+      "epoch": 0.5456812640747427,
+      "grad_norm": 6.5496829902143645,
+      "learning_rate": 2.537680013802045e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4075,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.0625,
+      "step": 14660
+    },
+    {
+      "epoch": 0.5460534886750665,
+      "grad_norm": 6.909218409126343,
+      "learning_rate": 2.534431951843524e-07,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.417,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.9375,
+      "step": 14670
+    },
+    {
+      "epoch": 0.5464257132753904,
+      "grad_norm": 6.348519960071836,
+      "learning_rate": 2.5311838317522924e-07,
+      "logits/chosen": -3.578125,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3841,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.0625,
+      "step": 14680
+    },
+    {
+      "epoch": 0.5467979378757142,
+      "grad_norm": 6.937539087482061,
+      "learning_rate": 2.5279356590122674e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3941,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.0625,
+      "step": 14690
+    },
+    {
+      "epoch": 0.547170162476038,
+      "grad_norm": 6.832408581548504,
+      "learning_rate": 2.524687439107458e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4046,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -8.8125,
+      "step": 14700
+    },
+    {
+      "epoch": 0.5475423870763618,
+      "grad_norm": 7.087045214872431,
+      "learning_rate": 2.5214391775219506e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4062,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.625,
+      "step": 14710
+    },
+    {
+      "epoch": 0.5479146116766858,
+      "grad_norm": 6.7539895024040595,
+      "learning_rate": 2.5181908797399033e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4139,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -8.8125,
+      "step": 14720
+    },
+    {
+      "epoch": 0.5482868362770096,
+      "grad_norm": 7.0379271465608335,
+      "learning_rate": 2.5149425512455346e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3925,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -8.6875,
+      "step": 14730
+    },
+    {
+      "epoch": 0.5486590608773334,
+      "grad_norm": 8.444929390256299,
+      "learning_rate": 2.5116941975231153e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3954,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -8.6875,
+      "step": 14740
+    },
+    {
+      "epoch": 0.5490312854776572,
+      "grad_norm": 6.37472531782827,
+      "learning_rate": 2.5084458240569595e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3953,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -8.6875,
+      "step": 14750
+    },
+    {
+      "epoch": 0.549403510077981,
+      "grad_norm": 6.670881706016348,
+      "learning_rate": 2.5051974363314126e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3658,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -8.75,
+      "step": 14760
+    },
+    {
+      "epoch": 0.5497757346783049,
+      "grad_norm": 7.496262564377888,
+      "learning_rate": 2.501949039830846e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3798,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": -9.125,
+      "step": 14770
+    },
+    {
+      "epoch": 0.5501479592786287,
+      "grad_norm": 6.279229063064921,
+      "learning_rate": 2.4987006400396445e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.3787,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -8.5625,
+      "step": 14780
+    },
+    {
+      "epoch": 0.5505201838789525,
+      "grad_norm": 6.801680844243791,
+      "learning_rate": 2.4954522424422e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -864.0,
+      "logps/rejected": -1016.0,
+      "loss": 0.4005,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.5,
+      "step": 14790
+    },
+    {
+      "epoch": 0.5508924084792763,
+      "grad_norm": 8.321305950386638,
+      "learning_rate": 2.492203852522898e-07,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3855,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -8.625,
+      "step": 14800
+    },
+    {
+      "epoch": 0.5512646330796003,
+      "grad_norm": 7.522646117401838,
+      "learning_rate": 2.4889554757661154e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4106,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.0,
+      "step": 14810
+    },
+    {
+      "epoch": 0.5516368576799241,
+      "grad_norm": 6.708087708544129,
+      "learning_rate": 2.4857071176562026e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4013,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.375,
+      "rewards/rejected": -8.625,
+      "step": 14820
+    },
+    {
+      "epoch": 0.5520090822802479,
+      "grad_norm": 6.73490907132692,
+      "learning_rate": 2.482458783677481e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3863,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -8.875,
+      "step": 14830
+    },
+    {
+      "epoch": 0.5523813068805717,
+      "grad_norm": 6.459128869478243,
+      "learning_rate": 2.47921047931423e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3754,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -8.625,
+      "step": 14840
+    },
+    {
+      "epoch": 0.5527535314808956,
+      "grad_norm": 7.387064869057433,
+      "learning_rate": 2.47596221005068e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3975,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -8.625,
+      "step": 14850
+    },
+    {
+      "epoch": 0.5531257560812194,
+      "grad_norm": 6.613897630878896,
+      "learning_rate": 2.472713981371002e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3871,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.1875,
+      "step": 14860
+    },
+    {
+      "epoch": 0.5534979806815432,
+      "grad_norm": 6.168672428973416,
+      "learning_rate": 2.4694657987592964e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3735,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.6875,
+      "step": 14870
+    },
+    {
+      "epoch": 0.553870205281867,
+      "grad_norm": 7.128443596752006,
+      "learning_rate": 2.4662176676995895e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3861,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.25,
+      "step": 14880
+    },
+    {
+      "epoch": 0.554242429882191,
+      "grad_norm": 6.234975280250282,
+      "learning_rate": 2.4629695936758176e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.386,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.1875,
+      "step": 14890
+    },
+    {
+      "epoch": 0.5546146544825148,
+      "grad_norm": 8.727733919256348,
+      "learning_rate": 2.459721582171822e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.444,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -8.6875,
+      "step": 14900
+    },
+    {
+      "epoch": 0.5549868790828386,
+      "grad_norm": 6.718372906640212,
+      "learning_rate": 2.4564736386713383e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3935,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -8.9375,
+      "step": 14910
+    },
+    {
+      "epoch": 0.5553591036831624,
+      "grad_norm": 6.697925179882034,
+      "learning_rate": 2.453225768657987e-07,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3999,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -8.8125,
+      "step": 14920
+    },
+    {
+      "epoch": 0.5557313282834863,
+      "grad_norm": 6.905346423237574,
+      "learning_rate": 2.4499779776152644e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4241,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -8.75,
+      "step": 14930
+    },
+    {
+      "epoch": 0.5561035528838101,
+      "grad_norm": 6.785509789461729,
+      "learning_rate": 2.446730271026534e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.75,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3955,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -8.75,
+      "step": 14940
+    },
+    {
+      "epoch": 0.5564757774841339,
+      "grad_norm": 6.774184709690954,
+      "learning_rate": 2.443482654375017e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3994,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.75,
+      "step": 14950
+    },
+    {
+      "epoch": 0.5568480020844577,
+      "grad_norm": 5.73699971816976,
+      "learning_rate": 2.440235133143781e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.378,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -8.9375,
+      "step": 14960
+    },
+    {
+      "epoch": 0.5572202266847815,
+      "grad_norm": 6.863429176112081,
+      "learning_rate": 2.4369877128157353e-07,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3901,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -8.875,
+      "step": 14970
+    },
+    {
+      "epoch": 0.5575924512851055,
+      "grad_norm": 5.948404549621852,
+      "learning_rate": 2.433740398873616e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3979,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.9375,
+      "step": 14980
+    },
+    {
+      "epoch": 0.5579646758854293,
+      "grad_norm": 9.914428116382458,
+      "learning_rate": 2.4304931967999817e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3905,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -8.875,
+      "step": 14990
+    },
+    {
+      "epoch": 0.5583369004857531,
+      "grad_norm": 6.87679023232461,
+      "learning_rate": 2.4272461120772e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3867,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.25,
+      "step": 15000
+    },
+    {
+      "epoch": 0.5583369004857531,
+      "eval_logits/chosen": -3.796875,
+      "eval_logits/rejected": -3.59375,
+      "eval_logps/chosen": -992.0,
+      "eval_logps/rejected": -1112.0,
+      "eval_loss": 0.46097543835639954,
+      "eval_rewards/accuracies": 0.7531396746635437,
+      "eval_rewards/chosen": -7.9375,
+      "eval_rewards/margins": 1.4453125,
+      "eval_rewards/rejected": -9.375,
+      "eval_runtime": 6273.9008,
+      "eval_samples_per_second": 30.453,
+      "eval_steps_per_second": 0.476,
+      "step": 15000
+    },
+    {
+      "epoch": 0.558709125086077,
+      "grad_norm": 6.304434593226907,
+      "learning_rate": 2.4239991501874437e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4169,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -9.0625,
+      "step": 15010
+    },
+    {
+      "epoch": 0.5590813496864008,
+      "grad_norm": 8.114223374797405,
+      "learning_rate": 2.4207523166126737e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.375,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3996,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.0625,
+      "step": 15020
+    },
+    {
+      "epoch": 0.5594535742867246,
+      "grad_norm": 6.985558918614317,
+      "learning_rate": 2.417505616834638e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3903,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -9.0,
+      "step": 15030
+    },
+    {
+      "epoch": 0.5598257988870484,
+      "grad_norm": 7.613927888065651,
+      "learning_rate": 2.4142590563348586e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4094,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.375,
+      "rewards/rejected": -8.875,
+      "step": 15040
+    },
+    {
+      "epoch": 0.5601980234873722,
+      "grad_norm": 7.5723174479152675,
+      "learning_rate": 2.411012640594619e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3785,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -9.1875,
+      "step": 15050
+    },
+    {
+      "epoch": 0.5605702480876961,
+      "grad_norm": 7.684399684732663,
+      "learning_rate": 2.4077663750949624e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.381,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -8.875,
+      "step": 15060
+    },
+    {
+      "epoch": 0.56094247268802,
+      "grad_norm": 7.778253805648127,
+      "learning_rate": 2.404520265316675e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3986,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -8.8125,
+      "step": 15070
+    },
+    {
+      "epoch": 0.5613146972883438,
+      "grad_norm": 7.404450523178779,
+      "learning_rate": 2.4012743167402823e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4078,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -8.75,
+      "step": 15080
+    },
+    {
+      "epoch": 0.5616869218886676,
+      "grad_norm": 7.833740957912255,
+      "learning_rate": 2.3980285348460363e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3905,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -8.9375,
+      "step": 15090
+    },
+    {
+      "epoch": 0.5620591464889915,
+      "grad_norm": 7.62171966614336,
+      "learning_rate": 2.3947829251139076e-07,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.3873,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -8.5,
+      "step": 15100
+    },
+    {
+      "epoch": 0.5624313710893153,
+      "grad_norm": 6.759568359187905,
+      "learning_rate": 2.391537493023579e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3894,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.0625,
+      "step": 15110
+    },
+    {
+      "epoch": 0.5628035956896391,
+      "grad_norm": 8.005931150789195,
+      "learning_rate": 2.388292244054429e-07,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.5,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1012.0,
+      "loss": 0.4175,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -8.6875,
+      "step": 15120
+    },
+    {
+      "epoch": 0.5631758202899629,
+      "grad_norm": 7.315250084747323,
+      "learning_rate": 2.3850471836855297e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.396,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -8.6875,
+      "step": 15130
+    },
+    {
+      "epoch": 0.5635480448902868,
+      "grad_norm": 7.554577291894118,
+      "learning_rate": 2.3818023173956334e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4024,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -8.75,
+      "step": 15140
+    },
+    {
+      "epoch": 0.5639202694906106,
+      "grad_norm": 7.715322510769299,
+      "learning_rate": 2.3785576506631665e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.421875,
+      "logps/chosen": -860.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3807,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 2.015625,
+      "rewards/rejected": -8.9375,
+      "step": 15150
+    },
+    {
+      "epoch": 0.5642924940909345,
+      "grad_norm": 8.800579622122374,
+      "learning_rate": 2.3753131889662162e-07,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4111,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.359375,
+      "rewards/rejected": -8.625,
+      "step": 15160
+    },
+    {
+      "epoch": 0.5646647186912583,
+      "grad_norm": 6.216610213046552,
+      "learning_rate": 2.372068937782526e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3882,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -8.8125,
+      "step": 15170
+    },
+    {
+      "epoch": 0.5650369432915822,
+      "grad_norm": 6.3611903303471875,
+      "learning_rate": 2.368824902589481e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4007,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -8.75,
+      "step": 15180
+    },
+    {
+      "epoch": 0.565409167891906,
+      "grad_norm": 6.423728815759188,
+      "learning_rate": 2.3655810888641044e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.625,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3698,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -8.75,
+      "step": 15190
+    },
+    {
+      "epoch": 0.5657813924922298,
+      "grad_norm": 6.651807248619183,
+      "learning_rate": 2.362337502083045e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.75,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.411,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -8.8125,
+      "step": 15200
+    },
+    {
+      "epoch": 0.5661536170925536,
+      "grad_norm": 6.479238998244672,
+      "learning_rate": 2.3590941477225666e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4034,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -8.9375,
+      "step": 15210
+    },
+    {
+      "epoch": 0.5665258416928775,
+      "grad_norm": 7.359793755614737,
+      "learning_rate": 2.3558510312585425e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4127,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -8.75,
+      "step": 15220
+    },
+    {
+      "epoch": 0.5668980662932013,
+      "grad_norm": 7.569067529304266,
+      "learning_rate": 2.3526081581664434e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.4041,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -8.5625,
+      "step": 15230
+    },
+    {
+      "epoch": 0.5672702908935251,
+      "grad_norm": 6.719544611702365,
+      "learning_rate": 2.3493655339213303e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4079,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -8.75,
+      "step": 15240
+    },
+    {
+      "epoch": 0.567642515493849,
+      "grad_norm": 7.714438793167596,
+      "learning_rate": 2.346123163997841e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.75,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3959,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -8.8125,
+      "step": 15250
+    },
+    {
+      "epoch": 0.5680147400941729,
+      "grad_norm": 7.334707418739626,
+      "learning_rate": 2.3428810538701893e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3675,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -8.875,
+      "step": 15260
+    },
+    {
+      "epoch": 0.5683869646944967,
+      "grad_norm": 8.375917357172026,
+      "learning_rate": 2.3396392090121435e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3972,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -8.875,
+      "step": 15270
+    },
+    {
+      "epoch": 0.5687591892948205,
+      "grad_norm": 7.10601718160401,
+      "learning_rate": 2.3363976348970303e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.4098,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.125,
+      "step": 15280
+    },
+    {
+      "epoch": 0.5691314138951443,
+      "grad_norm": 7.47573956820412,
+      "learning_rate": 2.3331563369977163e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4044,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -8.8125,
+      "step": 15290
+    },
+    {
+      "epoch": 0.5695036384954681,
+      "grad_norm": 7.296147967523193,
+      "learning_rate": 2.3299153207866013e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3826,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.25,
+      "step": 15300
+    },
+    {
+      "epoch": 0.569875863095792,
+      "grad_norm": 7.27846362968299,
+      "learning_rate": 2.326674591735612e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3986,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -9.0625,
+      "step": 15310
+    },
+    {
+      "epoch": 0.5702480876961158,
+      "grad_norm": 6.49552098774796,
+      "learning_rate": 2.3234341553161858e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.391,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.375,
+      "step": 15320
+    },
+    {
+      "epoch": 0.5706203122964397,
+      "grad_norm": 7.021461532700091,
+      "learning_rate": 2.3201940169992723e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3911,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -9.125,
+      "step": 15330
+    },
+    {
+      "epoch": 0.5709925368967635,
+      "grad_norm": 7.337526254324888,
+      "learning_rate": 2.316954182255311e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4012,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.25,
+      "step": 15340
+    },
+    {
+      "epoch": 0.5713647614970874,
+      "grad_norm": 7.281747289233091,
+      "learning_rate": 2.3137146565542343e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4084,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.0625,
+      "step": 15350
+    },
+    {
+      "epoch": 0.5717369860974112,
+      "grad_norm": 7.412934833175832,
+      "learning_rate": 2.3104754453654514e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4101,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -9.0,
+      "step": 15360
+    },
+    {
+      "epoch": 0.572109210697735,
+      "grad_norm": 8.066563897159936,
+      "learning_rate": 2.307236554157838e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3908,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -8.8125,
+      "step": 15370
+    },
+    {
+      "epoch": 0.5724814352980588,
+      "grad_norm": 6.610646057920627,
+      "learning_rate": 2.3039979883997333e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3784,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -9.25,
+      "step": 15380
+    },
+    {
+      "epoch": 0.5728536598983827,
+      "grad_norm": 7.410150183253641,
+      "learning_rate": 2.3007597535589224e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.4047,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.3125,
+      "step": 15390
+    },
+    {
+      "epoch": 0.5732258844987065,
+      "grad_norm": 6.993977557124211,
+      "learning_rate": 2.297521855102638e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3926,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.4375,
+      "step": 15400
+    },
+    {
+      "epoch": 0.5735981090990303,
+      "grad_norm": 6.186432894968066,
+      "learning_rate": 2.2942842984975383e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3801,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.90625,
+      "rewards/rejected": -9.125,
+      "step": 15410
+    },
+    {
+      "epoch": 0.5739703336993542,
+      "grad_norm": 8.939193062960232,
+      "learning_rate": 2.2910470892097102e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3954,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.5,
+      "step": 15420
+    },
+    {
+      "epoch": 0.5743425582996781,
+      "grad_norm": 7.345151732320506,
+      "learning_rate": 2.287810232704649e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4065,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.25,
+      "step": 15430
+    },
+    {
+      "epoch": 0.5747147829000019,
+      "grad_norm": 7.629776597420945,
+      "learning_rate": 2.2845737344472564e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4003,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.25,
+      "step": 15440
+    },
+    {
+      "epoch": 0.5750870075003257,
+      "grad_norm": 6.826720425615687,
+      "learning_rate": 2.2813375999018334e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3761,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.125,
+      "step": 15450
+    },
+    {
+      "epoch": 0.5754592321006495,
+      "grad_norm": 8.029971238883949,
+      "learning_rate": 2.2781018345320585e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3907,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -9.1875,
+      "step": 15460
+    },
+    {
+      "epoch": 0.5758314567009734,
+      "grad_norm": 8.876150087152947,
+      "learning_rate": 2.2748664438009953e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3857,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 2.1875,
+      "rewards/rejected": -9.6875,
+      "step": 15470
+    },
+    {
+      "epoch": 0.5762036813012972,
+      "grad_norm": 7.987671003576862,
+      "learning_rate": 2.2716314331710684e-07,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3832,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.1875,
+      "step": 15480
+    },
+    {
+      "epoch": 0.576575905901621,
+      "grad_norm": 8.476345742779369,
+      "learning_rate": 2.268396808104066e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3735,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.25,
+      "step": 15490
+    },
+    {
+      "epoch": 0.5769481305019448,
+      "grad_norm": 6.5138578687250215,
+      "learning_rate": 2.2651625740611206e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3882,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.5625,
+      "step": 15500
+    },
+    {
+      "epoch": 0.5773203551022688,
+      "grad_norm": 6.283540927927959,
+      "learning_rate": 2.2619287365027075e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3904,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -9.3125,
+      "step": 15510
+    },
+    {
+      "epoch": 0.5776925797025926,
+      "grad_norm": 7.342770639740905,
+      "learning_rate": 2.2586953008886314e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3835,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.375,
+      "step": 15520
+    },
+    {
+      "epoch": 0.5780648043029164,
+      "grad_norm": 6.138873183384365,
+      "learning_rate": 2.2554622726780186e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3695,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.25,
+      "step": 15530
+    },
+    {
+      "epoch": 0.5784370289032402,
+      "grad_norm": 7.352452552277564,
+      "learning_rate": 2.25222965732931e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.4159,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.1875,
+      "step": 15540
+    },
+    {
+      "epoch": 0.578809253503564,
+      "grad_norm": 6.748433771543007,
+      "learning_rate": 2.2489974603002437e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.4264,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.0625,
+      "step": 15550
+    },
+    {
+      "epoch": 0.5791814781038879,
+      "grad_norm": 6.486490955839246,
+      "learning_rate": 2.2457656870478587e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3928,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.0625,
+      "step": 15560
+    },
+    {
+      "epoch": 0.5795537027042117,
+      "grad_norm": 6.841328241564237,
+      "learning_rate": 2.2425343430284716e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3931,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -8.8125,
+      "step": 15570
+    },
+    {
+      "epoch": 0.5799259273045355,
+      "grad_norm": 7.0731166379454935,
+      "learning_rate": 2.239303433697679e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3941,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -8.9375,
+      "step": 15580
+    },
+    {
+      "epoch": 0.5802981519048593,
+      "grad_norm": 8.69384109328274,
+      "learning_rate": 2.2360729645103418e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3949,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -8.9375,
+      "step": 15590
+    },
+    {
+      "epoch": 0.5806703765051833,
+      "grad_norm": 7.818460832254398,
+      "learning_rate": 2.232842940920579e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.625,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3928,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -8.0,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -9.4375,
+      "step": 15600
+    },
+    {
+      "epoch": 0.5810426011055071,
+      "grad_norm": 8.177532589626665,
+      "learning_rate": 2.229613368381755e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3986,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.25,
+      "step": 15610
+    },
+    {
+      "epoch": 0.5814148257058309,
+      "grad_norm": 8.522944478988173,
+      "learning_rate": 2.2263842523464744e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.386,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.5625,
+      "step": 15620
+    },
+    {
+      "epoch": 0.5817870503061547,
+      "grad_norm": 7.660836572396954,
+      "learning_rate": 2.2231555982665728e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3975,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.375,
+      "step": 15630
+    },
+    {
+      "epoch": 0.5821592749064786,
+      "grad_norm": 9.5113032927714,
+      "learning_rate": 2.2199274115931012e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3849,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -9.1875,
+      "step": 15640
+    },
+    {
+      "epoch": 0.5825314995068024,
+      "grad_norm": 7.211526747983167,
+      "learning_rate": 2.216699697776326e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3764,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.953125,
+      "rewards/rejected": -9.3125,
+      "step": 15650
+    },
+    {
+      "epoch": 0.5829037241071262,
+      "grad_norm": 6.347640605160803,
+      "learning_rate": 2.2134724622657114e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -9.3125,
+      "step": 15660
+    },
+    {
+      "epoch": 0.58327594870745,
+      "grad_norm": 7.950090556246327,
+      "learning_rate": 2.2102457105099177e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3935,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.1875,
+      "step": 15670
+    },
+    {
+      "epoch": 0.583648173307774,
+      "grad_norm": 7.907176444113626,
+      "learning_rate": 2.2070194479567858e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.4042,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.3125,
+      "step": 15680
+    },
+    {
+      "epoch": 0.5840203979080978,
+      "grad_norm": 8.830446089524017,
+      "learning_rate": 2.2037936800533319e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3817,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -8.9375,
+      "step": 15690
+    },
+    {
+      "epoch": 0.5843926225084216,
+      "grad_norm": 8.06568542508863,
+      "learning_rate": 2.2005684122457374e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3881,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.375,
+      "step": 15700
+    },
+    {
+      "epoch": 0.5847648471087454,
+      "grad_norm": 6.987683340986026,
+      "learning_rate": 2.1973436499793377e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3917,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.1875,
+      "step": 15710
+    },
+    {
+      "epoch": 0.5851370717090693,
+      "grad_norm": 7.443410309610095,
+      "learning_rate": 2.194119398698617e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3971,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.3125,
+      "step": 15720
+    },
+    {
+      "epoch": 0.5855092963093931,
+      "grad_norm": 7.211854957463693,
+      "learning_rate": 2.190895663847194e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3793,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.8671875,
+      "rewards/rejected": -9.1875,
+      "step": 15730
+    },
+    {
+      "epoch": 0.5858815209097169,
+      "grad_norm": 6.285128132126638,
+      "learning_rate": 2.1876724508678184e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3835,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -9.0,
+      "step": 15740
+    },
+    {
+      "epoch": 0.5862537455100407,
+      "grad_norm": 6.7100955909606705,
+      "learning_rate": 2.1844497652023563e-07,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3935,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.125,
+      "step": 15750
+    },
+    {
+      "epoch": 0.5866259701103645,
+      "grad_norm": 5.7565762201527075,
+      "learning_rate": 2.181227612291786e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3756,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.125,
+      "step": 15760
+    },
+    {
+      "epoch": 0.5869981947106885,
+      "grad_norm": 6.610554519765395,
+      "learning_rate": 2.1780059975761833e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.4025,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 2.015625,
+      "rewards/rejected": -9.3125,
+      "step": 15770
+    },
+    {
+      "epoch": 0.5873704193110123,
+      "grad_norm": 6.38759048756963,
+      "learning_rate": 2.1747849264947177e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3763,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.0,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -8.875,
+      "step": 15780
+    },
+    {
+      "epoch": 0.5877426439113361,
+      "grad_norm": 6.563105373864839,
+      "learning_rate": 2.1715644044856413e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3909,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -8.9375,
+      "step": 15790
+    },
+    {
+      "epoch": 0.58811486851166,
+      "grad_norm": 8.162039192517167,
+      "learning_rate": 2.1683444369862763e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4094,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -8.875,
+      "step": 15800
+    },
+    {
+      "epoch": 0.5884870931119838,
+      "grad_norm": 7.6421627781876875,
+      "learning_rate": 2.165125029433012e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3926,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.8125,
+      "step": 15810
+    },
+    {
+      "epoch": 0.5888593177123076,
+      "grad_norm": 7.16200998882505,
+      "learning_rate": 2.1619061872612892e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4116,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.140625,
+      "rewards/rejected": -8.8125,
+      "step": 15820
+    },
+    {
+      "epoch": 0.5892315423126314,
+      "grad_norm": 6.682682170075334,
+      "learning_rate": 2.158687915905597e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3794,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.1875,
+      "step": 15830
+    },
+    {
+      "epoch": 0.5896037669129552,
+      "grad_norm": 6.103265004810518,
+      "learning_rate": 2.155470220799459e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3948,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.3125,
+      "step": 15840
+    },
+    {
+      "epoch": 0.5899759915132791,
+      "grad_norm": 6.787959679810598,
+      "learning_rate": 2.152253107375427e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.392,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.1875,
+      "step": 15850
+    },
+    {
+      "epoch": 0.590348216113603,
+      "grad_norm": 7.662170829705447,
+      "learning_rate": 2.1490365810650686e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -988.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.4011,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.5625,
+      "step": 15860
+    },
+    {
+      "epoch": 0.5907204407139268,
+      "grad_norm": 8.643648774693663,
+      "learning_rate": 2.1458206472989626e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3715,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -9.125,
+      "step": 15870
+    },
+    {
+      "epoch": 0.5910926653142506,
+      "grad_norm": 7.684880758132232,
+      "learning_rate": 2.1426053115066875e-07,
+      "logits/chosen": -3.59375,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3974,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.0625,
+      "step": 15880
+    },
+    {
+      "epoch": 0.5914648899145745,
+      "grad_norm": 6.311889668620401,
+      "learning_rate": 2.1393905791168093e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3725,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -8.875,
+      "step": 15890
+    },
+    {
+      "epoch": 0.5918371145148983,
+      "grad_norm": 8.174318519439355,
+      "learning_rate": 2.136176455556879e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3781,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -8.9375,
+      "step": 15900
+    },
+    {
+      "epoch": 0.5922093391152221,
+      "grad_norm": 6.92945233582831,
+      "learning_rate": 2.132962946253416e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3794,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -8.9375,
+      "step": 15910
+    },
+    {
+      "epoch": 0.5925815637155459,
+      "grad_norm": 7.956568704007788,
+      "learning_rate": 2.129750056631906e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4064,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -9.25,
+      "step": 15920
+    },
+    {
+      "epoch": 0.5929537883158698,
+      "grad_norm": 8.700208878841817,
+      "learning_rate": 2.1265377921167855e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3848,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -8.875,
+      "step": 15930
+    },
+    {
+      "epoch": 0.5933260129161936,
+      "grad_norm": 8.978247581311013,
+      "learning_rate": 2.1233261581314385e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3872,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.125,
+      "step": 15940
+    },
+    {
+      "epoch": 0.5936982375165175,
+      "grad_norm": 7.797462512248357,
+      "learning_rate": 2.1201151600981814e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4014,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 2.0,
+      "rewards/rejected": -9.3125,
+      "step": 15950
+    },
+    {
+      "epoch": 0.5940704621168413,
+      "grad_norm": 7.34329261558189,
+      "learning_rate": 2.1169048034382598e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3869,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -9.0625,
+      "step": 15960
+    },
+    {
+      "epoch": 0.5944426867171652,
+      "grad_norm": 6.38463139428593,
+      "learning_rate": 2.1136950935718348e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.4208,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -9.0,
+      "step": 15970
+    },
+    {
+      "epoch": 0.594814911317489,
+      "grad_norm": 7.840263810341536,
+      "learning_rate": 2.1104860359179756e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3789,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.3125,
+      "step": 15980
+    },
+    {
+      "epoch": 0.5951871359178128,
+      "grad_norm": 8.705456464119012,
+      "learning_rate": 2.1072776358946507e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3559,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.25,
+      "step": 15990
+    },
+    {
+      "epoch": 0.5955593605181366,
+      "grad_norm": 8.859386379883384,
+      "learning_rate": 2.1040698989187175e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3624,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.4375,
+      "step": 16000
+    },
+    {
+      "epoch": 0.5959315851184604,
+      "grad_norm": 7.777288166956703,
+      "learning_rate": 2.1008628304059155e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3887,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.0,
+      "step": 16010
+    },
+    {
+      "epoch": 0.5963038097187843,
+      "grad_norm": 8.152633141943644,
+      "learning_rate": 2.0976564357708538e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3965,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.9765625,
+      "rewards/rejected": -9.625,
+      "step": 16020
+    },
+    {
+      "epoch": 0.5966760343191081,
+      "grad_norm": 6.468339881267823,
+      "learning_rate": 2.0944507204270044e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3752,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.3125,
+      "step": 16030
+    },
+    {
+      "epoch": 0.597048258919432,
+      "grad_norm": 6.389721872724469,
+      "learning_rate": 2.0912456897866942e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4078,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -8.9375,
+      "step": 16040
+    },
+    {
+      "epoch": 0.5974204835197559,
+      "grad_norm": 6.554518358563945,
+      "learning_rate": 2.0880413492610904e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.03125,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -8.8125,
+      "step": 16050
+    },
+    {
+      "epoch": 0.5977927081200797,
+      "grad_norm": 7.334320750511469,
+      "learning_rate": 2.0848377042601992e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.404,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": -9.25,
+      "step": 16060
+    },
+    {
+      "epoch": 0.5981649327204035,
+      "grad_norm": 6.809948281349834,
+      "learning_rate": 2.081634760192849e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4134,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.0625,
+      "step": 16070
+    },
+    {
+      "epoch": 0.5985371573207273,
+      "grad_norm": 7.651604450945221,
+      "learning_rate": 2.078432522466687e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3862,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -9.125,
+      "step": 16080
+    },
+    {
+      "epoch": 0.5989093819210511,
+      "grad_norm": 8.115473304646807,
+      "learning_rate": 2.0752309964881664e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4097,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.125,
+      "step": 16090
+    },
+    {
+      "epoch": 0.599281606521375,
+      "grad_norm": 7.373492099339753,
+      "learning_rate": 2.0720301876625416e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4099,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.125,
+      "step": 16100
+    },
+    {
+      "epoch": 0.5996538311216988,
+      "grad_norm": 7.847577344929947,
+      "learning_rate": 2.0688301013938504e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3835,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.375,
+      "step": 16110
+    },
+    {
+      "epoch": 0.6000260557220227,
+      "grad_norm": 6.3562299864089296,
+      "learning_rate": 2.065630743084917e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4026,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -8.9375,
+      "step": 16120
+    },
+    {
+      "epoch": 0.6003982803223465,
+      "grad_norm": 7.527581839044944,
+      "learning_rate": 2.062432118137334e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3901,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.8125,
+      "step": 16130
+    },
+    {
+      "epoch": 0.6007705049226704,
+      "grad_norm": 6.897446926296911,
+      "learning_rate": 2.0592342319514547e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3846,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -8.9375,
+      "step": 16140
+    },
+    {
+      "epoch": 0.6011427295229942,
+      "grad_norm": 7.600174905122629,
+      "learning_rate": 2.0560370899263867e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3922,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.0,
+      "step": 16150
+    },
+    {
+      "epoch": 0.601514954123318,
+      "grad_norm": 6.95946998251373,
+      "learning_rate": 2.0528406974599808e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4013,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -9.0,
+      "step": 16160
+    },
+    {
+      "epoch": 0.6018871787236418,
+      "grad_norm": 7.364649281275915,
+      "learning_rate": 2.0496450599488224e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3792,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.375,
+      "step": 16170
+    },
+    {
+      "epoch": 0.6022594033239657,
+      "grad_norm": 8.766916912792944,
+      "learning_rate": 2.0464501827882202e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3934,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -8.875,
+      "step": 16180
+    },
+    {
+      "epoch": 0.6026316279242895,
+      "grad_norm": 7.300740515082562,
+      "learning_rate": 2.0432560713722043e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4027,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.25,
+      "step": 16190
+    },
+    {
+      "epoch": 0.6030038525246133,
+      "grad_norm": 7.443948679078861,
+      "learning_rate": 2.040062731093505e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3913,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -8.9375,
+      "step": 16200
+    },
+    {
+      "epoch": 0.6033760771249372,
+      "grad_norm": 8.699199509356124,
+      "learning_rate": 2.0368701673435567e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3694,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.6875,
+      "step": 16210
+    },
+    {
+      "epoch": 0.6037483017252611,
+      "grad_norm": 6.645957156117027,
+      "learning_rate": 2.0336783855124806e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3836,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.9375,
+      "step": 16220
+    },
+    {
+      "epoch": 0.6041205263255849,
+      "grad_norm": 8.433118470831792,
+      "learning_rate": 2.0304873909890763e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3947,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -8.9375,
+      "step": 16230
+    },
+    {
+      "epoch": 0.6044927509259087,
+      "grad_norm": 7.555559984656153,
+      "learning_rate": 2.027297189160817e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3925,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.1875,
+      "step": 16240
+    },
+    {
+      "epoch": 0.6048649755262325,
+      "grad_norm": 6.095879613409558,
+      "learning_rate": 2.0241077854138337e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3806,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.375,
+      "step": 16250
+    },
+    {
+      "epoch": 0.6052372001265564,
+      "grad_norm": 7.878540103921838,
+      "learning_rate": 2.0209191851329149e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.396,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -9.25,
+      "step": 16260
+    },
+    {
+      "epoch": 0.6056094247268802,
+      "grad_norm": 5.798139759595799,
+      "learning_rate": 2.0177313937014873e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3849,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.125,
+      "step": 16270
+    },
+    {
+      "epoch": 0.605981649327204,
+      "grad_norm": 7.411121657220518,
+      "learning_rate": 2.0145444165016164e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3992,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.3125,
+      "step": 16280
+    },
+    {
+      "epoch": 0.6063538739275278,
+      "grad_norm": 5.714861430463715,
+      "learning_rate": 2.0113582589139917e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3886,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.0,
+      "step": 16290
+    },
+    {
+      "epoch": 0.6067260985278518,
+      "grad_norm": 8.087727372755937,
+      "learning_rate": 2.0081729263179165e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.37,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -8.75,
+      "step": 16300
+    },
+    {
+      "epoch": 0.6070983231281756,
+      "grad_norm": 7.717400431321285,
+      "learning_rate": 2.0049884240913056e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3928,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.9375,
+      "step": 16310
+    },
+    {
+      "epoch": 0.6074705477284994,
+      "grad_norm": 6.753239562175732,
+      "learning_rate": 2.0018047576106666e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3794,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -8.9375,
+      "step": 16320
+    },
+    {
+      "epoch": 0.6078427723288232,
+      "grad_norm": 8.86975180417233,
+      "learning_rate": 1.998621932251102e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3727,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -9.1875,
+      "step": 16330
+    },
+    {
+      "epoch": 0.608214996929147,
+      "grad_norm": 6.830905036985514,
+      "learning_rate": 1.9954399533862883e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4058,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.0,
+      "step": 16340
+    },
+    {
+      "epoch": 0.6085872215294709,
+      "grad_norm": 7.999803025256423,
+      "learning_rate": 1.9922588263884785e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3872,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -9.0625,
+      "step": 16350
+    },
+    {
+      "epoch": 0.6089594461297947,
+      "grad_norm": 6.672693148582262,
+      "learning_rate": 1.9890785566284817e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3704,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -9.125,
+      "step": 16360
+    },
+    {
+      "epoch": 0.6093316707301185,
+      "grad_norm": 9.747693435308475,
+      "learning_rate": 1.9858991494756635e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4002,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.25,
+      "step": 16370
+    },
+    {
+      "epoch": 0.6097038953304423,
+      "grad_norm": 7.450654375216106,
+      "learning_rate": 1.9827206102979343e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.4026,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -9.375,
+      "step": 16380
+    },
+    {
+      "epoch": 0.6100761199307663,
+      "grad_norm": 6.707662206668727,
+      "learning_rate": 1.9795429444617334e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4181,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -8.875,
+      "step": 16390
+    },
+    {
+      "epoch": 0.6104483445310901,
+      "grad_norm": 6.849090535667424,
+      "learning_rate": 1.9763661573320322e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4056,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -8.9375,
+      "step": 16400
+    },
+    {
+      "epoch": 0.6108205691314139,
+      "grad_norm": 7.097289410037448,
+      "learning_rate": 1.9731902542723128e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.425,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -8.625,
+      "step": 16410
+    },
+    {
+      "epoch": 0.6111927937317377,
+      "grad_norm": 7.164980921628045,
+      "learning_rate": 1.9700152406445704e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3785,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.0,
+      "step": 16420
+    },
+    {
+      "epoch": 0.6115650183320616,
+      "grad_norm": 10.607248417429686,
+      "learning_rate": 1.9668411218092928e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3879,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -8.75,
+      "step": 16430
+    },
+    {
+      "epoch": 0.6119372429323854,
+      "grad_norm": 7.058264735219548,
+      "learning_rate": 1.963667903125461e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3946,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -8.9375,
+      "step": 16440
+    },
+    {
+      "epoch": 0.6123094675327092,
+      "grad_norm": 7.606249554793642,
+      "learning_rate": 1.9604955899505346e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3912,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -8.8125,
+      "step": 16450
+    },
+    {
+      "epoch": 0.612681692133033,
+      "grad_norm": 6.488798563264654,
+      "learning_rate": 1.9573241876404445e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3871,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.9296875,
+      "rewards/rejected": -9.125,
+      "step": 16460
+    },
+    {
+      "epoch": 0.613053916733357,
+      "grad_norm": 6.921098132922328,
+      "learning_rate": 1.9541537015495864e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3809,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -8.9375,
+      "step": 16470
+    },
+    {
+      "epoch": 0.6134261413336808,
+      "grad_norm": 7.016973913971312,
+      "learning_rate": 1.9509841370308034e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4053,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -8.9375,
+      "step": 16480
+    },
+    {
+      "epoch": 0.6137983659340046,
+      "grad_norm": 7.474945753351283,
+      "learning_rate": 1.9478154994353897e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3861,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -8.9375,
+      "step": 16490
+    },
+    {
+      "epoch": 0.6141705905343284,
+      "grad_norm": 9.08077530097975,
+      "learning_rate": 1.9446477941130677e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4091,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -8.5625,
+      "step": 16500
+    },
+    {
+      "epoch": 0.6145428151346523,
+      "grad_norm": 8.471951951496047,
+      "learning_rate": 1.9414810264119907e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.328125,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3945,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -9.25,
+      "step": 16510
+    },
+    {
+      "epoch": 0.6149150397349761,
+      "grad_norm": 7.57224979136791,
+      "learning_rate": 1.9383152016787266e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3817,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -8.9375,
+      "step": 16520
+    },
+    {
+      "epoch": 0.6152872643352999,
+      "grad_norm": 7.780286315551446,
+      "learning_rate": 1.935150325258253e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3791,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.0,
+      "step": 16530
+    },
+    {
+      "epoch": 0.6156594889356237,
+      "grad_norm": 7.555788581404402,
+      "learning_rate": 1.931986402493944e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3911,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.125,
+      "step": 16540
+    },
+    {
+      "epoch": 0.6160317135359475,
+      "grad_norm": 7.072823055415816,
+      "learning_rate": 1.9288234387275656e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3923,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.125,
+      "step": 16550
+    },
+    {
+      "epoch": 0.6164039381362715,
+      "grad_norm": 7.912582250216763,
+      "learning_rate": 1.9256614392992647e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3859,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -8.875,
+      "step": 16560
+    },
+    {
+      "epoch": 0.6167761627365953,
+      "grad_norm": 6.80910213253247,
+      "learning_rate": 1.9225004095475587e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3948,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -8.75,
+      "step": 16570
+    },
+    {
+      "epoch": 0.6171483873369191,
+      "grad_norm": 7.197189641819027,
+      "learning_rate": 1.9193403548093294e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4108,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -8.625,
+      "step": 16580
+    },
+    {
+      "epoch": 0.617520611937243,
+      "grad_norm": 6.6152631148982275,
+      "learning_rate": 1.9161812804198107e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3784,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -6.9375,
+      "rewards/margins": 1.8671875,
+      "rewards/rejected": -8.8125,
+      "step": 16590
+    },
+    {
+      "epoch": 0.6178928365375668,
+      "grad_norm": 8.37990925342298,
+      "learning_rate": 1.9130231917125833e-07,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3999,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -8.75,
+      "step": 16600
+    },
+    {
+      "epoch": 0.6182650611378906,
+      "grad_norm": 8.423270588922644,
+      "learning_rate": 1.9098660940195618e-07,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3937,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -8.75,
+      "step": 16610
+    },
+    {
+      "epoch": 0.6186372857382144,
+      "grad_norm": 7.578165995037437,
+      "learning_rate": 1.906709992670989e-07,
+      "logits/chosen": -3.640625,
+      "logits/rejected": -3.40625,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4183,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -8.875,
+      "step": 16620
+    },
+    {
+      "epoch": 0.6190095103385382,
+      "grad_norm": 7.534058017228509,
+      "learning_rate": 1.9035548929954257e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3881,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.0625,
+      "step": 16630
+    },
+    {
+      "epoch": 0.6193817349388621,
+      "grad_norm": 8.067499077365724,
+      "learning_rate": 1.9004008003197398e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3962,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.9375,
+      "step": 16640
+    },
+    {
+      "epoch": 0.619753959539186,
+      "grad_norm": 5.6949080983987415,
+      "learning_rate": 1.8972477199691014e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -860.0,
+      "logps/rejected": -1008.0,
+      "loss": 0.3737,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -6.78125,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -8.4375,
+      "step": 16650
+    },
+    {
+      "epoch": 0.6201261841395098,
+      "grad_norm": 6.790203332343799,
+      "learning_rate": 1.8940956572669692e-07,
+      "logits/chosen": -3.5625,
+      "logits/rejected": -3.28125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4069,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -8.8125,
+      "step": 16660
+    },
+    {
+      "epoch": 0.6204984087398336,
+      "grad_norm": 6.266235398680496,
+      "learning_rate": 1.8909446175350856e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3802,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -8.625,
+      "step": 16670
+    },
+    {
+      "epoch": 0.6208706333401575,
+      "grad_norm": 7.586969004582857,
+      "learning_rate": 1.887794606093465e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3792,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.1875,
+      "step": 16680
+    },
+    {
+      "epoch": 0.6212428579404813,
+      "grad_norm": 7.5379060512758285,
+      "learning_rate": 1.8846456282603856e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3848,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.0625,
+      "step": 16690
+    },
+    {
+      "epoch": 0.6216150825408051,
+      "grad_norm": 6.967033810745793,
+      "learning_rate": 1.8814976893523804e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3979,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.1875,
+      "step": 16700
+    },
+    {
+      "epoch": 0.6219873071411289,
+      "grad_norm": 7.8333742329763645,
+      "learning_rate": 1.8783507946842291e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3843,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.0625,
+      "step": 16710
+    },
+    {
+      "epoch": 0.6223595317414528,
+      "grad_norm": 7.59450552809616,
+      "learning_rate": 1.8752049495689483e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4091,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.25,
+      "step": 16720
+    },
+    {
+      "epoch": 0.6227317563417766,
+      "grad_norm": 6.961657268445571,
+      "learning_rate": 1.872060159317782e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3599,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.125,
+      "step": 16730
+    },
+    {
+      "epoch": 0.6231039809421005,
+      "grad_norm": 7.193756831220989,
+      "learning_rate": 1.8689164292401932e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.396,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.3125,
+      "step": 16740
+    },
+    {
+      "epoch": 0.6234762055424243,
+      "grad_norm": 9.481045375771625,
+      "learning_rate": 1.865773764643855e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4122,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -9.125,
+      "step": 16750
+    },
+    {
+      "epoch": 0.6238484301427482,
+      "grad_norm": 6.718897354921702,
+      "learning_rate": 1.8626321708346432e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3783,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.1875,
+      "step": 16760
+    },
+    {
+      "epoch": 0.624220654743072,
+      "grad_norm": 7.191869417400532,
+      "learning_rate": 1.8594916531166226e-07,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.406,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -9.1875,
+      "step": 16770
+    },
+    {
+      "epoch": 0.6245928793433958,
+      "grad_norm": 8.555159523485905,
+      "learning_rate": 1.8563522167920445e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3751,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 2.046875,
+      "rewards/rejected": -9.3125,
+      "step": 16780
+    },
+    {
+      "epoch": 0.6249651039437196,
+      "grad_norm": 8.38897969973936,
+      "learning_rate": 1.853213867161332e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3832,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.9296875,
+      "rewards/rejected": -9.625,
+      "step": 16790
+    },
+    {
+      "epoch": 0.6253373285440434,
+      "grad_norm": 7.544195224554637,
+      "learning_rate": 1.8500766095230746e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3875,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.09375,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -8.9375,
+      "step": 16800
+    },
+    {
+      "epoch": 0.6257095531443673,
+      "grad_norm": 6.844031086278621,
+      "learning_rate": 1.8469404491740193e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3935,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -9.0,
+      "step": 16810
+    },
+    {
+      "epoch": 0.6260817777446911,
+      "grad_norm": 5.922638075299952,
+      "learning_rate": 1.8438053914090574e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3943,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.125,
+      "step": 16820
+    },
+    {
+      "epoch": 0.626454002345015,
+      "grad_norm": 7.297419400103341,
+      "learning_rate": 1.8406714415212216e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3939,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -8.9375,
+      "step": 16830
+    },
+    {
+      "epoch": 0.6268262269453388,
+      "grad_norm": 7.219009747734024,
+      "learning_rate": 1.8375386048016722e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3795,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.0625,
+      "step": 16840
+    },
+    {
+      "epoch": 0.6271984515456627,
+      "grad_norm": 6.447556405237432,
+      "learning_rate": 1.8344068865396922e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3657,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.1875,
+      "step": 16850
+    },
+    {
+      "epoch": 0.6275706761459865,
+      "grad_norm": 7.309133018586995,
+      "learning_rate": 1.8312762920226731e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3914,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -9.0625,
+      "step": 16860
+    },
+    {
+      "epoch": 0.6279429007463103,
+      "grad_norm": 7.683641854183945,
+      "learning_rate": 1.8281468265361125e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4082,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.25,
+      "step": 16870
+    },
+    {
+      "epoch": 0.6283151253466341,
+      "grad_norm": 8.467173980039464,
+      "learning_rate": 1.8250184953636e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4156,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.0625,
+      "step": 16880
+    },
+    {
+      "epoch": 0.628687349946958,
+      "grad_norm": 7.475191852154221,
+      "learning_rate": 1.8218913037868102e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3771,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.1875,
+      "step": 16890
+    },
+    {
+      "epoch": 0.6290595745472818,
+      "grad_norm": 8.476717398121197,
+      "learning_rate": 1.8187652570854945e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3827,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.0,
+      "step": 16900
+    },
+    {
+      "epoch": 0.6294317991476057,
+      "grad_norm": 6.682119699895684,
+      "learning_rate": 1.81564036053747e-07,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.375,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4016,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -8.9375,
+      "step": 16910
+    },
+    {
+      "epoch": 0.6298040237479295,
+      "grad_norm": 11.239610674818202,
+      "learning_rate": 1.812516619418613e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1024.0,
+      "loss": 0.3986,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -8.6875,
+      "step": 16920
+    },
+    {
+      "epoch": 0.6301762483482534,
+      "grad_norm": 6.773760322516131,
+      "learning_rate": 1.8093940390028483e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3974,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -8.9375,
+      "step": 16930
+    },
+    {
+      "epoch": 0.6305484729485772,
+      "grad_norm": 6.599877635887675,
+      "learning_rate": 1.806272624562143e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.382,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.125,
+      "step": 16940
+    },
+    {
+      "epoch": 0.630920697548901,
+      "grad_norm": 6.769584656269559,
+      "learning_rate": 1.8031523813664923e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3681,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.25,
+      "step": 16950
+    },
+    {
+      "epoch": 0.6312929221492248,
+      "grad_norm": 7.4382151068844715,
+      "learning_rate": 1.800033314683917e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.4043,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.4375,
+      "step": 16960
+    },
+    {
+      "epoch": 0.6316651467495487,
+      "grad_norm": 8.812071855086787,
+      "learning_rate": 1.7969154297804507e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.4049,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -9.4375,
+      "step": 16970
+    },
+    {
+      "epoch": 0.6320373713498725,
+      "grad_norm": 8.430906129049367,
+      "learning_rate": 1.7937987319201297e-07,
+      "logits/chosen": -3.640625,
+      "logits/rejected": -3.5,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3797,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.1875,
+      "step": 16980
+    },
+    {
+      "epoch": 0.6324095959501963,
+      "grad_norm": 6.922624378984371,
+      "learning_rate": 1.79068322636499e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3972,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -9.1875,
+      "step": 16990
+    },
+    {
+      "epoch": 0.6327818205505202,
+      "grad_norm": 7.315176404869083,
+      "learning_rate": 1.7875689183750504e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3923,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -8.75,
+      "step": 17000
+    },
+    {
+      "epoch": 0.6331540451508441,
+      "grad_norm": 7.831448422759494,
+      "learning_rate": 1.7844558132083117e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -872.0,
+      "logps/rejected": -1004.0,
+      "loss": 0.3979,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -6.96875,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -8.5,
+      "step": 17010
+    },
+    {
+      "epoch": 0.6335262697511679,
+      "grad_norm": 7.276248494068267,
+      "learning_rate": 1.7813439161207412e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3865,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -8.75,
+      "step": 17020
+    },
+    {
+      "epoch": 0.6338984943514917,
+      "grad_norm": 6.901648329104485,
+      "learning_rate": 1.778233232366268e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.3805,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.75,
+      "step": 17030
+    },
+    {
+      "epoch": 0.6342707189518155,
+      "grad_norm": 6.689179476129809,
+      "learning_rate": 1.7751237671967707e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3791,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.0625,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -8.9375,
+      "step": 17040
+    },
+    {
+      "epoch": 0.6346429435521393,
+      "grad_norm": 7.776488407338435,
+      "learning_rate": 1.772015525862073e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3902,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.75,
+      "step": 17050
+    },
+    {
+      "epoch": 0.6350151681524632,
+      "grad_norm": 8.180322086778977,
+      "learning_rate": 1.7689085136099325e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3603,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.0625,
+      "step": 17060
+    },
+    {
+      "epoch": 0.635387392752787,
+      "grad_norm": 7.529364605071619,
+      "learning_rate": 1.7658027356860284e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3878,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.25,
+      "step": 17070
+    },
+    {
+      "epoch": 0.6357596173531108,
+      "grad_norm": 9.806825166930818,
+      "learning_rate": 1.7626981973339597e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4092,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -8.9375,
+      "step": 17080
+    },
+    {
+      "epoch": 0.6361318419534348,
+      "grad_norm": 6.888230127652128,
+      "learning_rate": 1.7595949037952298e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.382,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.8671875,
+      "rewards/rejected": -9.1875,
+      "step": 17090
+    },
+    {
+      "epoch": 0.6365040665537586,
+      "grad_norm": 7.819578502928127,
+      "learning_rate": 1.7564928603092432e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3926,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.125,
+      "step": 17100
+    },
+    {
+      "epoch": 0.6368762911540824,
+      "grad_norm": 7.094530049310864,
+      "learning_rate": 1.7533920721132894e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3704,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -9.25,
+      "step": 17110
+    },
+    {
+      "epoch": 0.6372485157544062,
+      "grad_norm": 7.453399575901283,
+      "learning_rate": 1.7502925444425448e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3953,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -9.0625,
+      "step": 17120
+    },
+    {
+      "epoch": 0.63762074035473,
+      "grad_norm": 8.111920833743985,
+      "learning_rate": 1.7471942825300512e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3903,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.3125,
+      "step": 17130
+    },
+    {
+      "epoch": 0.6379929649550539,
+      "grad_norm": 7.875967563280191,
+      "learning_rate": 1.744097291606718e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3624,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -8.9375,
+      "step": 17140
+    },
+    {
+      "epoch": 0.6383651895553777,
+      "grad_norm": 8.12658999845993,
+      "learning_rate": 1.741001576901308e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.348,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.3125,
+      "step": 17150
+    },
+    {
+      "epoch": 0.6387374141557015,
+      "grad_norm": 7.253558899944683,
+      "learning_rate": 1.7379071436404268e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3765,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.4375,
+      "step": 17160
+    },
+    {
+      "epoch": 0.6391096387560253,
+      "grad_norm": 10.57276392005003,
+      "learning_rate": 1.73481399704852e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.4108,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.4375,
+      "step": 17170
+    },
+    {
+      "epoch": 0.6394818633563493,
+      "grad_norm": 7.785937303751153,
+      "learning_rate": 1.7317221423478556e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3977,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.125,
+      "step": 17180
+    },
+    {
+      "epoch": 0.6398540879566731,
+      "grad_norm": 8.93550874607475,
+      "learning_rate": 1.728631584758528e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3935,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.4375,
+      "step": 17190
+    },
+    {
+      "epoch": 0.6402263125569969,
+      "grad_norm": 8.802223218889527,
+      "learning_rate": 1.7255423294984342e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3854,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.125,
+      "step": 17200
+    },
+    {
+      "epoch": 0.6405985371573207,
+      "grad_norm": 6.611364438479658,
+      "learning_rate": 1.7224543817832777e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3945,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.375,
+      "step": 17210
+    },
+    {
+      "epoch": 0.6409707617576446,
+      "grad_norm": 7.930572194678817,
+      "learning_rate": 1.719367746826553e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3788,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.1875,
+      "step": 17220
+    },
+    {
+      "epoch": 0.6413429863579684,
+      "grad_norm": 7.249459984033662,
+      "learning_rate": 1.716282429839535e-07,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.359375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3953,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.3125,
+      "step": 17230
+    },
+    {
+      "epoch": 0.6417152109582922,
+      "grad_norm": 6.8023943374575415,
+      "learning_rate": 1.7131984360312795e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3716,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -8.875,
+      "step": 17240
+    },
+    {
+      "epoch": 0.642087435558616,
+      "grad_norm": 6.790902517611173,
+      "learning_rate": 1.7101157706086016e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3742,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.8671875,
+      "rewards/rejected": -9.25,
+      "step": 17250
+    },
+    {
+      "epoch": 0.6424596601589398,
+      "grad_norm": 7.210927167164257,
+      "learning_rate": 1.70703443877608e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4179,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -8.9375,
+      "step": 17260
+    },
+    {
+      "epoch": 0.6428318847592638,
+      "grad_norm": 6.577817095219949,
+      "learning_rate": 1.703954445736036e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.383,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.125,
+      "step": 17270
+    },
+    {
+      "epoch": 0.6432041093595876,
+      "grad_norm": 7.623599785087364,
+      "learning_rate": 1.7008757966885368e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3864,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 2.015625,
+      "rewards/rejected": -9.375,
+      "step": 17280
+    },
+    {
+      "epoch": 0.6435763339599114,
+      "grad_norm": 7.739876539936512,
+      "learning_rate": 1.6977984968313735e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3624,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.9296875,
+      "rewards/rejected": -9.375,
+      "step": 17290
+    },
+    {
+      "epoch": 0.6439485585602353,
+      "grad_norm": 7.545119025178648,
+      "learning_rate": 1.6947225513600646e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.387,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.25,
+      "step": 17300
+    },
+    {
+      "epoch": 0.6443207831605591,
+      "grad_norm": 15.76490575993181,
+      "learning_rate": 1.6916479654678417e-07,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.378,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.1875,
+      "step": 17310
+    },
+    {
+      "epoch": 0.6446930077608829,
+      "grad_norm": 5.737841525428915,
+      "learning_rate": 1.6885747443456362e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.5,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3694,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.0,
+      "step": 17320
+    },
+    {
+      "epoch": 0.6450652323612067,
+      "grad_norm": 7.138613269156902,
+      "learning_rate": 1.6855028931820826e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3506,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": -9.375,
+      "step": 17330
+    },
+    {
+      "epoch": 0.6454374569615305,
+      "grad_norm": 7.042094122105951,
+      "learning_rate": 1.6824324171634946e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.5,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3797,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 2.015625,
+      "rewards/rejected": -9.5625,
+      "step": 17340
+    },
+    {
+      "epoch": 0.6458096815618545,
+      "grad_norm": 6.778785421852726,
+      "learning_rate": 1.679363321473871e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3949,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -9.4375,
+      "step": 17350
+    },
+    {
+      "epoch": 0.6461819061621783,
+      "grad_norm": 9.533871060843625,
+      "learning_rate": 1.6762956112948756e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4092,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.4375,
+      "step": 17360
+    },
+    {
+      "epoch": 0.6465541307625021,
+      "grad_norm": 6.985718230366509,
+      "learning_rate": 1.6732292918058356e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3866,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -9.4375,
+      "step": 17370
+    },
+    {
+      "epoch": 0.6469263553628259,
+      "grad_norm": 7.270240359276295,
+      "learning_rate": 1.670164368183728e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3665,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -9.375,
+      "step": 17380
+    },
+    {
+      "epoch": 0.6472985799631498,
+      "grad_norm": 5.791812381802103,
+      "learning_rate": 1.6671008456031746e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3793,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.25,
+      "step": 17390
+    },
+    {
+      "epoch": 0.6476708045634736,
+      "grad_norm": 6.409672710425608,
+      "learning_rate": 1.6640387292364337e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4021,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -9.0,
+      "step": 17400
+    },
+    {
+      "epoch": 0.6480430291637974,
+      "grad_norm": 7.101222239341493,
+      "learning_rate": 1.6609780242533845e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3612,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -9.25,
+      "step": 17410
+    },
+    {
+      "epoch": 0.6484152537641212,
+      "grad_norm": 7.233978746984645,
+      "learning_rate": 1.657918735821528e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3871,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.3125,
+      "step": 17420
+    },
+    {
+      "epoch": 0.6487874783644451,
+      "grad_norm": 6.618369731241595,
+      "learning_rate": 1.6548608691059702e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.4114,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.5625,
+      "step": 17430
+    },
+    {
+      "epoch": 0.649159702964769,
+      "grad_norm": 7.384625456220775,
+      "learning_rate": 1.6518044292694192e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3811,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -9.3125,
+      "step": 17440
+    },
+    {
+      "epoch": 0.6495319275650928,
+      "grad_norm": 7.460711541105908,
+      "learning_rate": 1.6487494214721726e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3998,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": -9.625,
+      "step": 17450
+    },
+    {
+      "epoch": 0.6499041521654166,
+      "grad_norm": 7.352805287405187,
+      "learning_rate": 1.6456958508721102e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3722,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 2.03125,
+      "rewards/rejected": -9.5,
+      "step": 17460
+    },
+    {
+      "epoch": 0.6502763767657405,
+      "grad_norm": 8.45893085829002,
+      "learning_rate": 1.6426437226246885e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.395,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 2.125,
+      "rewards/rejected": -9.5625,
+      "step": 17470
+    },
+    {
+      "epoch": 0.6506486013660643,
+      "grad_norm": 9.353184086507241,
+      "learning_rate": 1.6395930418829228e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3878,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.375,
+      "step": 17480
+    },
+    {
+      "epoch": 0.6510208259663881,
+      "grad_norm": 7.877546649224869,
+      "learning_rate": 1.63654381379739e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3856,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.3125,
+      "step": 17490
+    },
+    {
+      "epoch": 0.6513930505667119,
+      "grad_norm": 7.711441193139582,
+      "learning_rate": 1.6334960435162112e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3758,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.9453125,
+      "rewards/rejected": -9.3125,
+      "step": 17500
+    },
+    {
+      "epoch": 0.6517652751670358,
+      "grad_norm": 7.451389704977642,
+      "learning_rate": 1.6304497361850483e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3676,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 2.078125,
+      "rewards/rejected": -9.5,
+      "step": 17510
+    },
+    {
+      "epoch": 0.6521374997673596,
+      "grad_norm": 8.4681784783912,
+      "learning_rate": 1.6274048969470912e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3721,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.9453125,
+      "rewards/rejected": -9.25,
+      "step": 17520
+    },
+    {
+      "epoch": 0.6525097243676835,
+      "grad_norm": 7.6496524926552025,
+      "learning_rate": 1.6243615309430535e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.386,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -9.25,
+      "step": 17530
+    },
+    {
+      "epoch": 0.6528819489680073,
+      "grad_norm": 7.795735270367461,
+      "learning_rate": 1.621319643311159e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3885,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.5,
+      "step": 17540
+    },
+    {
+      "epoch": 0.6532541735683312,
+      "grad_norm": 7.278415624883429,
+      "learning_rate": 1.618279239187138e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3956,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.125,
+      "step": 17550
+    },
+    {
+      "epoch": 0.653626398168655,
+      "grad_norm": 6.796088150519844,
+      "learning_rate": 1.6152403237042145e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.296875,
+      "logps/chosen": -888.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.403,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -8.8125,
+      "step": 17560
+    },
+    {
+      "epoch": 0.6539986227689788,
+      "grad_norm": 7.715408966225244,
+      "learning_rate": 1.6122029019930987e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3736,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -9.25,
+      "step": 17570
+    },
+    {
+      "epoch": 0.6543708473693026,
+      "grad_norm": 7.143515343647237,
+      "learning_rate": 1.609166979181981e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3942,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.1875,
+      "step": 17580
+    },
+    {
+      "epoch": 0.6547430719696264,
+      "grad_norm": 8.416179909096329,
+      "learning_rate": 1.6061325603965193e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.75,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3759,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.0,
+      "step": 17590
+    },
+    {
+      "epoch": 0.6551152965699503,
+      "grad_norm": 6.9707672038639314,
+      "learning_rate": 1.6030996507598327e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3784,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -8.9375,
+      "step": 17600
+    },
+    {
+      "epoch": 0.6554875211702741,
+      "grad_norm": 8.611599387947996,
+      "learning_rate": 1.6000682553924922e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3939,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -8.9375,
+      "step": 17610
+    },
+    {
+      "epoch": 0.655859745770598,
+      "grad_norm": 8.46781737314284,
+      "learning_rate": 1.5970383794125128e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3865,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.3125,
+      "step": 17620
+    },
+    {
+      "epoch": 0.6562319703709218,
+      "grad_norm": 6.357379130731856,
+      "learning_rate": 1.5940100279353434e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.369,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.25,
+      "step": 17630
+    },
+    {
+      "epoch": 0.6566041949712457,
+      "grad_norm": 7.81546262031526,
+      "learning_rate": 1.5909832060738592e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.75,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4057,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.125,
+      "step": 17640
+    },
+    {
+      "epoch": 0.6569764195715695,
+      "grad_norm": 7.9725094738272695,
+      "learning_rate": 1.5879579189383549e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3596,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.9765625,
+      "rewards/rejected": -9.375,
+      "step": 17650
+    },
+    {
+      "epoch": 0.6573486441718933,
+      "grad_norm": 6.58370021018874,
+      "learning_rate": 1.5849341716365306e-07,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3845,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.375,
+      "step": 17660
+    },
+    {
+      "epoch": 0.6577208687722171,
+      "grad_norm": 8.32289943348736,
+      "learning_rate": 1.5819119692734892e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3826,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.125,
+      "step": 17670
+    },
+    {
+      "epoch": 0.658093093372541,
+      "grad_norm": 8.438492402459849,
+      "learning_rate": 1.5788913169517237e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.75,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3759,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.125,
+      "step": 17680
+    },
+    {
+      "epoch": 0.6584653179728648,
+      "grad_norm": 7.786568589824084,
+      "learning_rate": 1.5758722197711122e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.75,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3869,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 2.203125,
+      "rewards/rejected": -9.5,
+      "step": 17690
+    },
+    {
+      "epoch": 0.6588375425731887,
+      "grad_norm": 6.340120099963788,
+      "learning_rate": 1.5728546828289042e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3792,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -9.3125,
+      "step": 17700
+    },
+    {
+      "epoch": 0.6592097671735125,
+      "grad_norm": 6.480794637414619,
+      "learning_rate": 1.5698387112197177e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3767,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 2.25,
+      "rewards/rejected": -9.625,
+      "step": 17710
+    },
+    {
+      "epoch": 0.6595819917738364,
+      "grad_norm": 7.691378824651244,
+      "learning_rate": 1.5668243100355262e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3818,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.1875,
+      "step": 17720
+    },
+    {
+      "epoch": 0.6599542163741602,
+      "grad_norm": 8.770212993871144,
+      "learning_rate": 1.5638114843656524e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3615,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.0625,
+      "step": 17730
+    },
+    {
+      "epoch": 0.660326440974484,
+      "grad_norm": 6.9755361853610465,
+      "learning_rate": 1.5608002392967603e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3694,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.5625,
+      "step": 17740
+    },
+    {
+      "epoch": 0.6606986655748078,
+      "grad_norm": 7.988625982378954,
+      "learning_rate": 1.5577905799128422e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3827,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.828125,
+      "rewards/rejected": -9.375,
+      "step": 17750
+    },
+    {
+      "epoch": 0.6610708901751317,
+      "grad_norm": 7.991723252338846,
+      "learning_rate": 1.5547825112952166e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4052,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -8.9375,
+      "step": 17760
+    },
+    {
+      "epoch": 0.6614431147754555,
+      "grad_norm": 7.997154343806163,
+      "learning_rate": 1.551776038522513e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3999,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.4375,
+      "step": 17770
+    },
+    {
+      "epoch": 0.6618153393757793,
+      "grad_norm": 6.956609761808537,
+      "learning_rate": 1.54877116667067e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4062,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.125,
+      "step": 17780
+    },
+    {
+      "epoch": 0.6621875639761032,
+      "grad_norm": 6.057690662030335,
+      "learning_rate": 1.5457679008129203e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3927,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.125,
+      "step": 17790
+    },
+    {
+      "epoch": 0.6625597885764271,
+      "grad_norm": 7.5949255006277605,
+      "learning_rate": 1.542766246019787e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3987,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.0,
+      "step": 17800
+    },
+    {
+      "epoch": 0.6629320131767509,
+      "grad_norm": 7.683763890522869,
+      "learning_rate": 1.539766207359073e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3839,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.0,
+      "step": 17810
+    },
+    {
+      "epoch": 0.6633042377770747,
+      "grad_norm": 6.532733183884192,
+      "learning_rate": 1.5367677898958515e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3905,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.125,
+      "step": 17820
+    },
+    {
+      "epoch": 0.6636764623773985,
+      "grad_norm": 7.871885734956697,
+      "learning_rate": 1.53377099869246e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3957,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.0625,
+      "step": 17830
+    },
+    {
+      "epoch": 0.6640486869777223,
+      "grad_norm": 6.808802768073819,
+      "learning_rate": 1.5307758388084884e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3843,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.96875,
+      "rewards/rejected": -9.3125,
+      "step": 17840
+    },
+    {
+      "epoch": 0.6644209115780462,
+      "grad_norm": 8.030763868651215,
+      "learning_rate": 1.5277823153007754e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.4119,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.0,
+      "step": 17850
+    },
+    {
+      "epoch": 0.66479313617837,
+      "grad_norm": 7.571473484851187,
+      "learning_rate": 1.5247904332233936e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3844,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -9.0625,
+      "step": 17860
+    },
+    {
+      "epoch": 0.6651653607786938,
+      "grad_norm": 6.738871790251661,
+      "learning_rate": 1.521800197627647e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3778,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -9.4375,
+      "step": 17870
+    },
+    {
+      "epoch": 0.6655375853790177,
+      "grad_norm": 8.025373089505704,
+      "learning_rate": 1.5188116135620577e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4229,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -9.1875,
+      "step": 17880
+    },
+    {
+      "epoch": 0.6659098099793416,
+      "grad_norm": 6.979429716508786,
+      "learning_rate": 1.515824686072361e-07,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3823,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -9.0625,
+      "step": 17890
+    },
+    {
+      "epoch": 0.6662820345796654,
+      "grad_norm": 6.671378584773726,
+      "learning_rate": 1.5128394202014948e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3773,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -8.9375,
+      "step": 17900
+    },
+    {
+      "epoch": 0.6666542591799892,
+      "grad_norm": 7.714565162408182,
+      "learning_rate": 1.5098558209895914e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3738,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.25,
+      "step": 17910
+    },
+    {
+      "epoch": 0.667026483780313,
+      "grad_norm": 6.276840013586024,
+      "learning_rate": 1.50687389347397e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3692,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.125,
+      "step": 17920
+    },
+    {
+      "epoch": 0.6673987083806369,
+      "grad_norm": 7.3506717867648295,
+      "learning_rate": 1.5038936426891259e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3831,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -8.875,
+      "step": 17930
+    },
+    {
+      "epoch": 0.6677709329809607,
+      "grad_norm": 7.774949445370347,
+      "learning_rate": 1.5009150736667247e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3741,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.921875,
+      "rewards/rejected": -9.0625,
+      "step": 17940
+    },
+    {
+      "epoch": 0.6681431575812845,
+      "grad_norm": 6.870185693234993,
+      "learning_rate": 1.4979381914355928e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3817,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.375,
+      "step": 17950
+    },
+    {
+      "epoch": 0.6685153821816083,
+      "grad_norm": 8.482904606439572,
+      "learning_rate": 1.4949630010217087e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.4041,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -9.0625,
+      "step": 17960
+    },
+    {
+      "epoch": 0.6688876067819323,
+      "grad_norm": 8.397600846863826,
+      "learning_rate": 1.491989507448192e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3844,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.0,
+      "step": 17970
+    },
+    {
+      "epoch": 0.6692598313822561,
+      "grad_norm": 8.130276189145672,
+      "learning_rate": 1.4890177157353017e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3706,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.828125,
+      "rewards/rejected": -9.0,
+      "step": 17980
+    },
+    {
+      "epoch": 0.6696320559825799,
+      "grad_norm": 7.648259189875532,
+      "learning_rate": 1.486047630900421e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.389,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -9.0,
+      "step": 17990
+    },
+    {
+      "epoch": 0.6700042805829037,
+      "grad_norm": 6.07605007352532,
+      "learning_rate": 1.4830792579580514e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4008,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -8.9375,
+      "step": 18000
+    },
+    {
+      "epoch": 0.6703765051832276,
+      "grad_norm": 8.554490023642392,
+      "learning_rate": 1.4801126019198045e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3838,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.3125,
+      "step": 18010
+    },
+    {
+      "epoch": 0.6707487297835514,
+      "grad_norm": 6.641347048249407,
+      "learning_rate": 1.4771476677943934e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3867,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.4375,
+      "step": 18020
+    },
+    {
+      "epoch": 0.6711209543838752,
+      "grad_norm": 8.125100279140668,
+      "learning_rate": 1.4741844605876241e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3875,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.25,
+      "step": 18030
+    },
+    {
+      "epoch": 0.671493178984199,
+      "grad_norm": 7.533340335951724,
+      "learning_rate": 1.4712229853023845e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3837,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -9.125,
+      "step": 18040
+    },
+    {
+      "epoch": 0.6718654035845228,
+      "grad_norm": 6.614066059678995,
+      "learning_rate": 1.468263246938643e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4014,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.375,
+      "step": 18050
+    },
+    {
+      "epoch": 0.6722376281848468,
+      "grad_norm": 7.254591252547914,
+      "learning_rate": 1.4653052504934327e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4061,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.375,
+      "step": 18060
+    },
+    {
+      "epoch": 0.6726098527851706,
+      "grad_norm": 7.162032675450714,
+      "learning_rate": 1.4623490009608453e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3701,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -9.0,
+      "step": 18070
+    },
+    {
+      "epoch": 0.6729820773854944,
+      "grad_norm": 7.04059154120853,
+      "learning_rate": 1.4593945033320252e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.41,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.0,
+      "step": 18080
+    },
+    {
+      "epoch": 0.6733543019858182,
+      "grad_norm": 6.893010869851695,
+      "learning_rate": 1.4564417625951558e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3872,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -9.125,
+      "step": 18090
+    },
+    {
+      "epoch": 0.6737265265861421,
+      "grad_norm": 8.39628289291,
+      "learning_rate": 1.4534907837354577e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.5,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3808,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": -9.4375,
+      "step": 18100
+    },
+    {
+      "epoch": 0.6740987511864659,
+      "grad_norm": 7.507796246027934,
+      "learning_rate": 1.4505415717351752e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.5,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3845,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.375,
+      "step": 18110
+    },
+    {
+      "epoch": 0.6744709757867897,
+      "grad_norm": 7.35796444155586,
+      "learning_rate": 1.4475941315735704e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3718,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -9.1875,
+      "step": 18120
+    },
+    {
+      "epoch": 0.6748432003871135,
+      "grad_norm": 8.321434858983238,
+      "learning_rate": 1.4446484682269117e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3911,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.25,
+      "step": 18130
+    },
+    {
+      "epoch": 0.6752154249874375,
+      "grad_norm": 7.115933853084183,
+      "learning_rate": 1.4417045866684698e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4002,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.1875,
+      "step": 18140
+    },
+    {
+      "epoch": 0.6755876495877613,
+      "grad_norm": 7.542830705248919,
+      "learning_rate": 1.4387624918685072e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3939,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -9.1875,
+      "step": 18150
+    },
+    {
+      "epoch": 0.6759598741880851,
+      "grad_norm": 6.866479859807674,
+      "learning_rate": 1.4358221887942684e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.372,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.3125,
+      "step": 18160
+    },
+    {
+      "epoch": 0.6763320987884089,
+      "grad_norm": 6.9269356581410815,
+      "learning_rate": 1.432883682409975e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3891,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.1875,
+      "step": 18170
+    },
+    {
+      "epoch": 0.6767043233887328,
+      "grad_norm": 7.5346146422333575,
+      "learning_rate": 1.4299469776768116e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.4042,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.5,
+      "step": 18180
+    },
+    {
+      "epoch": 0.6770765479890566,
+      "grad_norm": 7.003066318704803,
+      "learning_rate": 1.4270120795529245e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3839,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.0,
+      "step": 18190
+    },
+    {
+      "epoch": 0.6774487725893804,
+      "grad_norm": 7.573161612694521,
+      "learning_rate": 1.424078992993408e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4023,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.0,
+      "step": 18200
+    },
+    {
+      "epoch": 0.6778209971897042,
+      "grad_norm": 8.683266084544984,
+      "learning_rate": 1.4211477229502996e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1020.0,
+      "loss": 0.3862,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.125,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -8.5625,
+      "step": 18210
+    },
+    {
+      "epoch": 0.6781932217900281,
+      "grad_norm": 7.693060684866391,
+      "learning_rate": 1.418218274372567e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3931,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -8.875,
+      "step": 18220
+    },
+    {
+      "epoch": 0.678565446390352,
+      "grad_norm": 6.905951124138664,
+      "learning_rate": 1.4152906522061047e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3996,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -9.0625,
+      "step": 18230
+    },
+    {
+      "epoch": 0.6789376709906758,
+      "grad_norm": 7.384635884111532,
+      "learning_rate": 1.412364861393724e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3875,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -9.0,
+      "step": 18240
+    },
+    {
+      "epoch": 0.6793098955909996,
+      "grad_norm": 8.364081560387007,
+      "learning_rate": 1.4094409068751428e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3899,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.0,
+      "step": 18250
+    },
+    {
+      "epoch": 0.6796821201913235,
+      "grad_norm": 8.601876354662146,
+      "learning_rate": 1.406518793586981e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3794,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.25,
+      "step": 18260
+    },
+    {
+      "epoch": 0.6800543447916473,
+      "grad_norm": 8.819539522394559,
+      "learning_rate": 1.4035985264627456e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4006,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.25,
+      "step": 18270
+    },
+    {
+      "epoch": 0.6804265693919711,
+      "grad_norm": 7.350958374800609,
+      "learning_rate": 1.400680110432831e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3843,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.25,
+      "step": 18280
+    },
+    {
+      "epoch": 0.6807987939922949,
+      "grad_norm": 7.132154356650205,
+      "learning_rate": 1.3977635504245047e-07,
+      "logits/chosen": -3.546875,
+      "logits/rejected": -3.328125,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3604,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.3125,
+      "step": 18290
+    },
+    {
+      "epoch": 0.6811710185926187,
+      "grad_norm": 8.657679967173712,
+      "learning_rate": 1.394848851361901e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.40625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3921,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.1875,
+      "step": 18300
+    },
+    {
+      "epoch": 0.6815432431929426,
+      "grad_norm": 8.359870108483511,
+      "learning_rate": 1.3919360181660107e-07,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3995,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.1875,
+      "step": 18310
+    },
+    {
+      "epoch": 0.6819154677932665,
+      "grad_norm": 9.378476061301928,
+      "learning_rate": 1.3890250557546763e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3845,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.1875,
+      "step": 18320
+    },
+    {
+      "epoch": 0.6822876923935903,
+      "grad_norm": 8.602130020835867,
+      "learning_rate": 1.3861159690425806e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4025,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -8.9375,
+      "step": 18330
+    },
+    {
+      "epoch": 0.6826599169939142,
+      "grad_norm": 8.021595648031884,
+      "learning_rate": 1.3832087629412402e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3848,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.0,
+      "step": 18340
+    },
+    {
+      "epoch": 0.683032141594238,
+      "grad_norm": 7.402560429241675,
+      "learning_rate": 1.3803034423589982e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3866,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.25,
+      "step": 18350
+    },
+    {
+      "epoch": 0.6834043661945618,
+      "grad_norm": 6.63553992344501,
+      "learning_rate": 1.37740001220101e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3894,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -9.1875,
+      "step": 18360
+    },
+    {
+      "epoch": 0.6837765907948856,
+      "grad_norm": 7.506600031750079,
+      "learning_rate": 1.3744984773692425e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3837,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.96875,
+      "rewards/rejected": -9.625,
+      "step": 18370
+    },
+    {
+      "epoch": 0.6841488153952094,
+      "grad_norm": 7.424224785763181,
+      "learning_rate": 1.3715988427624632e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3884,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -9.0625,
+      "step": 18380
+    },
+    {
+      "epoch": 0.6845210399955333,
+      "grad_norm": 8.234275999065385,
+      "learning_rate": 1.3687011132762288e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3755,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -9.4375,
+      "step": 18390
+    },
+    {
+      "epoch": 0.6848932645958571,
+      "grad_norm": 9.010785268811787,
+      "learning_rate": 1.3658052938028834e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4019,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -9.3125,
+      "step": 18400
+    },
+    {
+      "epoch": 0.685265489196181,
+      "grad_norm": 7.086300824155898,
+      "learning_rate": 1.362911389231541e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -988.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3836,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.4375,
+      "step": 18410
+    },
+    {
+      "epoch": 0.6856377137965048,
+      "grad_norm": 7.02549916739348,
+      "learning_rate": 1.3600194044480866e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4096,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.125,
+      "step": 18420
+    },
+    {
+      "epoch": 0.6860099383968287,
+      "grad_norm": 8.302558557760715,
+      "learning_rate": 1.3571293443351627e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4009,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.5,
+      "step": 18430
+    },
+    {
+      "epoch": 0.6863821629971525,
+      "grad_norm": 7.16662530441311,
+      "learning_rate": 1.3542412137721643e-07,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3819,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.3125,
+      "step": 18440
+    },
+    {
+      "epoch": 0.6867543875974763,
+      "grad_norm": 7.250320510589996,
+      "learning_rate": 1.351355017635224e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4053,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.4375,
+      "step": 18450
+    },
+    {
+      "epoch": 0.6871266121978001,
+      "grad_norm": 6.019292678197596,
+      "learning_rate": 1.3484707607972134e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3675,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.375,
+      "step": 18460
+    },
+    {
+      "epoch": 0.687498836798124,
+      "grad_norm": 6.16050497037097,
+      "learning_rate": 1.3455884481277253e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.367,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.625,
+      "step": 18470
+    },
+    {
+      "epoch": 0.6878710613984478,
+      "grad_norm": 7.831060143008705,
+      "learning_rate": 1.342708084493075e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3557,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": -9.5625,
+      "step": 18480
+    },
+    {
+      "epoch": 0.6882432859987716,
+      "grad_norm": 8.267383124788537,
+      "learning_rate": 1.339829674756285e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.388,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -9.3125,
+      "step": 18490
+    },
+    {
+      "epoch": 0.6886155105990955,
+      "grad_norm": 9.132429083854552,
+      "learning_rate": 1.336953223777077e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3678,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.0625,
+      "step": 18500
+    },
+    {
+      "epoch": 0.6889877351994194,
+      "grad_norm": 8.505335690781891,
+      "learning_rate": 1.3340787364118688e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1032.0,
+      "loss": 0.4108,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -8.8125,
+      "step": 18510
+    },
+    {
+      "epoch": 0.6893599597997432,
+      "grad_norm": 7.726529456599936,
+      "learning_rate": 1.331206217513759e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3947,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -8.9375,
+      "step": 18520
+    },
+    {
+      "epoch": 0.689732184400067,
+      "grad_norm": 8.872996791038082,
+      "learning_rate": 1.328335671932529e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.4074,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.0,
+      "step": 18530
+    },
+    {
+      "epoch": 0.6901044090003908,
+      "grad_norm": 8.092977051796215,
+      "learning_rate": 1.3254671045146222e-07,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4059,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.0,
+      "step": 18540
+    },
+    {
+      "epoch": 0.6904766336007147,
+      "grad_norm": 7.22462235567666,
+      "learning_rate": 1.322600520103146e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.5,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4094,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 2.046875,
+      "rewards/rejected": -9.25,
+      "step": 18550
+    },
+    {
+      "epoch": 0.6908488582010385,
+      "grad_norm": 9.479235670142375,
+      "learning_rate": 1.319735923537857e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3882,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.125,
+      "step": 18560
+    },
+    {
+      "epoch": 0.6912210828013623,
+      "grad_norm": 7.308703004239579,
+      "learning_rate": 1.3168733196551596e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.39,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.3125,
+      "step": 18570
+    },
+    {
+      "epoch": 0.6915933074016862,
+      "grad_norm": 6.80595819075359,
+      "learning_rate": 1.314012713288092e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3763,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.0625,
+      "step": 18580
+    },
+    {
+      "epoch": 0.69196553200201,
+      "grad_norm": 7.3270344793655875,
+      "learning_rate": 1.3111541092663168e-07,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.28125,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3871,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.3125,
+      "step": 18590
+    },
+    {
+      "epoch": 0.6923377566023339,
+      "grad_norm": 6.051578141509356,
+      "learning_rate": 1.308297512416121e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3873,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -8.9375,
+      "step": 18600
+    },
+    {
+      "epoch": 0.6927099812026577,
+      "grad_norm": 6.682986169838317,
+      "learning_rate": 1.3054429275603983e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3686,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.3125,
+      "step": 18610
+    },
+    {
+      "epoch": 0.6930822058029815,
+      "grad_norm": 6.235463298433478,
+      "learning_rate": 1.302590359518651e-07,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -992.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3869,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.171875,
+      "rewards/rejected": -9.0625,
+      "step": 18620
+    },
+    {
+      "epoch": 0.6934544304033053,
+      "grad_norm": 7.002236343649768,
+      "learning_rate": 1.29973981310697e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3865,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -9.375,
+      "step": 18630
+    },
+    {
+      "epoch": 0.6938266550036292,
+      "grad_norm": 7.964794938641898,
+      "learning_rate": 1.2968912931380375e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3878,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.90625,
+      "rewards/rejected": -9.625,
+      "step": 18640
+    },
+    {
+      "epoch": 0.694198879603953,
+      "grad_norm": 8.431792682688622,
+      "learning_rate": 1.2940448044211134e-07,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3886,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.1875,
+      "step": 18650
+    },
+    {
+      "epoch": 0.6945711042042768,
+      "grad_norm": 9.568570362662708,
+      "learning_rate": 1.2912003517620252e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.75,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3846,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.1875,
+      "step": 18660
+    },
+    {
+      "epoch": 0.6949433288046007,
+      "grad_norm": 7.797695703828273,
+      "learning_rate": 1.2883579399631689e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3971,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.5625,
+      "step": 18670
+    },
+    {
+      "epoch": 0.6953155534049246,
+      "grad_norm": 7.652610669504206,
+      "learning_rate": 1.2855175738234886e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3711,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.25,
+      "step": 18680
+    },
+    {
+      "epoch": 0.6956877780052484,
+      "grad_norm": 8.12784834288147,
+      "learning_rate": 1.282679258138479e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3695,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.375,
+      "step": 18690
+    },
+    {
+      "epoch": 0.6960600026055722,
+      "grad_norm": 10.802294094844664,
+      "learning_rate": 1.27984299770017e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3767,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.5625,
+      "step": 18700
+    },
+    {
+      "epoch": 0.696432227205896,
+      "grad_norm": 8.078290276621237,
+      "learning_rate": 1.277008797297123e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3818,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.375,
+      "step": 18710
+    },
+    {
+      "epoch": 0.6968044518062199,
+      "grad_norm": 6.944979135981245,
+      "learning_rate": 1.2741766617144218e-07,
+      "logits/chosen": -3.609375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1152.0,
+      "loss": 0.3776,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.96875,
+      "rewards/rejected": -9.8125,
+      "step": 18720
+    },
+    {
+      "epoch": 0.6971766764065437,
+      "grad_norm": 8.16437300140285,
+      "learning_rate": 1.2713465957336633e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3793,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -9.125,
+      "step": 18730
+    },
+    {
+      "epoch": 0.6975489010068675,
+      "grad_norm": 8.32917419932018,
+      "learning_rate": 1.268518604132951e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.396,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -9.25,
+      "step": 18740
+    },
+    {
+      "epoch": 0.6979211256071913,
+      "grad_norm": 8.083311461732476,
+      "learning_rate": 1.2656926916868833e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.386,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.0625,
+      "step": 18750
+    },
+    {
+      "epoch": 0.6982933502075153,
+      "grad_norm": 7.558118595013816,
+      "learning_rate": 1.262868863166554e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3797,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.96875,
+      "rewards/rejected": -9.375,
+      "step": 18760
+    },
+    {
+      "epoch": 0.6986655748078391,
+      "grad_norm": 7.4578964964410535,
+      "learning_rate": 1.260047123339532e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.4108,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.5,
+      "step": 18770
+    },
+    {
+      "epoch": 0.6990377994081629,
+      "grad_norm": 7.300139790287824,
+      "learning_rate": 1.2572274769698654e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4149,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -8.0,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -9.25,
+      "step": 18780
+    },
+    {
+      "epoch": 0.6994100240084867,
+      "grad_norm": 8.266171224416356,
+      "learning_rate": 1.2544099288180626e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3782,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.25,
+      "step": 18790
+    },
+    {
+      "epoch": 0.6997822486088106,
+      "grad_norm": 9.046171443224837,
+      "learning_rate": 1.2515944836410942e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.4079,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.4375,
+      "step": 18800
+    },
+    {
+      "epoch": 0.7001544732091344,
+      "grad_norm": 7.33581480514214,
+      "learning_rate": 1.248781146192377e-07,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -880.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3836,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.21875,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.0,
+      "step": 18810
+    },
+    {
+      "epoch": 0.7005266978094582,
+      "grad_norm": 9.24243454512702,
+      "learning_rate": 1.2459699212217713e-07,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3829,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.25,
+      "rewards/margins": 2.015625,
+      "rewards/rejected": -9.25,
+      "step": 18820
+    },
+    {
+      "epoch": 0.700898922409782,
+      "grad_norm": 7.535894780193304,
+      "learning_rate": 1.2431608134755712e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.375,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3797,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.5,
+      "step": 18830
+    },
+    {
+      "epoch": 0.7012711470101058,
+      "grad_norm": 8.091285798686318,
+      "learning_rate": 1.2403538276964926e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3858,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.1875,
+      "step": 18840
+    },
+    {
+      "epoch": 0.7016433716104298,
+      "grad_norm": 7.43413272524799,
+      "learning_rate": 1.2375489686236724e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3894,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.125,
+      "step": 18850
+    },
+    {
+      "epoch": 0.7020155962107536,
+      "grad_norm": 7.746951630529741,
+      "learning_rate": 1.2347462409926556e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3928,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.1875,
+      "step": 18860
+    },
+    {
+      "epoch": 0.7023878208110774,
+      "grad_norm": 7.082472461929826,
+      "learning_rate": 1.23194564953539e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3942,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -9.25,
+      "step": 18870
+    },
+    {
+      "epoch": 0.7027600454114012,
+      "grad_norm": 6.973848857528124,
+      "learning_rate": 1.229147198980214e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.40625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3523,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 2.125,
+      "rewards/rejected": -9.4375,
+      "step": 18880
+    },
+    {
+      "epoch": 0.7031322700117251,
+      "grad_norm": 8.119192524376171,
+      "learning_rate": 1.2263508940518532e-07,
+      "logits/chosen": -3.578125,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3882,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.25,
+      "step": 18890
+    },
+    {
+      "epoch": 0.7035044946120489,
+      "grad_norm": 9.448985232634346,
+      "learning_rate": 1.2235567394714112e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3824,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.1875,
+      "step": 18900
+    },
+    {
+      "epoch": 0.7038767192123727,
+      "grad_norm": 7.68914088328252,
+      "learning_rate": 1.2207647399563597e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3682,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.375,
+      "step": 18910
+    },
+    {
+      "epoch": 0.7042489438126965,
+      "grad_norm": 7.801215329845928,
+      "learning_rate": 1.2179749002205342e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4071,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.3125,
+      "step": 18920
+    },
+    {
+      "epoch": 0.7046211684130205,
+      "grad_norm": 8.439628020060733,
+      "learning_rate": 1.21518722497412e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.5,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3601,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": -9.375,
+      "step": 18930
+    },
+    {
+      "epoch": 0.7049933930133443,
+      "grad_norm": 8.337174293133925,
+      "learning_rate": 1.212401718923651e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.37,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.25,
+      "step": 18940
+    },
+    {
+      "epoch": 0.7053656176136681,
+      "grad_norm": 7.441976727284874,
+      "learning_rate": 1.2096183867719981e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3927,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.96875,
+      "rewards/rejected": -9.4375,
+      "step": 18950
+    },
+    {
+      "epoch": 0.7057378422139919,
+      "grad_norm": 6.922873939243078,
+      "learning_rate": 1.206837233218363e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3659,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.125,
+      "step": 18960
+    },
+    {
+      "epoch": 0.7061100668143158,
+      "grad_norm": 7.043289827445419,
+      "learning_rate": 1.2040582629582658e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3795,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.5,
+      "step": 18970
+    },
+    {
+      "epoch": 0.7064822914146396,
+      "grad_norm": 7.871198398541639,
+      "learning_rate": 1.2012814806835432e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4049,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.25,
+      "step": 18980
+    },
+    {
+      "epoch": 0.7068545160149634,
+      "grad_norm": 7.959471998441874,
+      "learning_rate": 1.198506891082338e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.5,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3941,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.125,
+      "step": 18990
+    },
+    {
+      "epoch": 0.7072267406152872,
+      "grad_norm": 7.035038194827172,
+      "learning_rate": 1.1957344988390903e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3895,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.375,
+      "step": 19000
+    },
+    {
+      "epoch": 0.707598965215611,
+      "grad_norm": 8.636248814666374,
+      "learning_rate": 1.192964308634531e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3823,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.8671875,
+      "rewards/rejected": -9.25,
+      "step": 19010
+    },
+    {
+      "epoch": 0.707971189815935,
+      "grad_norm": 7.0040908914217725,
+      "learning_rate": 1.1901963251456704e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3903,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.4375,
+      "step": 19020
+    },
+    {
+      "epoch": 0.7083434144162588,
+      "grad_norm": 9.701430496278931,
+      "learning_rate": 1.1874305530457967e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3771,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.1875,
+      "step": 19030
+    },
+    {
+      "epoch": 0.7087156390165826,
+      "grad_norm": 6.855117920448521,
+      "learning_rate": 1.1846669970044629e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3643,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.1875,
+      "step": 19040
+    },
+    {
+      "epoch": 0.7090878636169065,
+      "grad_norm": 8.265564464609023,
+      "learning_rate": 1.181905661687482e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3923,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -9.1875,
+      "step": 19050
+    },
+    {
+      "epoch": 0.7094600882172303,
+      "grad_norm": 6.874734618220727,
+      "learning_rate": 1.179146551756914e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3813,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.90625,
+      "rewards/rejected": -9.625,
+      "step": 19060
+    },
+    {
+      "epoch": 0.7098323128175541,
+      "grad_norm": 8.913594605875394,
+      "learning_rate": 1.1763896718710656e-07,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3669,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.3125,
+      "step": 19070
+    },
+    {
+      "epoch": 0.7102045374178779,
+      "grad_norm": 7.41334444682267,
+      "learning_rate": 1.1736350266844766e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3949,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 2.046875,
+      "rewards/rejected": -9.5625,
+      "step": 19080
+    },
+    {
+      "epoch": 0.7105767620182017,
+      "grad_norm": 8.75408203111484,
+      "learning_rate": 1.1708826208479145e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4085,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.375,
+      "step": 19090
+    },
+    {
+      "epoch": 0.7109489866185256,
+      "grad_norm": 6.990040493335933,
+      "learning_rate": 1.1681324590083663e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3708,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.375,
+      "step": 19100
+    },
+    {
+      "epoch": 0.7113212112188495,
+      "grad_norm": 7.716963592708909,
+      "learning_rate": 1.1653845458090286e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3728,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.3125,
+      "step": 19110
+    },
+    {
+      "epoch": 0.7116934358191733,
+      "grad_norm": 7.3883929041787555,
+      "learning_rate": 1.1626388858893033e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3712,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.1875,
+      "step": 19120
+    },
+    {
+      "epoch": 0.7120656604194971,
+      "grad_norm": 9.091454804636204,
+      "learning_rate": 1.1598954838847877e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3993,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -9.375,
+      "step": 19130
+    },
+    {
+      "epoch": 0.712437885019821,
+      "grad_norm": 6.660099120896379,
+      "learning_rate": 1.1571543444272674e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3802,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.125,
+      "step": 19140
+    },
+    {
+      "epoch": 0.7128101096201448,
+      "grad_norm": 7.406599519246462,
+      "learning_rate": 1.1544154721447058e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3807,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.9296875,
+      "rewards/rejected": -9.375,
+      "step": 19150
+    },
+    {
+      "epoch": 0.7131823342204686,
+      "grad_norm": 6.850006605130867,
+      "learning_rate": 1.1516788716612413e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3712,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.0625,
+      "step": 19160
+    },
+    {
+      "epoch": 0.7135545588207924,
+      "grad_norm": 7.0285145400381115,
+      "learning_rate": 1.1489445475971751e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.421875,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3991,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -8.75,
+      "step": 19170
+    },
+    {
+      "epoch": 0.7139267834211163,
+      "grad_norm": 6.78795941814448,
+      "learning_rate": 1.1462125045689661e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3675,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -8.9375,
+      "step": 19180
+    },
+    {
+      "epoch": 0.7142990080214401,
+      "grad_norm": 8.365212910726626,
+      "learning_rate": 1.1434827471892222e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3762,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.125,
+      "step": 19190
+    },
+    {
+      "epoch": 0.714671232621764,
+      "grad_norm": 20.930114779097597,
+      "learning_rate": 1.1407552800666895e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4026,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.5,
+      "step": 19200
+    },
+    {
+      "epoch": 0.7150434572220878,
+      "grad_norm": 7.152474616529817,
+      "learning_rate": 1.138030107806251e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1160.0,
+      "loss": 0.3961,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.6875,
+      "step": 19210
+    },
+    {
+      "epoch": 0.7154156818224117,
+      "grad_norm": 7.773034532449301,
+      "learning_rate": 1.1353072350089135e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4141,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.125,
+      "step": 19220
+    },
+    {
+      "epoch": 0.7157879064227355,
+      "grad_norm": 7.67273194782146,
+      "learning_rate": 1.1325866662718025e-07,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.385,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": -9.5625,
+      "step": 19230
+    },
+    {
+      "epoch": 0.7161601310230593,
+      "grad_norm": 7.424889183510072,
+      "learning_rate": 1.129868406188151e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3776,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.0,
+      "step": 19240
+    },
+    {
+      "epoch": 0.7165323556233831,
+      "grad_norm": 7.6193348533351815,
+      "learning_rate": 1.1271524593472973e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -996.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3772,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.375,
+      "step": 19250
+    },
+    {
+      "epoch": 0.716904580223707,
+      "grad_norm": 8.510848796314397,
+      "learning_rate": 1.1244388303346722e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4023,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.3125,
+      "step": 19260
+    },
+    {
+      "epoch": 0.7172768048240308,
+      "grad_norm": 7.610768449531311,
+      "learning_rate": 1.1217275237317942e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3877,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.375,
+      "step": 19270
+    },
+    {
+      "epoch": 0.7176490294243546,
+      "grad_norm": 8.512355345254507,
+      "learning_rate": 1.1190185441162613e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4014,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -9.125,
+      "step": 19280
+    },
+    {
+      "epoch": 0.7180212540246785,
+      "grad_norm": 7.531354891047735,
+      "learning_rate": 1.1163118960617402e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4052,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -9.3125,
+      "step": 19290
+    },
+    {
+      "epoch": 0.7183934786250024,
+      "grad_norm": 8.538965682100951,
+      "learning_rate": 1.1136075841379636e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.379,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.375,
+      "step": 19300
+    },
+    {
+      "epoch": 0.7187657032253262,
+      "grad_norm": 8.08798120046632,
+      "learning_rate": 1.1109056129107192e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.402,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -9.125,
+      "step": 19310
+    },
+    {
+      "epoch": 0.71913792782565,
+      "grad_norm": 8.124411284023273,
+      "learning_rate": 1.1082059869418428e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3798,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.25,
+      "step": 19320
+    },
+    {
+      "epoch": 0.7195101524259738,
+      "grad_norm": 8.333154684433953,
+      "learning_rate": 1.1055087107892123e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3814,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.4375,
+      "step": 19330
+    },
+    {
+      "epoch": 0.7198823770262976,
+      "grad_norm": 7.476419242766577,
+      "learning_rate": 1.102813789006734e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3787,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.125,
+      "step": 19340
+    },
+    {
+      "epoch": 0.7202546016266215,
+      "grad_norm": 8.325558354033056,
+      "learning_rate": 1.1001212261443435e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3602,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.5,
+      "step": 19350
+    },
+    {
+      "epoch": 0.7206268262269453,
+      "grad_norm": 7.674522951043031,
+      "learning_rate": 1.0974310267479919e-07,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3999,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.3125,
+      "step": 19360
+    },
+    {
+      "epoch": 0.7209990508272692,
+      "grad_norm": 7.411130915813204,
+      "learning_rate": 1.0947431953596412e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3838,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.1875,
+      "step": 19370
+    },
+    {
+      "epoch": 0.721371275427593,
+      "grad_norm": 8.651573301013912,
+      "learning_rate": 1.0920577365172528e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3939,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.1875,
+      "step": 19380
+    },
+    {
+      "epoch": 0.7217435000279169,
+      "grad_norm": 9.456610855384243,
+      "learning_rate": 1.0893746547547861e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -892.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.385,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.28125,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.0625,
+      "step": 19390
+    },
+    {
+      "epoch": 0.7221157246282407,
+      "grad_norm": 8.275002796025563,
+      "learning_rate": 1.0866939546021822e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3971,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -9.0625,
+      "step": 19400
+    },
+    {
+      "epoch": 0.7224879492285645,
+      "grad_norm": 6.776957307297741,
+      "learning_rate": 1.084015640585367e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.38,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.953125,
+      "rewards/rejected": -9.3125,
+      "step": 19410
+    },
+    {
+      "epoch": 0.7228601738288883,
+      "grad_norm": 7.876010536025482,
+      "learning_rate": 1.0813397172262354e-07,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3867,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.4375,
+      "step": 19420
+    },
+    {
+      "epoch": 0.7232323984292122,
+      "grad_norm": 7.6111798620878925,
+      "learning_rate": 1.0786661890426436e-07,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3837,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -9.3125,
+      "step": 19430
+    },
+    {
+      "epoch": 0.723604623029536,
+      "grad_norm": 8.875890548536699,
+      "learning_rate": 1.0759950605484078e-07,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3783,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.4375,
+      "step": 19440
+    },
+    {
+      "epoch": 0.7239768476298598,
+      "grad_norm": 7.139278504129174,
+      "learning_rate": 1.0733263362532885e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4023,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.828125,
+      "rewards/rejected": -9.3125,
+      "step": 19450
+    },
+    {
+      "epoch": 0.7243490722301837,
+      "grad_norm": 9.200148867106291,
+      "learning_rate": 1.0706600206629931e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3805,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -9.3125,
+      "step": 19460
+    },
+    {
+      "epoch": 0.7247212968305076,
+      "grad_norm": 7.3719496924308725,
+      "learning_rate": 1.0679961182791561e-07,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3817,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.25,
+      "step": 19470
+    },
+    {
+      "epoch": 0.7250935214308314,
+      "grad_norm": 8.03116297286317,
+      "learning_rate": 1.0653346335993424e-07,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3936,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.4375,
+      "step": 19480
+    },
+    {
+      "epoch": 0.7254657460311552,
+      "grad_norm": 7.518574259984923,
+      "learning_rate": 1.0626755711170302e-07,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3867,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -9.0625,
+      "step": 19490
+    },
+    {
+      "epoch": 0.725837970631479,
+      "grad_norm": 7.282859874864732,
+      "learning_rate": 1.0600189353216132e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3896,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.125,
+      "step": 19500
+    },
+    {
+      "epoch": 0.7262101952318029,
+      "grad_norm": 7.833298672394872,
+      "learning_rate": 1.0573647306983866e-07,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3833,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.375,
+      "step": 19510
+    },
+    {
+      "epoch": 0.7265824198321267,
+      "grad_norm": 7.072891334665583,
+      "learning_rate": 1.0547129617285377e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3797,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -9.375,
+      "step": 19520
+    },
+    {
+      "epoch": 0.7269546444324505,
+      "grad_norm": 7.672237760055815,
+      "learning_rate": 1.0520636328891467e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3822,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.375,
+      "step": 19530
+    },
+    {
+      "epoch": 0.7273268690327743,
+      "grad_norm": 8.07209985081442,
+      "learning_rate": 1.0494167486531677e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.355,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 2.078125,
+      "rewards/rejected": -9.5625,
+      "step": 19540
+    },
+    {
+      "epoch": 0.7276990936330983,
+      "grad_norm": 8.338060924781983,
+      "learning_rate": 1.0467723134894357e-07,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3762,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -9.4375,
+      "step": 19550
+    },
+    {
+      "epoch": 0.7280713182334221,
+      "grad_norm": 7.650734735768076,
+      "learning_rate": 1.0441303318626434e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3819,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.5,
+      "step": 19560
+    },
+    {
+      "epoch": 0.7284435428337459,
+      "grad_norm": 7.695932210472061,
+      "learning_rate": 1.0414908082333451e-07,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3855,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 2.140625,
+      "rewards/rejected": -9.6875,
+      "step": 19570
+    },
+    {
+      "epoch": 0.7288157674340697,
+      "grad_norm": 8.35569718132435,
+      "learning_rate": 1.0388537470579448e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3964,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -9.5,
+      "step": 19580
+    },
+    {
+      "epoch": 0.7291879920343936,
+      "grad_norm": 8.501836049144211,
+      "learning_rate": 1.0362191527886857e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3873,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": -9.5625,
+      "step": 19590
+    },
+    {
+      "epoch": 0.7295602166347174,
+      "grad_norm": 8.386269626326106,
+      "learning_rate": 1.0335870298736526e-07,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3961,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -8.9375,
+      "step": 19600
+    },
+    {
+      "epoch": 0.7299324412350412,
+      "grad_norm": 8.517234583309014,
+      "learning_rate": 1.030957382756751e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.4056,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.5,
+      "step": 19610
+    },
+    {
+      "epoch": 0.730304665835365,
+      "grad_norm": 6.668715277373133,
+      "learning_rate": 1.0283302158777113e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3904,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.1875,
+      "step": 19620
+    },
+    {
+      "epoch": 0.7306768904356888,
+      "grad_norm": 6.568912095456722,
+      "learning_rate": 1.0257055336720726e-07,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3664,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.3125,
+      "step": 19630
+    },
+    {
+      "epoch": 0.7310491150360128,
+      "grad_norm": 7.873174831673161,
+      "learning_rate": 1.0230833405711822e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3631,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.96875,
+      "rewards/rejected": -9.375,
+      "step": 19640
+    },
+    {
+      "epoch": 0.7314213396363366,
+      "grad_norm": 7.1763131529852275,
+      "learning_rate": 1.020463641002183e-07,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3774,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.1875,
+      "step": 19650
+    },
+    {
+      "epoch": 0.7317935642366604,
+      "grad_norm": 7.932583915752739,
+      "learning_rate": 1.0178464393880093e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3855,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.921875,
+      "rewards/rejected": -9.375,
+      "step": 19660
+    },
+    {
+      "epoch": 0.7321657888369842,
+      "grad_norm": 8.163695812235677,
+      "learning_rate": 1.0152317401473782e-07,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3804,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.25,
+      "step": 19670
+    },
+    {
+      "epoch": 0.7325380134373081,
+      "grad_norm": 9.024767704889392,
+      "learning_rate": 1.0126195476947779e-07,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3964,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.25,
+      "step": 19680
+    },
+    {
+      "epoch": 0.7329102380376319,
+      "grad_norm": 9.321773705568685,
+      "learning_rate": 1.0100098664404719e-07,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3829,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.1875,
+      "step": 19690
+    },
+    {
+      "epoch": 0.7332824626379557,
+      "grad_norm": 7.256947973367732,
+      "learning_rate": 1.0074027007904764e-07,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3759,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.25,
+      "step": 19700
+    },
+    {
+      "epoch": 0.7336546872382795,
+      "grad_norm": 7.102238423113803,
+      "learning_rate": 1.0047980551465653e-07,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3996,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.0625,
+      "step": 19710
+    },
+    {
+      "epoch": 0.7340269118386035,
+      "grad_norm": 6.839653397779881,
+      "learning_rate": 1.002195933906255e-07,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3681,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.25,
+      "step": 19720
+    },
+    {
+      "epoch": 0.7343991364389273,
+      "grad_norm": 7.123579304196685,
+      "learning_rate": 9.995963414628011e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3813,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -9.25,
+      "step": 19730
+    },
+    {
+      "epoch": 0.7347713610392511,
+      "grad_norm": 7.726269239289189,
+      "learning_rate": 9.969992822051904e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3773,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.3125,
+      "step": 19740
+    },
+    {
+      "epoch": 0.7351435856395749,
+      "grad_norm": 9.15410704498772,
+      "learning_rate": 9.944047605181319e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3783,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.25,
+      "step": 19750
+    },
+    {
+      "epoch": 0.7355158102398988,
+      "grad_norm": 7.641898838289834,
+      "learning_rate": 9.918127807820512e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3711,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 2.015625,
+      "rewards/rejected": -9.6875,
+      "step": 19760
+    },
+    {
+      "epoch": 0.7358880348402226,
+      "grad_norm": 7.8638503721680255,
+      "learning_rate": 9.892233473730799e-08,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3943,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -9.1875,
+      "step": 19770
+    },
+    {
+      "epoch": 0.7362602594405464,
+      "grad_norm": 7.126440868269323,
+      "learning_rate": 9.866364646630531e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3851,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.421875,
+      "rewards/rejected": -9.0,
+      "step": 19780
+    },
+    {
+      "epoch": 0.7366324840408702,
+      "grad_norm": 8.492718636847714,
+      "learning_rate": 9.84052137019499e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3821,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": -9.375,
+      "step": 19790
+    },
+    {
+      "epoch": 0.737004708641194,
+      "grad_norm": 7.795228276134456,
+      "learning_rate": 9.814703688056319e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3826,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.25,
+      "step": 19800
+    },
+    {
+      "epoch": 0.737376933241518,
+      "grad_norm": 7.507640258961813,
+      "learning_rate": 9.78891164380343e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3871,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.5,
+      "step": 19810
+    },
+    {
+      "epoch": 0.7377491578418418,
+      "grad_norm": 6.980266851087673,
+      "learning_rate": 9.763145280981974e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3992,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.375,
+      "step": 19820
+    },
+    {
+      "epoch": 0.7381213824421656,
+      "grad_norm": 7.3323689212383005,
+      "learning_rate": 9.737404643094235e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3728,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.25,
+      "step": 19830
+    },
+    {
+      "epoch": 0.7384936070424895,
+      "grad_norm": 6.9386483885580095,
+      "learning_rate": 9.711689773599067e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3644,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.5625,
+      "step": 19840
+    },
+    {
+      "epoch": 0.7388658316428133,
+      "grad_norm": 7.915830672937266,
+      "learning_rate": 9.686000715911819e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3835,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.90625,
+      "rewards/rejected": -9.25,
+      "step": 19850
+    },
+    {
+      "epoch": 0.7392380562431371,
+      "grad_norm": 8.95352560800234,
+      "learning_rate": 9.660337513404243e-08,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.4012,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.4375,
+      "step": 19860
+    },
+    {
+      "epoch": 0.7396102808434609,
+      "grad_norm": 8.075919008988745,
+      "learning_rate": 9.634700209404464e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3895,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.625,
+      "step": 19870
+    },
+    {
+      "epoch": 0.7399825054437847,
+      "grad_norm": 7.891910169540815,
+      "learning_rate": 9.609088847196866e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3773,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.4375,
+      "step": 19880
+    },
+    {
+      "epoch": 0.7403547300441086,
+      "grad_norm": 7.850702440358002,
+      "learning_rate": 9.583503470022053e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.421875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3988,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": -9.5625,
+      "step": 19890
+    },
+    {
+      "epoch": 0.7407269546444325,
+      "grad_norm": 6.037381520254476,
+      "learning_rate": 9.557944121076724e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3477,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -9.5,
+      "step": 19900
+    },
+    {
+      "epoch": 0.7410991792447563,
+      "grad_norm": 7.413380692041023,
+      "learning_rate": 9.532410843513666e-08,
+      "logits/chosen": -4.03125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3718,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.4375,
+      "step": 19910
+    },
+    {
+      "epoch": 0.7414714038450801,
+      "grad_norm": 7.582639668769937,
+      "learning_rate": 9.50690368044163e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3894,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.9453125,
+      "rewards/rejected": -9.3125,
+      "step": 19920
+    },
+    {
+      "epoch": 0.741843628445404,
+      "grad_norm": 7.427820329026051,
+      "learning_rate": 9.481422674925288e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.384,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.3125,
+      "step": 19930
+    },
+    {
+      "epoch": 0.7422158530457278,
+      "grad_norm": 8.479133339239517,
+      "learning_rate": 9.455967869985151e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3801,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -9.375,
+      "step": 19940
+    },
+    {
+      "epoch": 0.7425880776460516,
+      "grad_norm": 8.12132333751996,
+      "learning_rate": 9.430539308597474e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3979,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.125,
+      "step": 19950
+    },
+    {
+      "epoch": 0.7429603022463754,
+      "grad_norm": 6.853107294575637,
+      "learning_rate": 9.405137033694222e-08,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3791,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.3125,
+      "step": 19960
+    },
+    {
+      "epoch": 0.7433325268466993,
+      "grad_norm": 8.136209201667418,
+      "learning_rate": 9.37976108816298e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3667,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.0625,
+      "step": 19970
+    },
+    {
+      "epoch": 0.7437047514470231,
+      "grad_norm": 7.961009166615228,
+      "learning_rate": 9.354411514846882e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3895,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.9921875,
+      "rewards/rejected": -9.375,
+      "step": 19980
+    },
+    {
+      "epoch": 0.744076976047347,
+      "grad_norm": 7.961107418899133,
+      "learning_rate": 9.329088356544518e-08,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3712,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.25,
+      "step": 19990
+    },
+    {
+      "epoch": 0.7444492006476708,
+      "grad_norm": 6.707492237509258,
+      "learning_rate": 9.303791656009896e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.4019,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.5,
+      "step": 20000
+    },
+    {
+      "epoch": 0.7444492006476708,
+      "eval_logits/chosen": -3.84375,
+      "eval_logits/rejected": -3.65625,
+      "eval_logps/chosen": -984.0,
+      "eval_logps/rejected": -1104.0,
+      "eval_loss": 0.46099165081977844,
+      "eval_rewards/accuracies": 0.7533908486366272,
+      "eval_rewards/chosen": -7.84375,
+      "eval_rewards/margins": 1.515625,
+      "eval_rewards/rejected": -9.375,
+      "eval_runtime": 6273.6564,
+      "eval_samples_per_second": 30.454,
+      "eval_steps_per_second": 0.476,
+      "step": 20000
+    },
+    {
+      "epoch": 0.7448214252479947,
+      "grad_norm": 7.2449139191141025,
+      "learning_rate": 9.278521455952354e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3833,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.3125,
+      "step": 20010
+    },
+    {
+      "epoch": 0.7451936498483185,
+      "grad_norm": 8.0500684158033,
+      "learning_rate": 9.253277799036488e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3924,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.1875,
+      "step": 20020
+    },
+    {
+      "epoch": 0.7455658744486423,
+      "grad_norm": 8.619208887200024,
+      "learning_rate": 9.228060727882083e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3857,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.375,
+      "step": 20030
+    },
+    {
+      "epoch": 0.7459380990489661,
+      "grad_norm": 7.616474905220966,
+      "learning_rate": 9.202870285064021e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3779,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.125,
+      "step": 20040
+    },
+    {
+      "epoch": 0.74631032364929,
+      "grad_norm": 7.743687183078503,
+      "learning_rate": 9.177706513112246e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3781,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -9.125,
+      "step": 20050
+    },
+    {
+      "epoch": 0.7466825482496138,
+      "grad_norm": 8.499116797598386,
+      "learning_rate": 9.152569454511663e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3953,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.8671875,
+      "rewards/rejected": -9.625,
+      "step": 20060
+    },
+    {
+      "epoch": 0.7470547728499376,
+      "grad_norm": 7.639015754021952,
+      "learning_rate": 9.12745915170209e-08,
+      "logits/chosen": -3.640625,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3891,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -8.9375,
+      "step": 20070
+    },
+    {
+      "epoch": 0.7474269974502615,
+      "grad_norm": 7.663238106058885,
+      "learning_rate": 9.102375647078145e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3925,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.125,
+      "step": 20080
+    },
+    {
+      "epoch": 0.7477992220505854,
+      "grad_norm": 8.662648024141351,
+      "learning_rate": 9.077318982989222e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3614,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -7.15625,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": -9.125,
+      "step": 20090
+    },
+    {
+      "epoch": 0.7481714466509092,
+      "grad_norm": 9.085477473382282,
+      "learning_rate": 9.052289201739397e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3918,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.0625,
+      "step": 20100
+    },
+    {
+      "epoch": 0.748543671251233,
+      "grad_norm": 10.343320043773335,
+      "learning_rate": 9.027286345587354e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3835,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.375,
+      "step": 20110
+    },
+    {
+      "epoch": 0.7489158958515568,
+      "grad_norm": 7.488897865027447,
+      "learning_rate": 9.002310456746335e-08,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3815,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -8.8125,
+      "step": 20120
+    },
+    {
+      "epoch": 0.7492881204518806,
+      "grad_norm": 7.939986273859413,
+      "learning_rate": 8.977361577384013e-08,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3715,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -8.9375,
+      "step": 20130
+    },
+    {
+      "epoch": 0.7496603450522045,
+      "grad_norm": 8.35250032373012,
+      "learning_rate": 8.952439749622497e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3782,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.25,
+      "step": 20140
+    },
+    {
+      "epoch": 0.7500325696525283,
+      "grad_norm": 9.045397555003415,
+      "learning_rate": 8.927545015538212e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3945,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.1875,
+      "step": 20150
+    },
+    {
+      "epoch": 0.7504047942528522,
+      "grad_norm": 8.164338565604146,
+      "learning_rate": 8.902677417161839e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3797,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 2.0625,
+      "rewards/rejected": -9.375,
+      "step": 20160
+    },
+    {
+      "epoch": 0.750777018853176,
+      "grad_norm": 7.616977540765588,
+      "learning_rate": 8.877836996478252e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3916,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.5625,
+      "step": 20170
+    },
+    {
+      "epoch": 0.7511492434534999,
+      "grad_norm": 8.69316932165126,
+      "learning_rate": 8.853023795426421e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.5,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3928,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -9.3125,
+      "step": 20180
+    },
+    {
+      "epoch": 0.7515214680538237,
+      "grad_norm": 9.353599314491538,
+      "learning_rate": 8.828237855899373e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3782,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.4375,
+      "step": 20190
+    },
+    {
+      "epoch": 0.7518936926541475,
+      "grad_norm": 9.047139286346642,
+      "learning_rate": 8.803479219744111e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3804,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.1875,
+      "step": 20200
+    },
+    {
+      "epoch": 0.7522659172544713,
+      "grad_norm": 9.164246828622453,
+      "learning_rate": 8.778747928761548e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3748,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -9.25,
+      "step": 20210
+    },
+    {
+      "epoch": 0.7526381418547952,
+      "grad_norm": 6.535277400751438,
+      "learning_rate": 8.7540440247064e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3726,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": -9.375,
+      "step": 20220
+    },
+    {
+      "epoch": 0.753010366455119,
+      "grad_norm": 7.533278046350786,
+      "learning_rate": 8.729367549287168e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3891,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.3125,
+      "step": 20230
+    },
+    {
+      "epoch": 0.7533825910554428,
+      "grad_norm": 8.002691659942183,
+      "learning_rate": 8.704718544166046e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3838,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.375,
+      "step": 20240
+    },
+    {
+      "epoch": 0.7537548156557667,
+      "grad_norm": 7.055337751504257,
+      "learning_rate": 8.680097050958834e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4064,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.3125,
+      "step": 20250
+    },
+    {
+      "epoch": 0.7541270402560906,
+      "grad_norm": 7.728428792164973,
+      "learning_rate": 8.655503111234905e-08,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3884,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.1875,
+      "step": 20260
+    },
+    {
+      "epoch": 0.7544992648564144,
+      "grad_norm": 6.300699140598843,
+      "learning_rate": 8.630936766517081e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -908.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3797,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -8.9375,
+      "step": 20270
+    },
+    {
+      "epoch": 0.7548714894567382,
+      "grad_norm": 9.178410009671971,
+      "learning_rate": 8.606398058281619e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.421875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3658,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 2.015625,
+      "rewards/rejected": -9.4375,
+      "step": 20280
+    },
+    {
+      "epoch": 0.755243714057062,
+      "grad_norm": 8.101701705174449,
+      "learning_rate": 8.581887027958107e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.625,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3714,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.375,
+      "step": 20290
+    },
+    {
+      "epoch": 0.7556159386573859,
+      "grad_norm": 8.559119711704989,
+      "learning_rate": 8.557403716929418e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3828,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.5,
+      "step": 20300
+    },
+    {
+      "epoch": 0.7559881632577097,
+      "grad_norm": 8.585380408192094,
+      "learning_rate": 8.532948166531592e-08,
+      "logits/chosen": -3.640625,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4067,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.3125,
+      "step": 20310
+    },
+    {
+      "epoch": 0.7563603878580335,
+      "grad_norm": 9.215859410638696,
+      "learning_rate": 8.508520418053839e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3951,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.125,
+      "step": 20320
+    },
+    {
+      "epoch": 0.7567326124583573,
+      "grad_norm": 7.544619012481699,
+      "learning_rate": 8.484120512738386e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -896.0,
+      "logps/rejected": -1040.0,
+      "loss": 0.3846,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -8.875,
+      "step": 20330
+    },
+    {
+      "epoch": 0.7571048370586813,
+      "grad_norm": 7.425122831543079,
+      "learning_rate": 8.459748491780502e-08,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3823,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.4375,
+      "step": 20340
+    },
+    {
+      "epoch": 0.7574770616590051,
+      "grad_norm": 7.891208826066001,
+      "learning_rate": 8.435404396328355e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3925,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -9.0,
+      "step": 20350
+    },
+    {
+      "epoch": 0.7578492862593289,
+      "grad_norm": 7.828661944836035,
+      "learning_rate": 8.411088267482944e-08,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3844,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.1875,
+      "step": 20360
+    },
+    {
+      "epoch": 0.7582215108596527,
+      "grad_norm": 8.391382605095894,
+      "learning_rate": 8.386800146298087e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.421875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3771,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.3125,
+      "step": 20370
+    },
+    {
+      "epoch": 0.7585937354599765,
+      "grad_norm": 7.538089930554933,
+      "learning_rate": 8.362540073780275e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.4107,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.484375,
+      "rewards/rejected": -9.0,
+      "step": 20380
+    },
+    {
+      "epoch": 0.7589659600603004,
+      "grad_norm": 7.330187781190223,
+      "learning_rate": 8.338308090888696e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3875,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.4375,
+      "step": 20390
+    },
+    {
+      "epoch": 0.7593381846606242,
+      "grad_norm": 7.932278986389012,
+      "learning_rate": 8.314104238535066e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4052,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.125,
+      "step": 20400
+    },
+    {
+      "epoch": 0.759710409260948,
+      "grad_norm": 7.580772323734146,
+      "learning_rate": 8.289928557583639e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3652,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": -9.3125,
+      "step": 20410
+    },
+    {
+      "epoch": 0.7600826338612718,
+      "grad_norm": 7.21145705666848,
+      "learning_rate": 8.265781088851068e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3766,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 2.0,
+      "rewards/rejected": -9.4375,
+      "step": 20420
+    },
+    {
+      "epoch": 0.7604548584615958,
+      "grad_norm": 7.649007613826136,
+      "learning_rate": 8.241661873106426e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.4051,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.3125,
+      "step": 20430
+    },
+    {
+      "epoch": 0.7608270830619196,
+      "grad_norm": 7.660158225405949,
+      "learning_rate": 8.217570951071055e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4056,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.25,
+      "step": 20440
+    },
+    {
+      "epoch": 0.7611993076622434,
+      "grad_norm": 7.899040819222953,
+      "learning_rate": 8.193508363418514e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3705,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.0625,
+      "step": 20450
+    },
+    {
+      "epoch": 0.7615715322625672,
+      "grad_norm": 7.812854325649219,
+      "learning_rate": 8.169474150774563e-08,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.4057,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.3125,
+      "step": 20460
+    },
+    {
+      "epoch": 0.7619437568628911,
+      "grad_norm": 8.480951647350905,
+      "learning_rate": 8.145468353717006e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3777,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -9.125,
+      "step": 20470
+    },
+    {
+      "epoch": 0.7623159814632149,
+      "grad_norm": 7.159897500973909,
+      "learning_rate": 8.121491012775733e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3805,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -9.0,
+      "step": 20480
+    },
+    {
+      "epoch": 0.7626882060635387,
+      "grad_norm": 7.427749043637957,
+      "learning_rate": 8.09754216843253e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3715,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -9.25,
+      "step": 20490
+    },
+    {
+      "epoch": 0.7630604306638625,
+      "grad_norm": 7.776650558616942,
+      "learning_rate": 8.073621861121113e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3637,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -9.0,
+      "step": 20500
+    },
+    {
+      "epoch": 0.7634326552641864,
+      "grad_norm": 7.494907492302637,
+      "learning_rate": 8.049730131227005e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3903,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.375,
+      "step": 20510
+    },
+    {
+      "epoch": 0.7638048798645103,
+      "grad_norm": 7.986983490899758,
+      "learning_rate": 8.02586701908746e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.4,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -9.0,
+      "step": 20520
+    },
+    {
+      "epoch": 0.7641771044648341,
+      "grad_norm": 8.662329988252482,
+      "learning_rate": 8.002032564991459e-08,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1152.0,
+      "loss": 0.3976,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -8.0,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.75,
+      "step": 20530
+    },
+    {
+      "epoch": 0.7645493290651579,
+      "grad_norm": 7.958387365138824,
+      "learning_rate": 7.978226809179558e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3653,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.375,
+      "step": 20540
+    },
+    {
+      "epoch": 0.7649215536654818,
+      "grad_norm": 8.766762151634595,
+      "learning_rate": 7.95444979184389e-08,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4004,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -9.4375,
+      "step": 20550
+    },
+    {
+      "epoch": 0.7652937782658056,
+      "grad_norm": 8.36734056390295,
+      "learning_rate": 7.930701553128036e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3975,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -9.0,
+      "step": 20560
+    },
+    {
+      "epoch": 0.7656660028661294,
+      "grad_norm": 7.462089415837083,
+      "learning_rate": 7.906982133127018e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3775,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -9.125,
+      "step": 20570
+    },
+    {
+      "epoch": 0.7660382274664532,
+      "grad_norm": 8.201381509037992,
+      "learning_rate": 7.883291571887196e-08,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3881,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -9.25,
+      "step": 20580
+    },
+    {
+      "epoch": 0.766410452066777,
+      "grad_norm": 7.544236742512034,
+      "learning_rate": 7.859629909406196e-08,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3866,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.375,
+      "step": 20590
+    },
+    {
+      "epoch": 0.766782676667101,
+      "grad_norm": 9.171874871278174,
+      "learning_rate": 7.83599718563287e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4024,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.25,
+      "step": 20600
+    },
+    {
+      "epoch": 0.7671549012674248,
+      "grad_norm": 8.808357376711694,
+      "learning_rate": 7.812393440467177e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3867,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.3125,
+      "step": 20610
+    },
+    {
+      "epoch": 0.7675271258677486,
+      "grad_norm": 7.070364723164209,
+      "learning_rate": 7.78881871376021e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3734,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.1875,
+      "step": 20620
+    },
+    {
+      "epoch": 0.7678993504680725,
+      "grad_norm": 6.868926350147307,
+      "learning_rate": 7.765273045314005e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3805,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.953125,
+      "rewards/rejected": -9.5,
+      "step": 20630
+    },
+    {
+      "epoch": 0.7682715750683963,
+      "grad_norm": 7.068605929969223,
+      "learning_rate": 7.741756474881588e-08,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4011,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.125,
+      "step": 20640
+    },
+    {
+      "epoch": 0.7686437996687201,
+      "grad_norm": 9.251346803650792,
+      "learning_rate": 7.718269042166817e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3761,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -9.0625,
+      "step": 20650
+    },
+    {
+      "epoch": 0.7690160242690439,
+      "grad_norm": 6.961282640759425,
+      "learning_rate": 7.694810786824388e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3984,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.1875,
+      "step": 20660
+    },
+    {
+      "epoch": 0.7693882488693677,
+      "grad_norm": 7.278710212530748,
+      "learning_rate": 7.671381748459715e-08,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3917,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.25,
+      "step": 20670
+    },
+    {
+      "epoch": 0.7697604734696916,
+      "grad_norm": 7.345151005821905,
+      "learning_rate": 7.647981966628899e-08,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -884.0,
+      "logps/rejected": -1048.0,
+      "loss": 0.3611,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.1875,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -8.9375,
+      "step": 20680
+    },
+    {
+      "epoch": 0.7701326980700155,
+      "grad_norm": 8.298512218652705,
+      "learning_rate": 7.624611480838647e-08,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3641,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -9.0625,
+      "step": 20690
+    },
+    {
+      "epoch": 0.7705049226703393,
+      "grad_norm": 8.283047606954991,
+      "learning_rate": 7.601270330546177e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3784,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.0625,
+      "step": 20700
+    },
+    {
+      "epoch": 0.7708771472706631,
+      "grad_norm": 7.2237400466981585,
+      "learning_rate": 7.577958555159209e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -904.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3677,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.0625,
+      "step": 20710
+    },
+    {
+      "epoch": 0.771249371870987,
+      "grad_norm": 7.854870268066652,
+      "learning_rate": 7.554676194035859e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.4014,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.5625,
+      "step": 20720
+    },
+    {
+      "epoch": 0.7716215964713108,
+      "grad_norm": 7.686189339835748,
+      "learning_rate": 7.531423286484584e-08,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3854,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.6875,
+      "step": 20730
+    },
+    {
+      "epoch": 0.7719938210716346,
+      "grad_norm": 8.526745892650448,
+      "learning_rate": 7.508199871764098e-08,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3758,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.5,
+      "step": 20740
+    },
+    {
+      "epoch": 0.7723660456719584,
+      "grad_norm": 8.738892131924189,
+      "learning_rate": 7.48500598908334e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.75,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3743,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.828125,
+      "rewards/rejected": -9.4375,
+      "step": 20750
+    },
+    {
+      "epoch": 0.7727382702722823,
+      "grad_norm": 7.648583661422197,
+      "learning_rate": 7.46184167760138e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3878,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -9.0,
+      "step": 20760
+    },
+    {
+      "epoch": 0.7731104948726061,
+      "grad_norm": 6.741573481712693,
+      "learning_rate": 7.43870697642737e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3857,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.5625,
+      "step": 20770
+    },
+    {
+      "epoch": 0.77348271947293,
+      "grad_norm": 7.762629046083911,
+      "learning_rate": 7.415601924620465e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3842,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.25,
+      "step": 20780
+    },
+    {
+      "epoch": 0.7738549440732538,
+      "grad_norm": 7.156684711630175,
+      "learning_rate": 7.392526561189752e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3365,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.4375,
+      "step": 20790
+    },
+    {
+      "epoch": 0.7742271686735777,
+      "grad_norm": 8.30894151512185,
+      "learning_rate": 7.369480925094204e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4042,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.5625,
+      "step": 20800
+    },
+    {
+      "epoch": 0.7745993932739015,
+      "grad_norm": 7.7278981671843505,
+      "learning_rate": 7.346465055242606e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3786,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.3125,
+      "rewards/rejected": -8.9375,
+      "step": 20810
+    },
+    {
+      "epoch": 0.7749716178742253,
+      "grad_norm": 9.85787052198705,
+      "learning_rate": 7.323478990493495e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.403,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -9.1875,
+      "step": 20820
+    },
+    {
+      "epoch": 0.7753438424745491,
+      "grad_norm": 7.90650095446261,
+      "learning_rate": 7.30052276965506e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3943,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.5,
+      "step": 20830
+    },
+    {
+      "epoch": 0.775716067074873,
+      "grad_norm": 7.648417586912853,
+      "learning_rate": 7.277596431485126e-08,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3817,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.3125,
+      "step": 20840
+    },
+    {
+      "epoch": 0.7760882916751968,
+      "grad_norm": 7.987626897291262,
+      "learning_rate": 7.254700014691059e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3872,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.25,
+      "step": 20850
+    },
+    {
+      "epoch": 0.7764605162755206,
+      "grad_norm": 7.4820770145769595,
+      "learning_rate": 7.231833557929712e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3858,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.5,
+      "step": 20860
+    },
+    {
+      "epoch": 0.7768327408758445,
+      "grad_norm": 7.980528159730055,
+      "learning_rate": 7.208997099807356e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3882,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.0625,
+      "step": 20870
+    },
+    {
+      "epoch": 0.7772049654761684,
+      "grad_norm": 7.201528683900252,
+      "learning_rate": 7.186190678879603e-08,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.75,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3893,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -9.0625,
+      "step": 20880
+    },
+    {
+      "epoch": 0.7775771900764922,
+      "grad_norm": 7.906700469268352,
+      "learning_rate": 7.163414333651357e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3763,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.8671875,
+      "rewards/rejected": -9.25,
+      "step": 20890
+    },
+    {
+      "epoch": 0.777949414676816,
+      "grad_norm": 7.84357517889164,
+      "learning_rate": 7.140668102576756e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4001,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.125,
+      "step": 20900
+    },
+    {
+      "epoch": 0.7783216392771398,
+      "grad_norm": 9.52346940339506,
+      "learning_rate": 7.117952024059084e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3965,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.4375,
+      "step": 20910
+    },
+    {
+      "epoch": 0.7786938638774636,
+      "grad_norm": 8.895084300542662,
+      "learning_rate": 7.095266136450714e-08,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4004,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -9.125,
+      "step": 20920
+    },
+    {
+      "epoch": 0.7790660884777875,
+      "grad_norm": 6.617738949192121,
+      "learning_rate": 7.072610478053051e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.40625,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3796,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.9921875,
+      "rewards/rejected": -9.4375,
+      "step": 20930
+    },
+    {
+      "epoch": 0.7794383130781113,
+      "grad_norm": 7.334368857089297,
+      "learning_rate": 7.049985087116469e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3778,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.25,
+      "step": 20940
+    },
+    {
+      "epoch": 0.7798105376784352,
+      "grad_norm": 7.827499943103778,
+      "learning_rate": 7.02739000184023e-08,
+      "logits/chosen": -3.625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3873,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.8671875,
+      "rewards/rejected": -9.375,
+      "step": 20950
+    },
+    {
+      "epoch": 0.780182762278759,
+      "grad_norm": 7.073236246827541,
+      "learning_rate": 7.004825260372449e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3954,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.828125,
+      "rewards/rejected": -9.1875,
+      "step": 20960
+    },
+    {
+      "epoch": 0.7805549868790829,
+      "grad_norm": 6.974769257269075,
+      "learning_rate": 6.982290900809976e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3879,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -9.25,
+      "step": 20970
+    },
+    {
+      "epoch": 0.7809272114794067,
+      "grad_norm": 9.114353031873724,
+      "learning_rate": 6.959786961198396e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4154,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.3125,
+      "step": 20980
+    },
+    {
+      "epoch": 0.7812994360797305,
+      "grad_norm": 6.824918980310406,
+      "learning_rate": 6.937313479531925e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3711,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.375,
+      "step": 20990
+    },
+    {
+      "epoch": 0.7816716606800543,
+      "grad_norm": 6.084188009581788,
+      "learning_rate": 6.914870493753363e-08,
+      "logits/chosen": -3.609375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -900.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3661,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.0,
+      "step": 21000
+    },
+    {
+      "epoch": 0.7820438852803782,
+      "grad_norm": 6.696970926093566,
+      "learning_rate": 6.892458041754001e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3732,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.375,
+      "step": 21010
+    },
+    {
+      "epoch": 0.782416109880702,
+      "grad_norm": 9.21679885128138,
+      "learning_rate": 6.870076161373603e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4062,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.359375,
+      "rewards/rejected": -9.125,
+      "step": 21020
+    },
+    {
+      "epoch": 0.7827883344810258,
+      "grad_norm": 7.334941055226403,
+      "learning_rate": 6.847724890400306e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3945,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.5625,
+      "step": 21030
+    },
+    {
+      "epoch": 0.7831605590813497,
+      "grad_norm": 7.467529195759886,
+      "learning_rate": 6.825404266570572e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3905,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.1875,
+      "step": 21040
+    },
+    {
+      "epoch": 0.7835327836816736,
+      "grad_norm": 8.223782485413617,
+      "learning_rate": 6.803114327569126e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3895,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -9.0,
+      "step": 21050
+    },
+    {
+      "epoch": 0.7839050082819974,
+      "grad_norm": 9.971316805166776,
+      "learning_rate": 6.780855111028863e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4056,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.5,
+      "step": 21060
+    },
+    {
+      "epoch": 0.7842772328823212,
+      "grad_norm": 8.448092564439245,
+      "learning_rate": 6.75862665453083e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4028,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.25,
+      "step": 21070
+    },
+    {
+      "epoch": 0.784649457482645,
+      "grad_norm": 7.03881650671188,
+      "learning_rate": 6.736428995604143e-08,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3828,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.25,
+      "step": 21080
+    },
+    {
+      "epoch": 0.7850216820829689,
+      "grad_norm": 9.20631352772174,
+      "learning_rate": 6.714262171725904e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3916,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -9.0,
+      "step": 21090
+    },
+    {
+      "epoch": 0.7853939066832927,
+      "grad_norm": 8.688912887060246,
+      "learning_rate": 6.692126220321181e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3914,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.125,
+      "step": 21100
+    },
+    {
+      "epoch": 0.7857661312836165,
+      "grad_norm": 6.990888326932167,
+      "learning_rate": 6.67002117876288e-08,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.4039,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 2.046875,
+      "rewards/rejected": -9.4375,
+      "step": 21110
+    },
+    {
+      "epoch": 0.7861383558839403,
+      "grad_norm": 5.801038338792877,
+      "learning_rate": 6.647947084371755e-08,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3733,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.1875,
+      "step": 21120
+    },
+    {
+      "epoch": 0.7865105804842643,
+      "grad_norm": 8.430930640850288,
+      "learning_rate": 6.6259039744163e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3768,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -9.1875,
+      "step": 21130
+    },
+    {
+      "epoch": 0.7868828050845881,
+      "grad_norm": 7.479601649180905,
+      "learning_rate": 6.6038918861127e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3861,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -9.25,
+      "step": 21140
+    },
+    {
+      "epoch": 0.7872550296849119,
+      "grad_norm": 7.85588374775539,
+      "learning_rate": 6.581910856624748e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.383,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -9.4375,
+      "step": 21150
+    },
+    {
+      "epoch": 0.7876272542852357,
+      "grad_norm": 6.937899965454852,
+      "learning_rate": 6.559960923063829e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3591,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.3125,
+      "step": 21160
+    },
+    {
+      "epoch": 0.7879994788855595,
+      "grad_norm": 6.7874778024240605,
+      "learning_rate": 6.538042122488786e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3872,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.5,
+      "step": 21170
+    },
+    {
+      "epoch": 0.7883717034858834,
+      "grad_norm": 8.795888213229773,
+      "learning_rate": 6.516154491905943e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3741,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.375,
+      "step": 21180
+    },
+    {
+      "epoch": 0.7887439280862072,
+      "grad_norm": 9.346947644444208,
+      "learning_rate": 6.494298068268986e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3868,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.25,
+      "step": 21190
+    },
+    {
+      "epoch": 0.789116152686531,
+      "grad_norm": 7.076504371381756,
+      "learning_rate": 6.472472888478888e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3655,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.4375,
+      "step": 21200
+    },
+    {
+      "epoch": 0.7894883772868548,
+      "grad_norm": 8.031942134792073,
+      "learning_rate": 6.450678989383901e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3801,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.5,
+      "step": 21210
+    },
+    {
+      "epoch": 0.7898606018871788,
+      "grad_norm": 8.435907011508373,
+      "learning_rate": 6.428916407779452e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3958,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.5,
+      "step": 21220
+    },
+    {
+      "epoch": 0.7902328264875026,
+      "grad_norm": 7.515021465655062,
+      "learning_rate": 6.407185180408104e-08,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3736,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.5,
+      "step": 21230
+    },
+    {
+      "epoch": 0.7906050510878264,
+      "grad_norm": 8.002643553370545,
+      "learning_rate": 6.385485343959457e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3805,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -9.1875,
+      "step": 21240
+    },
+    {
+      "epoch": 0.7909772756881502,
+      "grad_norm": 7.733178274349297,
+      "learning_rate": 6.363816935070152e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3746,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.375,
+      "step": 21250
+    },
+    {
+      "epoch": 0.7913495002884741,
+      "grad_norm": 8.497923324758514,
+      "learning_rate": 6.34217999032372e-08,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3657,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.25,
+      "step": 21260
+    },
+    {
+      "epoch": 0.7917217248887979,
+      "grad_norm": 7.709453277150648,
+      "learning_rate": 6.320574546250627e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4076,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.25,
+      "step": 21270
+    },
+    {
+      "epoch": 0.7920939494891217,
+      "grad_norm": 7.399149393571295,
+      "learning_rate": 6.29900063932812e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3901,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.921875,
+      "rewards/rejected": -9.3125,
+      "step": 21280
+    },
+    {
+      "epoch": 0.7924661740894455,
+      "grad_norm": 7.468768690110057,
+      "learning_rate": 6.277458305980198e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3737,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.125,
+      "step": 21290
+    },
+    {
+      "epoch": 0.7928383986897694,
+      "grad_norm": 8.00262264035312,
+      "learning_rate": 6.255947582577576e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3741,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.3125,
+      "step": 21300
+    },
+    {
+      "epoch": 0.7932106232900933,
+      "grad_norm": 7.869906460928921,
+      "learning_rate": 6.234468505437565e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3832,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.1875,
+      "step": 21310
+    },
+    {
+      "epoch": 0.7935828478904171,
+      "grad_norm": 7.855413520749451,
+      "learning_rate": 6.213021110824099e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3659,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.9453125,
+      "rewards/rejected": -9.5625,
+      "step": 21320
+    },
+    {
+      "epoch": 0.7939550724907409,
+      "grad_norm": 8.881522305857926,
+      "learning_rate": 6.191605434947566e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3609,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.625,
+      "step": 21330
+    },
+    {
+      "epoch": 0.7943272970910648,
+      "grad_norm": 8.851887089423007,
+      "learning_rate": 6.170221513964841e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.391,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -9.125,
+      "step": 21340
+    },
+    {
+      "epoch": 0.7946995216913886,
+      "grad_norm": 9.545828009028906,
+      "learning_rate": 6.148869383979172e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3821,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.25,
+      "step": 21350
+    },
+    {
+      "epoch": 0.7950717462917124,
+      "grad_norm": 7.690738372852881,
+      "learning_rate": 6.127549081040117e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -996.0,
+      "logps/rejected": -1168.0,
+      "loss": 0.3903,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -9.75,
+      "step": 21360
+    },
+    {
+      "epoch": 0.7954439708920362,
+      "grad_norm": 7.18595021003095,
+      "learning_rate": 6.106260641143546e-08,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -988.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3764,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.5625,
+      "step": 21370
+    },
+    {
+      "epoch": 0.79581619549236,
+      "grad_norm": 7.525377092734981,
+      "learning_rate": 6.085004100231475e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3646,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.4375,
+      "step": 21380
+    },
+    {
+      "epoch": 0.796188420092684,
+      "grad_norm": 6.281116453144437,
+      "learning_rate": 6.063779494192112e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3787,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.5,
+      "step": 21390
+    },
+    {
+      "epoch": 0.7965606446930078,
+      "grad_norm": 6.8282964878912225,
+      "learning_rate": 6.042586858859703e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4033,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.3125,
+      "step": 21400
+    },
+    {
+      "epoch": 0.7969328692933316,
+      "grad_norm": 8.38140312135992,
+      "learning_rate": 6.021426230014572e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.363,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.1875,
+      "step": 21410
+    },
+    {
+      "epoch": 0.7973050938936554,
+      "grad_norm": 7.068495129395425,
+      "learning_rate": 6.000297643382957e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3596,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.375,
+      "rewards/margins": 1.984375,
+      "rewards/rejected": -9.375,
+      "step": 21420
+    },
+    {
+      "epoch": 0.7976773184939793,
+      "grad_norm": 7.458588294741837,
+      "learning_rate": 5.979201134637015e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3895,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.1875,
+      "step": 21430
+    },
+    {
+      "epoch": 0.7980495430943031,
+      "grad_norm": 8.726943232174335,
+      "learning_rate": 5.9581367393947616e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3962,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 2.046875,
+      "rewards/rejected": -9.5,
+      "step": 21440
+    },
+    {
+      "epoch": 0.7984217676946269,
+      "grad_norm": 8.635525755596907,
+      "learning_rate": 5.937104493219952e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3696,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.25,
+      "step": 21450
+    },
+    {
+      "epoch": 0.7987939922949507,
+      "grad_norm": 12.25163092394332,
+      "learning_rate": 5.916104431622121e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.4011,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.5,
+      "step": 21460
+    },
+    {
+      "epoch": 0.7991662168952746,
+      "grad_norm": 7.916203604545604,
+      "learning_rate": 5.895136590056407e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3888,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -9.4375,
+      "step": 21470
+    },
+    {
+      "epoch": 0.7995384414955985,
+      "grad_norm": 7.158845599853865,
+      "learning_rate": 5.8742010039235957e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3815,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.3125,
+      "step": 21480
+    },
+    {
+      "epoch": 0.7999106660959223,
+      "grad_norm": 7.2715942586522635,
+      "learning_rate": 5.853297708569979e-08,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3742,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 2.09375,
+      "rewards/rejected": -9.5625,
+      "step": 21490
+    },
+    {
+      "epoch": 0.8002828906962461,
+      "grad_norm": 7.670673431290395,
+      "learning_rate": 5.832426739287355e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3788,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.5625,
+      "step": 21500
+    },
+    {
+      "epoch": 0.80065511529657,
+      "grad_norm": 7.383268127390173,
+      "learning_rate": 5.811588131312936e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3696,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.5,
+      "step": 21510
+    },
+    {
+      "epoch": 0.8010273398968938,
+      "grad_norm": 7.863542641628323,
+      "learning_rate": 5.790781919829299e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.39,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -9.4375,
+      "step": 21520
+    },
+    {
+      "epoch": 0.8013995644972176,
+      "grad_norm": 8.307749051086759,
+      "learning_rate": 5.770008139964333e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3796,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.5,
+      "step": 21530
+    },
+    {
+      "epoch": 0.8017717890975414,
+      "grad_norm": 6.93154452572152,
+      "learning_rate": 5.749266826791146e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4054,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.375,
+      "step": 21540
+    },
+    {
+      "epoch": 0.8021440136978653,
+      "grad_norm": 7.184734812830204,
+      "learning_rate": 5.728558015328061e-08,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3966,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.5,
+      "step": 21550
+    },
+    {
+      "epoch": 0.8025162382981891,
+      "grad_norm": 6.938804890196552,
+      "learning_rate": 5.707881740538509e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.388,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -9.3125,
+      "step": 21560
+    },
+    {
+      "epoch": 0.802888462898513,
+      "grad_norm": 7.771299813099175,
+      "learning_rate": 5.687238037331002e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3861,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.125,
+      "step": 21570
+    },
+    {
+      "epoch": 0.8032606874988368,
+      "grad_norm": 7.545997563494077,
+      "learning_rate": 5.666626940559038e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3688,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -9.125,
+      "step": 21580
+    },
+    {
+      "epoch": 0.8036329120991607,
+      "grad_norm": 9.395540010482339,
+      "learning_rate": 5.646048485021085e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.4006,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.375,
+      "step": 21590
+    },
+    {
+      "epoch": 0.8040051366994845,
+      "grad_norm": 7.324040400203472,
+      "learning_rate": 5.625502705460497e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3982,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.375,
+      "rewards/rejected": -8.8125,
+      "step": 21600
+    },
+    {
+      "epoch": 0.8043773612998083,
+      "grad_norm": 8.25138863960593,
+      "learning_rate": 5.6049896365654577e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3804,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.1875,
+      "step": 21610
+    },
+    {
+      "epoch": 0.8047495859001321,
+      "grad_norm": 8.445810399403587,
+      "learning_rate": 5.584509312968927e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3871,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.0625,
+      "step": 21620
+    },
+    {
+      "epoch": 0.805121810500456,
+      "grad_norm": 8.53127792517278,
+      "learning_rate": 5.5640617692485654e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3675,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.375,
+      "step": 21630
+    },
+    {
+      "epoch": 0.8054940351007798,
+      "grad_norm": 7.842176654888013,
+      "learning_rate": 5.5436470399267103e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3865,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.3125,
+      "step": 21640
+    },
+    {
+      "epoch": 0.8058662597011036,
+      "grad_norm": 8.47351936390772,
+      "learning_rate": 5.523265159470289e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3908,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.3125,
+      "step": 21650
+    },
+    {
+      "epoch": 0.8062384843014275,
+      "grad_norm": 7.930886755581482,
+      "learning_rate": 5.5029161622907716e-08,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3856,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.3125,
+      "step": 21660
+    },
+    {
+      "epoch": 0.8066107089017513,
+      "grad_norm": 7.080664371678285,
+      "learning_rate": 5.482600082744096e-08,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3616,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.8671875,
+      "rewards/rejected": -9.375,
+      "step": 21670
+    },
+    {
+      "epoch": 0.8069829335020752,
+      "grad_norm": 8.118231736235908,
+      "learning_rate": 5.4623169551306445e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3765,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.375,
+      "step": 21680
+    },
+    {
+      "epoch": 0.807355158102399,
+      "grad_norm": 8.207252460575733,
+      "learning_rate": 5.442066813695151e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1152.0,
+      "loss": 0.3849,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 2.0,
+      "rewards/rejected": -9.75,
+      "step": 21690
+    },
+    {
+      "epoch": 0.8077273827027228,
+      "grad_norm": 9.194929073054487,
+      "learning_rate": 5.421849692626665e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3835,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": -9.75,
+      "step": 21700
+    },
+    {
+      "epoch": 0.8080996073030466,
+      "grad_norm": 7.228471063312755,
+      "learning_rate": 5.401665626058491e-08,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3866,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -9.5625,
+      "step": 21710
+    },
+    {
+      "epoch": 0.8084718319033705,
+      "grad_norm": 7.491877341840319,
+      "learning_rate": 5.3815146480681056e-08,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3881,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.375,
+      "step": 21720
+    },
+    {
+      "epoch": 0.8088440565036943,
+      "grad_norm": 8.14711430504842,
+      "learning_rate": 5.361396792677142e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.372,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.1875,
+      "step": 21730
+    },
+    {
+      "epoch": 0.8092162811040181,
+      "grad_norm": 7.139363624794363,
+      "learning_rate": 5.341312093851305e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3964,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.34375,
+      "rewards/margins": 1.921875,
+      "rewards/rejected": -9.25,
+      "step": 21740
+    },
+    {
+      "epoch": 0.809588505704342,
+      "grad_norm": 9.368027970963224,
+      "learning_rate": 5.3212605855003254e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3978,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.3125,
+      "step": 21750
+    },
+    {
+      "epoch": 0.8099607303046659,
+      "grad_norm": 8.139093973625473,
+      "learning_rate": 5.301242301477879e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3818,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.25,
+      "step": 21760
+    },
+    {
+      "epoch": 0.8103329549049897,
+      "grad_norm": 7.676051667445268,
+      "learning_rate": 5.281257275581563e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3975,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": -9.5625,
+      "step": 21770
+    },
+    {
+      "epoch": 0.8107051795053135,
+      "grad_norm": 7.635836591331091,
+      "learning_rate": 5.261305541552824e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3776,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.5625,
+      "step": 21780
+    },
+    {
+      "epoch": 0.8110774041056373,
+      "grad_norm": 8.742397448535758,
+      "learning_rate": 5.241387133076894e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3837,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -9.3125,
+      "step": 21790
+    },
+    {
+      "epoch": 0.8114496287059612,
+      "grad_norm": 8.28732230721892,
+      "learning_rate": 5.2215020837827527e-08,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.364,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 2.03125,
+      "rewards/rejected": -9.5625,
+      "step": 21800
+    },
+    {
+      "epoch": 0.811821853306285,
+      "grad_norm": 7.120447569574624,
+      "learning_rate": 5.201650427243034e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3876,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.375,
+      "step": 21810
+    },
+    {
+      "epoch": 0.8121940779066088,
+      "grad_norm": 8.415991887782464,
+      "learning_rate": 5.181832196974015e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3913,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.9453125,
+      "rewards/rejected": -9.5625,
+      "step": 21820
+    },
+    {
+      "epoch": 0.8125663025069327,
+      "grad_norm": 7.751388908413297,
+      "learning_rate": 5.162047426435537e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3731,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.5,
+      "step": 21830
+    },
+    {
+      "epoch": 0.8129385271072566,
+      "grad_norm": 7.821635501625169,
+      "learning_rate": 5.142296149030945e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -992.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3895,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.5,
+      "step": 21840
+    },
+    {
+      "epoch": 0.8133107517075804,
+      "grad_norm": 8.109827520193788,
+      "learning_rate": 5.1225783981070245e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3555,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.90625,
+      "rewards/rejected": -9.625,
+      "step": 21850
+    },
+    {
+      "epoch": 0.8136829763079042,
+      "grad_norm": 8.255812022456158,
+      "learning_rate": 5.102894206953975e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3639,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.5625,
+      "step": 21860
+    },
+    {
+      "epoch": 0.814055200908228,
+      "grad_norm": 8.66621352703113,
+      "learning_rate": 5.083243608805332e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3856,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -9.1875,
+      "step": 21870
+    },
+    {
+      "epoch": 0.8144274255085518,
+      "grad_norm": 8.259224310304848,
+      "learning_rate": 5.0636266368379085e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3992,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.5625,
+      "step": 21880
+    },
+    {
+      "epoch": 0.8147996501088757,
+      "grad_norm": 7.660229391811261,
+      "learning_rate": 5.0440433241717596e-08,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3885,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.375,
+      "step": 21890
+    },
+    {
+      "epoch": 0.8151718747091995,
+      "grad_norm": 9.281232474063378,
+      "learning_rate": 5.024493703870084e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3833,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.96875,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.625,
+      "step": 21900
+    },
+    {
+      "epoch": 0.8155440993095233,
+      "grad_norm": 8.459260049264113,
+      "learning_rate": 5.004977808939223e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -992.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.407,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.5,
+      "step": 21910
+    },
+    {
+      "epoch": 0.8159163239098473,
+      "grad_norm": 7.416334189628268,
+      "learning_rate": 4.9854956723285687e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3717,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.625,
+      "step": 21920
+    },
+    {
+      "epoch": 0.8162885485101711,
+      "grad_norm": 7.342191426443672,
+      "learning_rate": 4.966047326930517e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3771,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": -9.6875,
+      "step": 21930
+    },
+    {
+      "epoch": 0.8166607731104949,
+      "grad_norm": 7.33109556956649,
+      "learning_rate": 4.94663280558042e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3851,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.96875,
+      "rewards/margins": 1.90625,
+      "rewards/rejected": -9.875,
+      "step": 21940
+    },
+    {
+      "epoch": 0.8170329977108187,
+      "grad_norm": 7.086034875288952,
+      "learning_rate": 4.9272521410565065e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -992.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3788,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.96875,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -9.5,
+      "step": 21950
+    },
+    {
+      "epoch": 0.8174052223111425,
+      "grad_norm": 8.423253023914196,
+      "learning_rate": 4.907905366079859e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.404,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.5,
+      "step": 21960
+    },
+    {
+      "epoch": 0.8177774469114664,
+      "grad_norm": 8.760946485376412,
+      "learning_rate": 4.888592513314338e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3751,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -9.4375,
+      "step": 21970
+    },
+    {
+      "epoch": 0.8181496715117902,
+      "grad_norm": 7.25364252406904,
+      "learning_rate": 4.869313615366541e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.4106,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -9.4375,
+      "step": 21980
+    },
+    {
+      "epoch": 0.818521896112114,
+      "grad_norm": 7.900921295521825,
+      "learning_rate": 4.8500687047857145e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.4049,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -9.3125,
+      "step": 21990
+    },
+    {
+      "epoch": 0.8188941207124378,
+      "grad_norm": 8.141610571198358,
+      "learning_rate": 4.830857814063744e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.75,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3952,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": -9.1875,
+      "step": 22000
+    },
+    {
+      "epoch": 0.8192663453127618,
+      "grad_norm": 8.414617263268871,
+      "learning_rate": 4.8116809756350767e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -992.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3981,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.5625,
+      "step": 22010
+    },
+    {
+      "epoch": 0.8196385699130856,
+      "grad_norm": 7.1666391629131825,
+      "learning_rate": 4.7925382218766604e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3867,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.1875,
+      "step": 22020
+    },
+    {
+      "epoch": 0.8200107945134094,
+      "grad_norm": 6.9540436968370285,
+      "learning_rate": 4.7734295851079046e-08,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.393,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.4921875,
+      "rewards/rejected": -9.25,
+      "step": 22030
+    },
+    {
+      "epoch": 0.8203830191137332,
+      "grad_norm": 7.987981480848643,
+      "learning_rate": 4.7543550975906013e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1152.0,
+      "loss": 0.374,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -8.0,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.8125,
+      "step": 22040
+    },
+    {
+      "epoch": 0.8207552437140571,
+      "grad_norm": 7.5582257218927165,
+      "learning_rate": 4.735314791528908e-08,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3847,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.25,
+      "step": 22050
+    },
+    {
+      "epoch": 0.8211274683143809,
+      "grad_norm": 7.302574112386361,
+      "learning_rate": 4.716308699069257e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3998,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.4375,
+      "step": 22060
+    },
+    {
+      "epoch": 0.8214996929147047,
+      "grad_norm": 7.707039088124113,
+      "learning_rate": 4.6973368523003376e-08,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.328125,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3735,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.3125,
+      "step": 22070
+    },
+    {
+      "epoch": 0.8218719175150285,
+      "grad_norm": 7.1691476773656015,
+      "learning_rate": 4.6783992832529845e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3786,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.9453125,
+      "rewards/rejected": -9.625,
+      "step": 22080
+    },
+    {
+      "epoch": 0.8222441421153523,
+      "grad_norm": 7.490246978484917,
+      "learning_rate": 4.6594960239001975e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3982,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.5,
+      "step": 22090
+    },
+    {
+      "epoch": 0.8226163667156763,
+      "grad_norm": 8.66371419962431,
+      "learning_rate": 4.6406271061570166e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3908,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.4375,
+      "step": 22100
+    },
+    {
+      "epoch": 0.8229885913160001,
+      "grad_norm": 8.273479884873563,
+      "learning_rate": 4.6217925618805356e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4069,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -9.0,
+      "step": 22110
+    },
+    {
+      "epoch": 0.8233608159163239,
+      "grad_norm": 8.36343937262187,
+      "learning_rate": 4.6029924228698e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3827,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.4375,
+      "step": 22120
+    },
+    {
+      "epoch": 0.8237330405166478,
+      "grad_norm": 8.344846397083662,
+      "learning_rate": 4.5842267208657524e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3873,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -9.1875,
+      "step": 22130
+    },
+    {
+      "epoch": 0.8241052651169716,
+      "grad_norm": 7.802170206864785,
+      "learning_rate": 4.565495487551213e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1056.0,
+      "loss": 0.3876,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.0625,
+      "step": 22140
+    },
+    {
+      "epoch": 0.8244774897172954,
+      "grad_norm": 7.161362213631662,
+      "learning_rate": 4.546798754550801e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3781,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.9765625,
+      "rewards/rejected": -9.5,
+      "step": 22150
+    },
+    {
+      "epoch": 0.8248497143176192,
+      "grad_norm": 6.332289107080237,
+      "learning_rate": 4.5281365534308946e-08,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3614,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.6875,
+      "step": 22160
+    },
+    {
+      "epoch": 0.825221938917943,
+      "grad_norm": 7.8646491348246945,
+      "learning_rate": 4.5095089156995505e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3866,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -9.5625,
+      "step": 22170
+    },
+    {
+      "epoch": 0.825594163518267,
+      "grad_norm": 7.724229859464605,
+      "learning_rate": 4.490915872806497e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3834,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.3125,
+      "step": 22180
+    },
+    {
+      "epoch": 0.8259663881185908,
+      "grad_norm": 6.946791617051151,
+      "learning_rate": 4.4723574561430246e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.890625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3772,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.375,
+      "step": 22190
+    },
+    {
+      "epoch": 0.8263386127189146,
+      "grad_norm": 7.888994067622612,
+      "learning_rate": 4.4538336970420004e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3682,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 2.03125,
+      "rewards/rejected": -9.6875,
+      "step": 22200
+    },
+    {
+      "epoch": 0.8267108373192384,
+      "grad_norm": 8.473637547091336,
+      "learning_rate": 4.435344626777754e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3886,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.3125,
+      "step": 22210
+    },
+    {
+      "epoch": 0.8270830619195623,
+      "grad_norm": 7.54840442443646,
+      "learning_rate": 4.41689027656604e-08,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3907,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.3125,
+      "step": 22220
+    },
+    {
+      "epoch": 0.8274552865198861,
+      "grad_norm": 6.124764322613412,
+      "learning_rate": 4.398470677564023e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3514,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 2.046875,
+      "rewards/rejected": -9.75,
+      "step": 22230
+    },
+    {
+      "epoch": 0.8278275111202099,
+      "grad_norm": 8.477275432889517,
+      "learning_rate": 4.380085860870156e-08,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3814,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.3125,
+      "rewards/margins": 2.015625,
+      "rewards/rejected": -9.375,
+      "step": 22240
+    },
+    {
+      "epoch": 0.8281997357205337,
+      "grad_norm": 7.899745014812589,
+      "learning_rate": 4.361735857524221e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3874,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.4375,
+      "step": 22250
+    },
+    {
+      "epoch": 0.8285719603208576,
+      "grad_norm": 7.259018744441789,
+      "learning_rate": 4.3434206985071667e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3679,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.3125,
+      "step": 22260
+    },
+    {
+      "epoch": 0.8289441849211815,
+      "grad_norm": 8.126572976754941,
+      "learning_rate": 4.325140414741152e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3985,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.3125,
+      "step": 22270
+    },
+    {
+      "epoch": 0.8293164095215053,
+      "grad_norm": 8.987699426090657,
+      "learning_rate": 4.3068950370894476e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3929,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -9.375,
+      "step": 22280
+    },
+    {
+      "epoch": 0.8296886341218291,
+      "grad_norm": 8.62917494609193,
+      "learning_rate": 4.288684596356365e-08,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.34375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3814,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -9.4375,
+      "step": 22290
+    },
+    {
+      "epoch": 0.830060858722153,
+      "grad_norm": 6.748824102690535,
+      "learning_rate": 4.270509123287277e-08,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -988.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.376,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.5,
+      "step": 22300
+    },
+    {
+      "epoch": 0.8304330833224768,
+      "grad_norm": 8.826158549637205,
+      "learning_rate": 4.2523686485684745e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3918,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.625,
+      "step": 22310
+    },
+    {
+      "epoch": 0.8308053079228006,
+      "grad_norm": 7.46883261922193,
+      "learning_rate": 4.23426320282719e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3872,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.5,
+      "step": 22320
+    },
+    {
+      "epoch": 0.8311775325231244,
+      "grad_norm": 8.698144534568675,
+      "learning_rate": 4.2161928166314834e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3986,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.4375,
+      "step": 22330
+    },
+    {
+      "epoch": 0.8315497571234483,
+      "grad_norm": 8.316780220489903,
+      "learning_rate": 4.1981575204902695e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.38,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.3125,
+      "step": 22340
+    },
+    {
+      "epoch": 0.8319219817237721,
+      "grad_norm": 8.818721119513864,
+      "learning_rate": 4.1801573448531744e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.625,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3758,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.5,
+      "step": 22350
+    },
+    {
+      "epoch": 0.832294206324096,
+      "grad_norm": 8.014180447078797,
+      "learning_rate": 4.1621923201105466e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3771,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.375,
+      "step": 22360
+    },
+    {
+      "epoch": 0.8326664309244198,
+      "grad_norm": 7.395196918998407,
+      "learning_rate": 4.144262476593402e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4057,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.5,
+      "step": 22370
+    },
+    {
+      "epoch": 0.8330386555247437,
+      "grad_norm": 7.785244864895281,
+      "learning_rate": 4.126367844573322e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3817,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.25,
+      "step": 22380
+    },
+    {
+      "epoch": 0.8334108801250675,
+      "grad_norm": 7.139314042393561,
+      "learning_rate": 4.108508454262485e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3758,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.96875,
+      "rewards/rejected": -9.375,
+      "step": 22390
+    },
+    {
+      "epoch": 0.8337831047253913,
+      "grad_norm": 7.945439356840114,
+      "learning_rate": 4.090684335813532e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3838,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -9.1875,
+      "step": 22400
+    },
+    {
+      "epoch": 0.8341553293257151,
+      "grad_norm": 9.233922082688919,
+      "learning_rate": 4.0728955193195803e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3932,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.4375,
+      "step": 22410
+    },
+    {
+      "epoch": 0.834527553926039,
+      "grad_norm": 8.087577996388815,
+      "learning_rate": 4.055142034814119e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3953,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.3125,
+      "step": 22420
+    },
+    {
+      "epoch": 0.8348997785263628,
+      "grad_norm": 7.79257402931684,
+      "learning_rate": 4.0374239122710015e-08,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3749,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 2.09375,
+      "rewards/rejected": -9.5,
+      "step": 22430
+    },
+    {
+      "epoch": 0.8352720031266866,
+      "grad_norm": 7.08253603904175,
+      "learning_rate": 4.019741181604383e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3915,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.5625,
+      "step": 22440
+    },
+    {
+      "epoch": 0.8356442277270105,
+      "grad_norm": 8.531559760161736,
+      "learning_rate": 4.002093872668655e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.4017,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.4375,
+      "step": 22450
+    },
+    {
+      "epoch": 0.8360164523273343,
+      "grad_norm": 9.848186504569375,
+      "learning_rate": 3.984482015258411e-08,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3995,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.828125,
+      "rewards/rejected": -9.5,
+      "step": 22460
+    },
+    {
+      "epoch": 0.8363886769276582,
+      "grad_norm": 7.420027708912492,
+      "learning_rate": 3.966905639108384e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3987,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -9.5,
+      "step": 22470
+    },
+    {
+      "epoch": 0.836760901527982,
+      "grad_norm": 10.037671937655398,
+      "learning_rate": 3.9493647738934046e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3879,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -9.25,
+      "step": 22480
+    },
+    {
+      "epoch": 0.8371331261283058,
+      "grad_norm": 7.86559931408963,
+      "learning_rate": 3.931859449228353e-08,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3818,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -9.6875,
+      "step": 22490
+    },
+    {
+      "epoch": 0.8375053507286296,
+      "grad_norm": 7.108645220874712,
+      "learning_rate": 3.91438969466811e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3823,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -9.25,
+      "step": 22500
+    },
+    {
+      "epoch": 0.8378775753289535,
+      "grad_norm": 6.535619624834789,
+      "learning_rate": 3.896955539707483e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3776,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -9.3125,
+      "step": 22510
+    },
+    {
+      "epoch": 0.8382497999292773,
+      "grad_norm": 7.32161716519995,
+      "learning_rate": 3.879557013781193e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3776,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.828125,
+      "rewards/rejected": -9.625,
+      "step": 22520
+    },
+    {
+      "epoch": 0.8386220245296011,
+      "grad_norm": 8.551366395470554,
+      "learning_rate": 3.862194146263803e-08,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3703,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -9.3125,
+      "step": 22530
+    },
+    {
+      "epoch": 0.838994249129925,
+      "grad_norm": 7.3589981510904074,
+      "learning_rate": 3.844866966469668e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3881,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.5,
+      "step": 22540
+    },
+    {
+      "epoch": 0.8393664737302489,
+      "grad_norm": 7.660731823126742,
+      "learning_rate": 3.827575503652902e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3905,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.625,
+      "step": 22550
+    },
+    {
+      "epoch": 0.8397386983305727,
+      "grad_norm": 7.755866966312417,
+      "learning_rate": 3.810319787007293e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3779,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.5625,
+      "step": 22560
+    },
+    {
+      "epoch": 0.8401109229308965,
+      "grad_norm": 7.452165822311574,
+      "learning_rate": 3.7930998456663033e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3643,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.625,
+      "step": 22570
+    },
+    {
+      "epoch": 0.8404831475312203,
+      "grad_norm": 8.295257572721892,
+      "learning_rate": 3.7759157087029784e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3853,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -8.0,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.6875,
+      "step": 22580
+    },
+    {
+      "epoch": 0.8408553721315442,
+      "grad_norm": 9.429769279145063,
+      "learning_rate": 3.7587674051299254e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.4071,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 2.015625,
+      "rewards/rejected": -9.5625,
+      "step": 22590
+    },
+    {
+      "epoch": 0.841227596731868,
+      "grad_norm": 8.194327466583253,
+      "learning_rate": 3.7416549638992406e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3989,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.5625,
+      "step": 22600
+    },
+    {
+      "epoch": 0.8415998213321918,
+      "grad_norm": 7.984678075267944,
+      "learning_rate": 3.724578413902477e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.4059,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.4375,
+      "step": 22610
+    },
+    {
+      "epoch": 0.8419720459325157,
+      "grad_norm": 7.891787583750504,
+      "learning_rate": 3.7075377839705964e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3731,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.125,
+      "step": 22620
+    },
+    {
+      "epoch": 0.8423442705328396,
+      "grad_norm": 8.196379644294845,
+      "learning_rate": 3.690533102873911e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3803,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.3125,
+      "step": 22630
+    },
+    {
+      "epoch": 0.8427164951331634,
+      "grad_norm": 6.899028729623778,
+      "learning_rate": 3.673564399322046e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.383,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 2.0,
+      "rewards/rejected": -9.625,
+      "step": 22640
+    },
+    {
+      "epoch": 0.8430887197334872,
+      "grad_norm": 8.064632974993069,
+      "learning_rate": 3.6566317019638634e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3959,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.4375,
+      "step": 22650
+    },
+    {
+      "epoch": 0.843460944333811,
+      "grad_norm": 6.407347229521458,
+      "learning_rate": 3.639735039387462e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3731,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.25,
+      "step": 22660
+    },
+    {
+      "epoch": 0.8438331689341348,
+      "grad_norm": 7.731913708046398,
+      "learning_rate": 3.6228744401200826e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3941,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.9765625,
+      "rewards/rejected": -9.625,
+      "step": 22670
+    },
+    {
+      "epoch": 0.8442053935344587,
+      "grad_norm": 8.095392367291199,
+      "learning_rate": 3.606049932628094e-08,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.369,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.3125,
+      "step": 22680
+    },
+    {
+      "epoch": 0.8445776181347825,
+      "grad_norm": 7.278351963105301,
+      "learning_rate": 3.58926154531691e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3718,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.5625,
+      "step": 22690
+    },
+    {
+      "epoch": 0.8449498427351063,
+      "grad_norm": 8.88402489012204,
+      "learning_rate": 3.572509306530977e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3872,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.3125,
+      "step": 22700
+    },
+    {
+      "epoch": 0.8453220673354302,
+      "grad_norm": 9.496584000361478,
+      "learning_rate": 3.555793244553712e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3829,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -9.6875,
+      "step": 22710
+    },
+    {
+      "epoch": 0.8456942919357541,
+      "grad_norm": 7.338831387421984,
+      "learning_rate": 3.53911338760744e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3873,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.25,
+      "step": 22720
+    },
+    {
+      "epoch": 0.8460665165360779,
+      "grad_norm": 7.1818151108290245,
+      "learning_rate": 3.522469763853381e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4047,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.375,
+      "step": 22730
+    },
+    {
+      "epoch": 0.8464387411364017,
+      "grad_norm": 8.76039515765638,
+      "learning_rate": 3.5058624013915516e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3719,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -9.25,
+      "step": 22740
+    },
+    {
+      "epoch": 0.8468109657367255,
+      "grad_norm": 7.377557131269985,
+      "learning_rate": 3.4892913282607714e-08,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.875,
+      "logps/chosen": -1004.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.398,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -8.0625,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": -9.4375,
+      "step": 22750
+    },
+    {
+      "epoch": 0.8471831903370494,
+      "grad_norm": 9.832517734822334,
+      "learning_rate": 3.472756572438579e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3949,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.25,
+      "step": 22760
+    },
+    {
+      "epoch": 0.8475554149373732,
+      "grad_norm": 7.364990500637907,
+      "learning_rate": 3.4562581618412136e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3882,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -9.0,
+      "step": 22770
+    },
+    {
+      "epoch": 0.847927639537697,
+      "grad_norm": 7.682500299309095,
+      "learning_rate": 3.4397961243235244e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3781,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.4375,
+      "step": 22780
+    },
+    {
+      "epoch": 0.8482998641380208,
+      "grad_norm": 8.361603081995874,
+      "learning_rate": 3.423370487678972e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3854,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.375,
+      "step": 22790
+    },
+    {
+      "epoch": 0.8486720887383448,
+      "grad_norm": 7.162098568776144,
+      "learning_rate": 3.406981279639554e-08,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.417,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -9.3125,
+      "step": 22800
+    },
+    {
+      "epoch": 0.8490443133386686,
+      "grad_norm": 8.023038139397185,
+      "learning_rate": 3.390628527875764e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3698,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.3125,
+      "step": 22810
+    },
+    {
+      "epoch": 0.8494165379389924,
+      "grad_norm": 7.503261719509011,
+      "learning_rate": 3.374312259996548e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3778,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -9.25,
+      "step": 22820
+    },
+    {
+      "epoch": 0.8497887625393162,
+      "grad_norm": 7.545575033391787,
+      "learning_rate": 3.358032503549246e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3818,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.3125,
+      "step": 22830
+    },
+    {
+      "epoch": 0.8501609871396401,
+      "grad_norm": 6.604086146481576,
+      "learning_rate": 3.3417892860195615e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3579,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -9.3125,
+      "step": 22840
+    },
+    {
+      "epoch": 0.8505332117399639,
+      "grad_norm": 8.373297998723224,
+      "learning_rate": 3.325582634831508e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3732,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.5,
+      "step": 22850
+    },
+    {
+      "epoch": 0.8509054363402877,
+      "grad_norm": 7.423083309738103,
+      "learning_rate": 3.309412577347365e-08,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3512,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.3125,
+      "step": 22860
+    },
+    {
+      "epoch": 0.8512776609406115,
+      "grad_norm": 9.428748799259077,
+      "learning_rate": 3.2932791408676275e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3719,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.3125,
+      "step": 22870
+    },
+    {
+      "epoch": 0.8516498855409353,
+      "grad_norm": 8.883740819567608,
+      "learning_rate": 3.277182352630953e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3827,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -9.625,
+      "step": 22880
+    },
+    {
+      "epoch": 0.8520221101412593,
+      "grad_norm": 7.042690284835938,
+      "learning_rate": 3.261122239814135e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3552,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.3125,
+      "step": 22890
+    },
+    {
+      "epoch": 0.8523943347415831,
+      "grad_norm": 7.134969799451995,
+      "learning_rate": 3.2450988295320446e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3666,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -9.5,
+      "step": 22900
+    },
+    {
+      "epoch": 0.8527665593419069,
+      "grad_norm": 6.894453134653101,
+      "learning_rate": 3.229112148837593e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.4021,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.4375,
+      "step": 22910
+    },
+    {
+      "epoch": 0.8531387839422307,
+      "grad_norm": 8.01465164039426,
+      "learning_rate": 3.2131622247216576e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3706,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.375,
+      "step": 22920
+    },
+    {
+      "epoch": 0.8535110085425546,
+      "grad_norm": 7.235677663614719,
+      "learning_rate": 3.197249084113082e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3796,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.25,
+      "step": 22930
+    },
+    {
+      "epoch": 0.8538832331428784,
+      "grad_norm": 7.559830163966948,
+      "learning_rate": 3.1813727538785943e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3721,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.375,
+      "step": 22940
+    },
+    {
+      "epoch": 0.8542554577432022,
+      "grad_norm": 9.088203859747761,
+      "learning_rate": 3.165533260822784e-08,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3794,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": -9.4375,
+      "step": 22950
+    },
+    {
+      "epoch": 0.854627682343526,
+      "grad_norm": 7.202206767305839,
+      "learning_rate": 3.1497306316880385e-08,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3683,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.625,
+      "step": 22960
+    },
+    {
+      "epoch": 0.85499990694385,
+      "grad_norm": 6.836963549660777,
+      "learning_rate": 3.133964893154503e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.5,
+      "logps/chosen": -996.0,
+      "logps/rejected": -1160.0,
+      "loss": 0.3733,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.6875,
+      "step": 22970
+    },
+    {
+      "epoch": 0.8553721315441738,
+      "grad_norm": 8.031521318603092,
+      "learning_rate": 3.118236071840052e-08,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3944,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.4375,
+      "step": 22980
+    },
+    {
+      "epoch": 0.8557443561444976,
+      "grad_norm": 9.151113074217673,
+      "learning_rate": 3.102544194300219e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3662,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.90625,
+      "rewards/rejected": -9.6875,
+      "step": 22990
+    },
+    {
+      "epoch": 0.8561165807448214,
+      "grad_norm": 7.0083281593171165,
+      "learning_rate": 3.0868892870281774e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1184.0,
+      "loss": 0.3734,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 2.171875,
+      "rewards/rejected": -10.0625,
+      "step": 23000
+    },
+    {
+      "epoch": 0.8564888053451453,
+      "grad_norm": 9.008110146832417,
+      "learning_rate": 3.071271376454665e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3728,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.984375,
+      "rewards/rejected": -9.75,
+      "step": 23010
+    },
+    {
+      "epoch": 0.8568610299454691,
+      "grad_norm": 8.175472584309263,
+      "learning_rate": 3.0556904889479736e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3805,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.9609375,
+      "rewards/rejected": -9.5625,
+      "step": 23020
+    },
+    {
+      "epoch": 0.8572332545457929,
+      "grad_norm": 7.025577755484629,
+      "learning_rate": 3.040146650813866e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.375,
+      "step": 23030
+    },
+    {
+      "epoch": 0.8576054791461167,
+      "grad_norm": 8.408211292605815,
+      "learning_rate": 3.02463988829558e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.4153,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.8671875,
+      "rewards/rejected": -9.4375,
+      "step": 23040
+    },
+    {
+      "epoch": 0.8579777037464406,
+      "grad_norm": 6.594067946771017,
+      "learning_rate": 3.009170227573749e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.361,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.5,
+      "step": 23050
+    },
+    {
+      "epoch": 0.8583499283467645,
+      "grad_norm": 8.276374859216219,
+      "learning_rate": 2.993737694766349e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3768,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.25,
+      "step": 23060
+    },
+    {
+      "epoch": 0.8587221529470883,
+      "grad_norm": 6.989512747948214,
+      "learning_rate": 2.978342315928692e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3761,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -9.25,
+      "step": 23070
+    },
+    {
+      "epoch": 0.8590943775474121,
+      "grad_norm": 7.852507668797501,
+      "learning_rate": 2.9629841170533408e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3819,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.921875,
+      "rewards/rejected": -9.625,
+      "step": 23080
+    },
+    {
+      "epoch": 0.859466602147736,
+      "grad_norm": 8.286351261584846,
+      "learning_rate": 2.9476631240701212e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.374,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.1875,
+      "step": 23090
+    },
+    {
+      "epoch": 0.8598388267480598,
+      "grad_norm": 7.816207389365674,
+      "learning_rate": 2.9323793628460018e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3747,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.4375,
+      "step": 23100
+    },
+    {
+      "epoch": 0.8602110513483836,
+      "grad_norm": 7.2818153013743245,
+      "learning_rate": 2.917132859185123e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.401,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.1875,
+      "step": 23110
+    },
+    {
+      "epoch": 0.8605832759487074,
+      "grad_norm": 7.996087241585381,
+      "learning_rate": 2.9019236388286907e-08,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3815,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.375,
+      "step": 23120
+    },
+    {
+      "epoch": 0.8609555005490312,
+      "grad_norm": 8.954523667661592,
+      "learning_rate": 2.886751727455003e-08,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4096,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.5,
+      "step": 23130
+    },
+    {
+      "epoch": 0.8613277251493551,
+      "grad_norm": 7.009420786021144,
+      "learning_rate": 2.8716171506793423e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3874,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.3125,
+      "step": 23140
+    },
+    {
+      "epoch": 0.861699949749679,
+      "grad_norm": 8.02362416129298,
+      "learning_rate": 2.8565199340539553e-08,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3771,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.375,
+      "step": 23150
+    },
+    {
+      "epoch": 0.8620721743500028,
+      "grad_norm": 7.37606317206679,
+      "learning_rate": 2.8414601030680273e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3778,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.5,
+      "step": 23160
+    },
+    {
+      "epoch": 0.8624443989503267,
+      "grad_norm": 8.455780155171958,
+      "learning_rate": 2.8264376831476034e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3769,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.4375,
+      "step": 23170
+    },
+    {
+      "epoch": 0.8628166235506505,
+      "grad_norm": 7.1705599388362735,
+      "learning_rate": 2.8114526996556002e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3769,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.4375,
+      "step": 23180
+    },
+    {
+      "epoch": 0.8631888481509743,
+      "grad_norm": 8.161453703849475,
+      "learning_rate": 2.796505177891692e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3815,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.625,
+      "step": 23190
+    },
+    {
+      "epoch": 0.8635610727512981,
+      "grad_norm": 7.235544059057991,
+      "learning_rate": 2.7815951430923247e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.382,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.1875,
+      "step": 23200
+    },
+    {
+      "epoch": 0.8639332973516219,
+      "grad_norm": 8.558291685305683,
+      "learning_rate": 2.7667226204306588e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3933,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.4375,
+      "step": 23210
+    },
+    {
+      "epoch": 0.8643055219519458,
+      "grad_norm": 7.934375788243854,
+      "learning_rate": 2.7518876350164923e-08,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.383,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.4375,
+      "step": 23220
+    },
+    {
+      "epoch": 0.8646777465522696,
+      "grad_norm": 8.49402132109239,
+      "learning_rate": 2.737090211896295e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3868,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.4375,
+      "step": 23230
+    },
+    {
+      "epoch": 0.8650499711525935,
+      "grad_norm": 7.324709795907441,
+      "learning_rate": 2.7223303760530726e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.392,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 2.0,
+      "rewards/rejected": -9.625,
+      "step": 23240
+    },
+    {
+      "epoch": 0.8654221957529173,
+      "grad_norm": 8.419987161609146,
+      "learning_rate": 2.7076081524064016e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.397,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -9.5,
+      "step": 23250
+    },
+    {
+      "epoch": 0.8657944203532412,
+      "grad_norm": 8.120961578927778,
+      "learning_rate": 2.692923565812333e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3774,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.625,
+      "step": 23260
+    },
+    {
+      "epoch": 0.866166644953565,
+      "grad_norm": 7.432177674973987,
+      "learning_rate": 2.678276641063404e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3674,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.5625,
+      "step": 23270
+    },
+    {
+      "epoch": 0.8665388695538888,
+      "grad_norm": 7.8277850457451,
+      "learning_rate": 2.663667402888531e-08,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3885,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.25,
+      "step": 23280
+    },
+    {
+      "epoch": 0.8669110941542126,
+      "grad_norm": 7.618575853065507,
+      "learning_rate": 2.649095875953028e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.75,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3882,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.25,
+      "step": 23290
+    },
+    {
+      "epoch": 0.8672833187545365,
+      "grad_norm": 7.697885187086435,
+      "learning_rate": 2.634562084858538e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3883,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -9.4375,
+      "step": 23300
+    },
+    {
+      "epoch": 0.8676555433548603,
+      "grad_norm": 8.23114887860297,
+      "learning_rate": 2.6200660541429692e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3732,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.96875,
+      "rewards/rejected": -9.75,
+      "step": 23310
+    },
+    {
+      "epoch": 0.8680277679551841,
+      "grad_norm": 9.164896557866575,
+      "learning_rate": 2.605607808280516e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3944,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.953125,
+      "rewards/rejected": -9.5,
+      "step": 23320
+    },
+    {
+      "epoch": 0.868399992555508,
+      "grad_norm": 7.559894670977543,
+      "learning_rate": 2.5911873716815442e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3833,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.375,
+      "step": 23330
+    },
+    {
+      "epoch": 0.8687722171558319,
+      "grad_norm": 7.256527179112857,
+      "learning_rate": 2.5768047686926086e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3727,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.375,
+      "step": 23340
+    },
+    {
+      "epoch": 0.8691444417561557,
+      "grad_norm": 7.042434515586065,
+      "learning_rate": 2.562460023596369e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.421875,
+      "logps/chosen": -912.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3949,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.375,
+      "step": 23350
+    },
+    {
+      "epoch": 0.8695166663564795,
+      "grad_norm": 6.290132048129441,
+      "learning_rate": 2.5481531606115825e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3954,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.5,
+      "step": 23360
+    },
+    {
+      "epoch": 0.8698888909568033,
+      "grad_norm": 7.592129336101318,
+      "learning_rate": 2.533884203893044e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3848,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.375,
+      "step": 23370
+    },
+    {
+      "epoch": 0.8702611155571272,
+      "grad_norm": 8.05046030097679,
+      "learning_rate": 2.5196531775315528e-08,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3849,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.4375,
+      "step": 23380
+    },
+    {
+      "epoch": 0.870633340157451,
+      "grad_norm": 8.334980120104557,
+      "learning_rate": 2.5054601055538667e-08,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.364,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.5,
+      "step": 23390
+    },
+    {
+      "epoch": 0.8710055647577748,
+      "grad_norm": 6.995209242901596,
+      "learning_rate": 2.4913050119226563e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3906,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.4375,
+      "step": 23400
+    },
+    {
+      "epoch": 0.8713777893580987,
+      "grad_norm": 7.368522666236933,
+      "learning_rate": 2.4771879205364832e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.4064,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.0625,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.6875,
+      "step": 23410
+    },
+    {
+      "epoch": 0.8717500139584226,
+      "grad_norm": 7.179543070215981,
+      "learning_rate": 2.4631088552297423e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -992.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3813,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -9.3125,
+      "step": 23420
+    },
+    {
+      "epoch": 0.8721222385587464,
+      "grad_norm": 6.969375729195452,
+      "learning_rate": 2.4490678397726396e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1152.0,
+      "loss": 0.3892,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 2.03125,
+      "rewards/rejected": -9.875,
+      "step": 23430
+    },
+    {
+      "epoch": 0.8724944631590702,
+      "grad_norm": 8.37784616949306,
+      "learning_rate": 2.4350648978711142e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3725,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.3125,
+      "step": 23440
+    },
+    {
+      "epoch": 0.872866687759394,
+      "grad_norm": 7.328634563396927,
+      "learning_rate": 2.421100053166844e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3869,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.5,
+      "step": 23450
+    },
+    {
+      "epoch": 0.8732389123597178,
+      "grad_norm": 6.996494210029249,
+      "learning_rate": 2.4071733292371944e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.364,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 2.03125,
+      "rewards/rejected": -9.5,
+      "step": 23460
+    },
+    {
+      "epoch": 0.8736111369600417,
+      "grad_norm": 7.4672031660140235,
+      "learning_rate": 2.3932847495951453e-08,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.366,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -9.5,
+      "step": 23470
+    },
+    {
+      "epoch": 0.8739833615603655,
+      "grad_norm": 7.683824837627566,
+      "learning_rate": 2.3794343376892982e-08,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.5,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3732,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.4375,
+      "step": 23480
+    },
+    {
+      "epoch": 0.8743555861606893,
+      "grad_norm": 7.158926842690174,
+      "learning_rate": 2.3656221169037926e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3897,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.5,
+      "step": 23490
+    },
+    {
+      "epoch": 0.8747278107610132,
+      "grad_norm": 7.63142793133654,
+      "learning_rate": 2.3518481105583104e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3925,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.5,
+      "step": 23500
+    },
+    {
+      "epoch": 0.8751000353613371,
+      "grad_norm": 7.437015785256054,
+      "learning_rate": 2.3381123419080022e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3832,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.625,
+      "step": 23510
+    },
+    {
+      "epoch": 0.8754722599616609,
+      "grad_norm": 7.507636291724757,
+      "learning_rate": 2.3244148341434707e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3972,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -9.25,
+      "step": 23520
+    },
+    {
+      "epoch": 0.8758444845619847,
+      "grad_norm": 7.950372150486875,
+      "learning_rate": 2.3107556103907022e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3754,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.4375,
+      "step": 23530
+    },
+    {
+      "epoch": 0.8762167091623085,
+      "grad_norm": 8.582897850737993,
+      "learning_rate": 2.2971346937110668e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3839,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 2.0625,
+      "rewards/rejected": -9.625,
+      "step": 23540
+    },
+    {
+      "epoch": 0.8765889337626324,
+      "grad_norm": 7.412439102486501,
+      "learning_rate": 2.283552107101247e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3906,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -9.375,
+      "step": 23550
+    },
+    {
+      "epoch": 0.8769611583629562,
+      "grad_norm": 8.232224985677536,
+      "learning_rate": 2.270007873493221e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.408,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -9.0625,
+      "step": 23560
+    },
+    {
+      "epoch": 0.87733338296328,
+      "grad_norm": 7.929402476657699,
+      "learning_rate": 2.256502015754211e-08,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.4098,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.4375,
+      "step": 23570
+    },
+    {
+      "epoch": 0.8777056075636038,
+      "grad_norm": 6.249545658203031,
+      "learning_rate": 2.2430345566866378e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.385,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.5,
+      "step": 23580
+    },
+    {
+      "epoch": 0.8780778321639278,
+      "grad_norm": 7.121101943554836,
+      "learning_rate": 2.2296055190281023e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3846,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.4375,
+      "step": 23590
+    },
+    {
+      "epoch": 0.8784500567642516,
+      "grad_norm": 7.445947921275176,
+      "learning_rate": 2.2162149254513374e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3824,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.1875,
+      "step": 23600
+    },
+    {
+      "epoch": 0.8788222813645754,
+      "grad_norm": 7.008136428397299,
+      "learning_rate": 2.2028627985641702e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3766,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -9.25,
+      "step": 23610
+    },
+    {
+      "epoch": 0.8791945059648992,
+      "grad_norm": 9.428053672567371,
+      "learning_rate": 2.1895491609094735e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3786,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.25,
+      "step": 23620
+    },
+    {
+      "epoch": 0.879566730565223,
+      "grad_norm": 6.965567056686397,
+      "learning_rate": 2.1762740349651476e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3924,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.1875,
+      "step": 23630
+    },
+    {
+      "epoch": 0.8799389551655469,
+      "grad_norm": 8.233107927923399,
+      "learning_rate": 2.1630374431440668e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3666,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.4375,
+      "step": 23640
+    },
+    {
+      "epoch": 0.8803111797658707,
+      "grad_norm": 7.990932484125737,
+      "learning_rate": 2.149839407794052e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3693,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -9.25,
+      "step": 23650
+    },
+    {
+      "epoch": 0.8806834043661945,
+      "grad_norm": 8.457721229650284,
+      "learning_rate": 2.136679951197823e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3574,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.4375,
+      "step": 23660
+    },
+    {
+      "epoch": 0.8810556289665183,
+      "grad_norm": 9.092648384261125,
+      "learning_rate": 2.1235590955729632e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4001,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.1875,
+      "step": 23670
+    },
+    {
+      "epoch": 0.8814278535668423,
+      "grad_norm": 8.329676556304953,
+      "learning_rate": 2.1104768630718915e-08,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3915,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.3125,
+      "step": 23680
+    },
+    {
+      "epoch": 0.8818000781671661,
+      "grad_norm": 7.824932105876317,
+      "learning_rate": 2.0974332757818153e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3728,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.125,
+      "step": 23690
+    },
+    {
+      "epoch": 0.8821723027674899,
+      "grad_norm": 8.249231385804052,
+      "learning_rate": 2.084428355724699e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3847,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.5,
+      "step": 23700
+    },
+    {
+      "epoch": 0.8825445273678137,
+      "grad_norm": 7.361717393223829,
+      "learning_rate": 2.0714621248572124e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.25,
+      "step": 23710
+    },
+    {
+      "epoch": 0.8829167519681376,
+      "grad_norm": 6.63534360905492,
+      "learning_rate": 2.0585346050707138e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3713,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.3125,
+      "step": 23720
+    },
+    {
+      "epoch": 0.8832889765684614,
+      "grad_norm": 8.807030508722738,
+      "learning_rate": 2.045645818191208e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3727,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.25,
+      "step": 23730
+    },
+    {
+      "epoch": 0.8836612011687852,
+      "grad_norm": 8.824500303269753,
+      "learning_rate": 2.0327957859792967e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3915,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.5,
+      "step": 23740
+    },
+    {
+      "epoch": 0.884033425769109,
+      "grad_norm": 7.777502681653569,
+      "learning_rate": 2.0199845301301616e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3804,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -9.4375,
+      "step": 23750
+    },
+    {
+      "epoch": 0.8844056503694329,
+      "grad_norm": 8.743415934980693,
+      "learning_rate": 2.007212072273501e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3889,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.4375,
+      "step": 23760
+    },
+    {
+      "epoch": 0.8847778749697568,
+      "grad_norm": 7.846106413119858,
+      "learning_rate": 1.9944784339735232e-08,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3952,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.4375,
+      "step": 23770
+    },
+    {
+      "epoch": 0.8851500995700806,
+      "grad_norm": 7.46948168535247,
+      "learning_rate": 1.981783636728887e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3649,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.4375,
+      "rewards/margins": 1.9921875,
+      "rewards/rejected": -9.4375,
+      "step": 23780
+    },
+    {
+      "epoch": 0.8855223241704044,
+      "grad_norm": 7.721275113500026,
+      "learning_rate": 1.9691277019726838e-08,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.859375,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.377,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.4375,
+      "step": 23790
+    },
+    {
+      "epoch": 0.8858945487707283,
+      "grad_norm": 7.443427468231076,
+      "learning_rate": 1.956510651072385e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3829,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.4375,
+      "step": 23800
+    },
+    {
+      "epoch": 0.8862667733710521,
+      "grad_norm": 8.99734584932058,
+      "learning_rate": 1.9439325053298123e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3948,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -8.0625,
+      "rewards/margins": 1.4296875,
+      "rewards/rejected": -9.5,
+      "step": 23810
+    },
+    {
+      "epoch": 0.8866389979713759,
+      "grad_norm": 7.972096153402294,
+      "learning_rate": 1.9313932859811033e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3986,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.9296875,
+      "rewards/rejected": -9.6875,
+      "step": 23820
+    },
+    {
+      "epoch": 0.8870112225716997,
+      "grad_norm": 7.308577147954605,
+      "learning_rate": 1.918893014196682e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3742,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.25,
+      "step": 23830
+    },
+    {
+      "epoch": 0.8873834471720236,
+      "grad_norm": 8.0342999596137,
+      "learning_rate": 1.9064317110812106e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3794,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.3125,
+      "step": 23840
+    },
+    {
+      "epoch": 0.8877556717723475,
+      "grad_norm": 6.769370536093424,
+      "learning_rate": 1.894009397673549e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3887,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.4375,
+      "step": 23850
+    },
+    {
+      "epoch": 0.8881278963726713,
+      "grad_norm": 10.114355579038438,
+      "learning_rate": 1.881626094946745e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3756,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.5625,
+      "step": 23860
+    },
+    {
+      "epoch": 0.8885001209729951,
+      "grad_norm": 7.591787194937937,
+      "learning_rate": 1.8692818238079778e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.4123,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.4375,
+      "step": 23870
+    },
+    {
+      "epoch": 0.888872345573319,
+      "grad_norm": 8.59697232232149,
+      "learning_rate": 1.8569766050985257e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3787,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.375,
+      "step": 23880
+    },
+    {
+      "epoch": 0.8892445701736428,
+      "grad_norm": 8.338137123278548,
+      "learning_rate": 1.8447104595937428e-08,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3686,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 2.125,
+      "rewards/rejected": -9.8125,
+      "step": 23890
+    },
+    {
+      "epoch": 0.8896167947739666,
+      "grad_norm": 7.062740904062344,
+      "learning_rate": 1.832483408002994e-08,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3976,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -9.125,
+      "step": 23900
+    },
+    {
+      "epoch": 0.8899890193742904,
+      "grad_norm": 6.061875564220678,
+      "learning_rate": 1.82029547096966e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3745,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.5,
+      "step": 23910
+    },
+    {
+      "epoch": 0.8903612439746142,
+      "grad_norm": 9.082797920973723,
+      "learning_rate": 1.8081466690710755e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3918,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.375,
+      "step": 23920
+    },
+    {
+      "epoch": 0.8907334685749381,
+      "grad_norm": 8.254154693838451,
+      "learning_rate": 1.7960370228185083e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3886,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.125,
+      "step": 23930
+    },
+    {
+      "epoch": 0.891105693175262,
+      "grad_norm": 8.295709625398294,
+      "learning_rate": 1.7839665526571014e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3661,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.625,
+      "step": 23940
+    },
+    {
+      "epoch": 0.8914779177755858,
+      "grad_norm": 7.819768258036864,
+      "learning_rate": 1.7719352789658775e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.392,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.4375,
+      "step": 23950
+    },
+    {
+      "epoch": 0.8918501423759096,
+      "grad_norm": 7.6720267294618525,
+      "learning_rate": 1.759943222057661e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.364,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.5625,
+      "step": 23960
+    },
+    {
+      "epoch": 0.8922223669762335,
+      "grad_norm": 11.897852411899958,
+      "learning_rate": 1.7479904021790863e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3925,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.625,
+      "step": 23970
+    },
+    {
+      "epoch": 0.8925945915765573,
+      "grad_norm": 10.611319732656625,
+      "learning_rate": 1.736076839510531e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.4051,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -9.375,
+      "step": 23980
+    },
+    {
+      "epoch": 0.8929668161768811,
+      "grad_norm": 6.714058338379632,
+      "learning_rate": 1.7242025541660875e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.4041,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.3125,
+      "step": 23990
+    },
+    {
+      "epoch": 0.8933390407772049,
+      "grad_norm": 6.125954344285272,
+      "learning_rate": 1.7123675661935483e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.4024,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.5625,
+      "step": 24000
+    },
+    {
+      "epoch": 0.8937112653775288,
+      "grad_norm": 9.384844756909843,
+      "learning_rate": 1.7005718955743432e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -988.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.4032,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.75,
+      "step": 24010
+    },
+    {
+      "epoch": 0.8940834899778526,
+      "grad_norm": 8.615495388964437,
+      "learning_rate": 1.6888155622235474e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.398,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.5625,
+      "step": 24020
+    },
+    {
+      "epoch": 0.8944557145781765,
+      "grad_norm": 7.869135045463567,
+      "learning_rate": 1.6770985859897863e-08,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3618,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.375,
+      "step": 24030
+    },
+    {
+      "epoch": 0.8948279391785003,
+      "grad_norm": 7.362574538668546,
+      "learning_rate": 1.6654209866552638e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -992.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3733,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.75,
+      "step": 24040
+    },
+    {
+      "epoch": 0.8952001637788242,
+      "grad_norm": 9.806680353282909,
+      "learning_rate": 1.6537827839356923e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4088,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.375,
+      "step": 24050
+    },
+    {
+      "epoch": 0.895572388379148,
+      "grad_norm": 8.58482624740034,
+      "learning_rate": 1.642183997480273e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3986,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -9.1875,
+      "step": 24060
+    },
+    {
+      "epoch": 0.8959446129794718,
+      "grad_norm": 7.170916296070867,
+      "learning_rate": 1.630624646871659e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3968,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 2.03125,
+      "rewards/rejected": -9.625,
+      "step": 24070
+    },
+    {
+      "epoch": 0.8963168375797956,
+      "grad_norm": 7.086395382937031,
+      "learning_rate": 1.619104751625913e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3452,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.375,
+      "step": 24080
+    },
+    {
+      "epoch": 0.8966890621801195,
+      "grad_norm": 6.888671145877337,
+      "learning_rate": 1.6076243311924974e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1064.0,
+      "loss": 0.3689,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -9.25,
+      "step": 24090
+    },
+    {
+      "epoch": 0.8970612867804433,
+      "grad_norm": 8.026012423504191,
+      "learning_rate": 1.5961834049542154e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.4113,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.4375,
+      "step": 24100
+    },
+    {
+      "epoch": 0.8974335113807671,
+      "grad_norm": 8.449592701000958,
+      "learning_rate": 1.5847819922272077e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3675,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 2.109375,
+      "rewards/rejected": -9.5,
+      "step": 24110
+    },
+    {
+      "epoch": 0.897805735981091,
+      "grad_norm": 7.616328800398696,
+      "learning_rate": 1.573420112260884e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3788,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.3125,
+      "step": 24120
+    },
+    {
+      "epoch": 0.8981779605814149,
+      "grad_norm": 8.121434555411774,
+      "learning_rate": 1.56209778423792e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3794,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -9.25,
+      "step": 24130
+    },
+    {
+      "epoch": 0.8985501851817387,
+      "grad_norm": 7.199913062183862,
+      "learning_rate": 1.550815027274216e-08,
+      "logits/chosen": -3.625,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -916.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3874,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.9296875,
+      "rewards/rejected": -9.4375,
+      "step": 24140
+    },
+    {
+      "epoch": 0.8989224097820625,
+      "grad_norm": 7.834115318469685,
+      "learning_rate": 1.5395718604188506e-08,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3792,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -9.25,
+      "step": 24150
+    },
+    {
+      "epoch": 0.8992946343823863,
+      "grad_norm": 7.066007652832291,
+      "learning_rate": 1.528368302654079e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3877,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -9.1875,
+      "step": 24160
+    },
+    {
+      "epoch": 0.8996668589827101,
+      "grad_norm": 7.709458252757219,
+      "learning_rate": 1.5172043728952672e-08,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.381,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.625,
+      "step": 24170
+    },
+    {
+      "epoch": 0.900039083583034,
+      "grad_norm": 8.926782992191475,
+      "learning_rate": 1.506080089990888e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3977,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": -9.3125,
+      "step": 24180
+    },
+    {
+      "epoch": 0.9004113081833578,
+      "grad_norm": 9.10356845704118,
+      "learning_rate": 1.4949954727224627e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3948,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.1875,
+      "step": 24190
+    },
+    {
+      "epoch": 0.9007835327836817,
+      "grad_norm": 6.504294296086254,
+      "learning_rate": 1.48395053980456e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3962,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.3125,
+      "step": 24200
+    },
+    {
+      "epoch": 0.9011557573840056,
+      "grad_norm": 6.757474926442064,
+      "learning_rate": 1.472945309884735e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.386,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -9.125,
+      "step": 24210
+    },
+    {
+      "epoch": 0.9015279819843294,
+      "grad_norm": 7.2983616340408535,
+      "learning_rate": 1.4619798015435198e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4136,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -9.375,
+      "step": 24220
+    },
+    {
+      "epoch": 0.9019002065846532,
+      "grad_norm": 7.720140323166754,
+      "learning_rate": 1.4510540332943799e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3529,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.4375,
+      "step": 24230
+    },
+    {
+      "epoch": 0.902272431184977,
+      "grad_norm": 8.476672522650148,
+      "learning_rate": 1.4401680235836766e-08,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3745,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 2.0,
+      "rewards/rejected": -9.5,
+      "step": 24240
+    },
+    {
+      "epoch": 0.9026446557853008,
+      "grad_norm": 7.566248208144642,
+      "learning_rate": 1.4293217907906663e-08,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3727,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.4375,
+      "step": 24250
+    },
+    {
+      "epoch": 0.9030168803856247,
+      "grad_norm": 8.742504638356277,
+      "learning_rate": 1.4185153532274313e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.379,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.3125,
+      "step": 24260
+    },
+    {
+      "epoch": 0.9033891049859485,
+      "grad_norm": 8.305964372214007,
+      "learning_rate": 1.4077487291388746e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3776,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -9.4375,
+      "step": 24270
+    },
+    {
+      "epoch": 0.9037613295862723,
+      "grad_norm": 7.468556622895356,
+      "learning_rate": 1.3970219367026692e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3743,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -9.5,
+      "step": 24280
+    },
+    {
+      "epoch": 0.9041335541865962,
+      "grad_norm": 7.992257195242798,
+      "learning_rate": 1.3863349940292562e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3589,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -9.5,
+      "step": 24290
+    },
+    {
+      "epoch": 0.9045057787869201,
+      "grad_norm": 7.9115847678859,
+      "learning_rate": 1.3756879191617831e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3739,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.375,
+      "step": 24300
+    },
+    {
+      "epoch": 0.9048780033872439,
+      "grad_norm": 8.300383475053408,
+      "learning_rate": 1.365080730076093e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.378,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.5,
+      "step": 24310
+    },
+    {
+      "epoch": 0.9052502279875677,
+      "grad_norm": 7.554307295537521,
+      "learning_rate": 1.3545134446806911e-08,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3793,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.5625,
+      "step": 24320
+    },
+    {
+      "epoch": 0.9056224525878915,
+      "grad_norm": 8.437668317453543,
+      "learning_rate": 1.343986080816703e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3743,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.375,
+      "step": 24330
+    },
+    {
+      "epoch": 0.9059946771882154,
+      "grad_norm": 8.72210225861552,
+      "learning_rate": 1.3334986562578614e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -988.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.375,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.125,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -9.5625,
+      "step": 24340
+    },
+    {
+      "epoch": 0.9063669017885392,
+      "grad_norm": 8.157291870928926,
+      "learning_rate": 1.3230511887104639e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3653,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.3125,
+      "step": 24350
+    },
+    {
+      "epoch": 0.906739126388863,
+      "grad_norm": 7.419196843865015,
+      "learning_rate": 1.3126436958133507e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3656,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -9.375,
+      "step": 24360
+    },
+    {
+      "epoch": 0.9071113509891868,
+      "grad_norm": 7.799395303034371,
+      "learning_rate": 1.3022761951378663e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3579,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.5,
+      "step": 24370
+    },
+    {
+      "epoch": 0.9074835755895108,
+      "grad_norm": 8.035944812606981,
+      "learning_rate": 1.2919487041878369e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3922,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -9.3125,
+      "step": 24380
+    },
+    {
+      "epoch": 0.9078558001898346,
+      "grad_norm": 7.44304132857029,
+      "learning_rate": 1.2816612403995397e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3924,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.375,
+      "step": 24390
+    },
+    {
+      "epoch": 0.9082280247901584,
+      "grad_norm": 7.850317381255466,
+      "learning_rate": 1.2714138211416758e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3801,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.5,
+      "step": 24400
+    },
+    {
+      "epoch": 0.9086002493904822,
+      "grad_norm": 7.672167977278572,
+      "learning_rate": 1.2612064637153336e-08,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3841,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.3125,
+      "step": 24410
+    },
+    {
+      "epoch": 0.908972473990806,
+      "grad_norm": 7.637465638332765,
+      "learning_rate": 1.251039185353961e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.4024,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.5,
+      "step": 24420
+    },
+    {
+      "epoch": 0.9093446985911299,
+      "grad_norm": 8.123031020217443,
+      "learning_rate": 1.2409120032233433e-08,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3836,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.375,
+      "step": 24430
+    },
+    {
+      "epoch": 0.9097169231914537,
+      "grad_norm": 8.788958772757523,
+      "learning_rate": 1.2308249344215704e-08,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.4029,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.5625,
+      "step": 24440
+    },
+    {
+      "epoch": 0.9100891477917775,
+      "grad_norm": 7.608099359312187,
+      "learning_rate": 1.220777995979011e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.625,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3778,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.828125,
+      "rewards/rejected": -9.6875,
+      "step": 24450
+    },
+    {
+      "epoch": 0.9104613723921013,
+      "grad_norm": 9.326755098618014,
+      "learning_rate": 1.210771204858263e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3599,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.375,
+      "step": 24460
+    },
+    {
+      "epoch": 0.9108335969924253,
+      "grad_norm": 7.9267426859429815,
+      "learning_rate": 1.200804577954162e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3949,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.75,
+      "step": 24470
+    },
+    {
+      "epoch": 0.9112058215927491,
+      "grad_norm": 6.54227519354369,
+      "learning_rate": 1.1908781320937256e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3862,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.5,
+      "step": 24480
+    },
+    {
+      "epoch": 0.9115780461930729,
+      "grad_norm": 8.300787377740791,
+      "learning_rate": 1.1809918840361282e-08,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3908,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.5,
+      "step": 24490
+    },
+    {
+      "epoch": 0.9119502707933967,
+      "grad_norm": 7.522409596731626,
+      "learning_rate": 1.171145850472688e-08,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3776,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -9.5625,
+      "step": 24500
+    },
+    {
+      "epoch": 0.9123224953937206,
+      "grad_norm": 7.699094846255386,
+      "learning_rate": 1.16134004802681e-08,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1160.0,
+      "loss": 0.3737,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": -9.6875,
+      "step": 24510
+    },
+    {
+      "epoch": 0.9126947199940444,
+      "grad_norm": 6.741315373186182,
+      "learning_rate": 1.1515744932539929e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -996.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3556,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.5625,
+      "step": 24520
+    },
+    {
+      "epoch": 0.9130669445943682,
+      "grad_norm": 9.524287978227134,
+      "learning_rate": 1.1418492026417703e-08,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.4003,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.5,
+      "step": 24530
+    },
+    {
+      "epoch": 0.913439169194692,
+      "grad_norm": 8.761360796146077,
+      "learning_rate": 1.1321641926097137e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3518,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.625,
+      "step": 24540
+    },
+    {
+      "epoch": 0.9138113937950159,
+      "grad_norm": 8.548143936738693,
+      "learning_rate": 1.1225194795093623e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3893,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.3125,
+      "step": 24550
+    },
+    {
+      "epoch": 0.9141836183953398,
+      "grad_norm": 7.356786296428599,
+      "learning_rate": 1.1129150796242436e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3677,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.9765625,
+      "rewards/rejected": -9.5625,
+      "step": 24560
+    },
+    {
+      "epoch": 0.9145558429956636,
+      "grad_norm": 8.134624126383903,
+      "learning_rate": 1.1033510091698117e-08,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4008,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.375,
+      "step": 24570
+    },
+    {
+      "epoch": 0.9149280675959874,
+      "grad_norm": 9.191711565618723,
+      "learning_rate": 1.0938272842934304e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3709,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -9.375,
+      "step": 24580
+    },
+    {
+      "epoch": 0.9153002921963113,
+      "grad_norm": 8.586915312959455,
+      "learning_rate": 1.0843439210743572e-08,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3999,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.4375,
+      "step": 24590
+    },
+    {
+      "epoch": 0.9156725167966351,
+      "grad_norm": 8.205902605994037,
+      "learning_rate": 1.0749009355236898e-08,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3885,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.375,
+      "step": 24600
+    },
+    {
+      "epoch": 0.9160447413969589,
+      "grad_norm": 8.255725667513316,
+      "learning_rate": 1.0654983435843645e-08,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3836,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -9.3125,
+      "step": 24610
+    },
+    {
+      "epoch": 0.9164169659972827,
+      "grad_norm": 8.926505569083105,
+      "learning_rate": 1.0561361611311188e-08,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3549,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 2.21875,
+      "rewards/rejected": -9.9375,
+      "step": 24620
+    },
+    {
+      "epoch": 0.9167891905976066,
+      "grad_norm": 7.936119776674965,
+      "learning_rate": 1.046814403970464e-08,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3886,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.5625,
+      "step": 24630
+    },
+    {
+      "epoch": 0.9171614151979305,
+      "grad_norm": 7.635210531673523,
+      "learning_rate": 1.037533087840664e-08,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3823,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.96875,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.5625,
+      "step": 24640
+    },
+    {
+      "epoch": 0.9175336397982543,
+      "grad_norm": 8.575933521526409,
+      "learning_rate": 1.0282922284116951e-08,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3916,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.96875,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.5625,
+      "step": 24650
+    },
+    {
+      "epoch": 0.9179058643985781,
+      "grad_norm": 11.640605377886343,
+      "learning_rate": 1.0190918412852384e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3984,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 2.046875,
+      "rewards/rejected": -9.6875,
+      "step": 24660
+    },
+    {
+      "epoch": 0.918278088998902,
+      "grad_norm": 8.139747004873367,
+      "learning_rate": 1.0099319419946406e-08,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3801,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 2.046875,
+      "rewards/rejected": -9.6875,
+      "step": 24670
+    },
+    {
+      "epoch": 0.9186503135992258,
+      "grad_norm": 9.978017658068548,
+      "learning_rate": 1.0008125460048921e-08,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3811,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.625,
+      "step": 24680
+    },
+    {
+      "epoch": 0.9190225381995496,
+      "grad_norm": 7.515756241230707,
+      "learning_rate": 9.917336687125938e-09,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3801,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.3125,
+      "step": 24690
+    },
+    {
+      "epoch": 0.9193947627998734,
+      "grad_norm": 6.734743377996093,
+      "learning_rate": 9.826953254459508e-09,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -996.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3801,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.0,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -9.5625,
+      "step": 24700
+    },
+    {
+      "epoch": 0.9197669874001972,
+      "grad_norm": 8.01737259049045,
+      "learning_rate": 9.736975314647205e-09,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3869,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.375,
+      "step": 24710
+    },
+    {
+      "epoch": 0.9201392120005211,
+      "grad_norm": 6.974434479985267,
+      "learning_rate": 9.647403019602068e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3956,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.4375,
+      "step": 24720
+    },
+    {
+      "epoch": 0.920511436600845,
+      "grad_norm": 7.683392163599637,
+      "learning_rate": 9.558236520552265e-09,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3794,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.5625,
+      "step": 24730
+    },
+    {
+      "epoch": 0.9208836612011688,
+      "grad_norm": 7.030826030041779,
+      "learning_rate": 9.469475968040763e-09,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.75,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3718,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.5625,
+      "step": 24740
+    },
+    {
+      "epoch": 0.9212558858014926,
+      "grad_norm": 9.099713273938125,
+      "learning_rate": 9.381121511925243e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -992.0,
+      "logps/rejected": -1160.0,
+      "loss": 0.4022,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.5,
+      "step": 24750
+    },
+    {
+      "epoch": 0.9216281104018165,
+      "grad_norm": 8.782674466854482,
+      "learning_rate": 9.293173301377772e-09,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.921875,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.387,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.5625,
+      "step": 24760
+    },
+    {
+      "epoch": 0.9220003350021403,
+      "grad_norm": 8.166861497095521,
+      "learning_rate": 9.205631484884485e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3872,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.625,
+      "step": 24770
+    },
+    {
+      "epoch": 0.9223725596024641,
+      "grad_norm": 8.662590351472907,
+      "learning_rate": 9.11849621024538e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3849,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.828125,
+      "rewards/rejected": -9.625,
+      "step": 24780
+    },
+    {
+      "epoch": 0.9227447842027879,
+      "grad_norm": 9.416919522717194,
+      "learning_rate": 9.031767624574083e-09,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3801,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.5625,
+      "step": 24790
+    },
+    {
+      "epoch": 0.9231170088031118,
+      "grad_norm": 8.003333767896079,
+      "learning_rate": 8.945445874297659e-09,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3801,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.515625,
+      "rewards/rejected": -9.375,
+      "step": 24800
+    },
+    {
+      "epoch": 0.9234892334034356,
+      "grad_norm": 8.177854924222288,
+      "learning_rate": 8.859531105156193e-09,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3762,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -9.3125,
+      "step": 24810
+    },
+    {
+      "epoch": 0.9238614580037595,
+      "grad_norm": 7.169427388075016,
+      "learning_rate": 8.774023462202768e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -992.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3838,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -9.6875,
+      "step": 24820
+    },
+    {
+      "epoch": 0.9242336826040833,
+      "grad_norm": 7.905858490028348,
+      "learning_rate": 8.688923089802957e-09,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3778,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.5,
+      "step": 24830
+    },
+    {
+      "epoch": 0.9246059072044072,
+      "grad_norm": 8.79668516589034,
+      "learning_rate": 8.604230131634832e-09,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -988.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.381,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.96875,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.75,
+      "step": 24840
+    },
+    {
+      "epoch": 0.924978131804731,
+      "grad_norm": 8.580434253660142,
+      "learning_rate": 8.519944730688594e-09,
+      "logits/chosen": -3.625,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3742,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.4375,
+      "step": 24850
+    },
+    {
+      "epoch": 0.9253503564050548,
+      "grad_norm": 9.715055464613178,
+      "learning_rate": 8.436067029266358e-09,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4074,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.46875,
+      "rewards/rejected": -9.375,
+      "step": 24860
+    },
+    {
+      "epoch": 0.9257225810053786,
+      "grad_norm": 7.971859200012785,
+      "learning_rate": 8.352597168981845e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3708,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -9.375,
+      "step": 24870
+    },
+    {
+      "epoch": 0.9260948056057025,
+      "grad_norm": 7.990376771678073,
+      "learning_rate": 8.269535290760271e-09,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.37,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.625,
+      "step": 24880
+    },
+    {
+      "epoch": 0.9264670302060263,
+      "grad_norm": 7.290364375365611,
+      "learning_rate": 8.18688153483793e-09,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3862,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.4375,
+      "step": 24890
+    },
+    {
+      "epoch": 0.9268392548063501,
+      "grad_norm": 8.158983456696227,
+      "learning_rate": 8.104636040762281e-09,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3867,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.5625,
+      "step": 24900
+    },
+    {
+      "epoch": 0.927211479406674,
+      "grad_norm": 7.092862182723741,
+      "learning_rate": 8.022798947391302e-09,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3767,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -8.0,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.5625,
+      "step": 24910
+    },
+    {
+      "epoch": 0.9275837040069979,
+      "grad_norm": 8.864238313394411,
+      "learning_rate": 7.941370392893526e-09,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3798,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.5,
+      "step": 24920
+    },
+    {
+      "epoch": 0.9279559286073217,
+      "grad_norm": 6.9544353166281825,
+      "learning_rate": 7.86035051474776e-09,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3606,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.25,
+      "step": 24930
+    },
+    {
+      "epoch": 0.9283281532076455,
+      "grad_norm": 7.1815304422499695,
+      "learning_rate": 7.779739449742723e-09,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -992.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3906,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.625,
+      "step": 24940
+    },
+    {
+      "epoch": 0.9287003778079693,
+      "grad_norm": 7.423891097177366,
+      "learning_rate": 7.699537333977101e-09,
+      "logits/chosen": -3.953125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.4018,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.625,
+      "step": 24950
+    },
+    {
+      "epoch": 0.9290726024082931,
+      "grad_norm": 9.061510486130416,
+      "learning_rate": 7.619744302858944e-09,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3617,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.375,
+      "step": 24960
+    },
+    {
+      "epoch": 0.929444827008617,
+      "grad_norm": 10.84337078464033,
+      "learning_rate": 7.540360491105763e-09,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4236,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -9.0,
+      "step": 24970
+    },
+    {
+      "epoch": 0.9298170516089408,
+      "grad_norm": 7.289127447021336,
+      "learning_rate": 7.461386032744099e-09,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3938,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.375,
+      "step": 24980
+    },
+    {
+      "epoch": 0.9301892762092646,
+      "grad_norm": 8.793529128155344,
+      "learning_rate": 7.382821061109407e-09,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.5,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3889,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.375,
+      "step": 24990
+    },
+    {
+      "epoch": 0.9305615008095885,
+      "grad_norm": 7.577486593713324,
+      "learning_rate": 7.304665708845803e-09,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3791,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.375,
+      "step": 25000
+    },
+    {
+      "epoch": 0.9305615008095885,
+      "eval_logits/chosen": -3.796875,
+      "eval_logits/rejected": -3.609375,
+      "eval_logps/chosen": -1000.0,
+      "eval_logps/rejected": -1120.0,
+      "eval_loss": 0.45970889925956726,
+      "eval_rewards/accuracies": 0.7556513547897339,
+      "eval_rewards/chosen": -7.96875,
+      "eval_rewards/margins": 1.4921875,
+      "eval_rewards/rejected": -9.4375,
+      "eval_runtime": 6272.6886,
+      "eval_samples_per_second": 30.459,
+      "eval_steps_per_second": 0.476,
+      "step": 25000
+    },
+    {
+      "epoch": 0.9309337254099124,
+      "grad_norm": 7.631697835806511,
+      "learning_rate": 7.2269201079057375e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3766,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.921875,
+      "rewards/rejected": -9.625,
+      "step": 25010
+    },
+    {
+      "epoch": 0.9313059500102362,
+      "grad_norm": 8.070707657785505,
+      "learning_rate": 7.149584389549962e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3959,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -9.4375,
+      "step": 25020
+    },
+    {
+      "epoch": 0.93167817461056,
+      "grad_norm": 8.025421810078772,
+      "learning_rate": 7.072658684347116e-09,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3868,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.625,
+      "step": 25030
+    },
+    {
+      "epoch": 0.9320503992108838,
+      "grad_norm": 6.194625094905456,
+      "learning_rate": 6.996143122173753e-09,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.5,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3574,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.953125,
+      "rewards/rejected": -9.5625,
+      "step": 25040
+    },
+    {
+      "epoch": 0.9324226238112077,
+      "grad_norm": 6.98009351294653,
+      "learning_rate": 6.920037832213788e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3749,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.625,
+      "step": 25050
+    },
+    {
+      "epoch": 0.9327948484115315,
+      "grad_norm": 9.032522811445986,
+      "learning_rate": 6.8443429429585755e-09,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3738,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.9296875,
+      "rewards/rejected": -9.5,
+      "step": 25060
+    },
+    {
+      "epoch": 0.9331670730118553,
+      "grad_norm": 6.209517150245189,
+      "learning_rate": 6.769058582206555e-09,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3785,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": -9.6875,
+      "step": 25070
+    },
+    {
+      "epoch": 0.9335392976121792,
+      "grad_norm": 9.51104922502353,
+      "learning_rate": 6.694184877062941e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3986,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.625,
+      "step": 25080
+    },
+    {
+      "epoch": 0.9339115222125031,
+      "grad_norm": 7.860594520966091,
+      "learning_rate": 6.619721953939866e-09,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -1008.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3943,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -8.1875,
+      "rewards/margins": 1.4609375,
+      "rewards/rejected": -9.625,
+      "step": 25090
+    },
+    {
+      "epoch": 0.9342837468128269,
+      "grad_norm": 8.943069700669607,
+      "learning_rate": 6.545669938555654e-09,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3878,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.5,
+      "step": 25100
+    },
+    {
+      "epoch": 0.9346559714131507,
+      "grad_norm": 8.670590824412393,
+      "learning_rate": 6.4720289559351e-09,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -996.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3834,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.625,
+      "step": 25110
+    },
+    {
+      "epoch": 0.9350281960134745,
+      "grad_norm": 6.779608787994875,
+      "learning_rate": 6.3987991304088885e-09,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.348,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.5625,
+      "step": 25120
+    },
+    {
+      "epoch": 0.9354004206137984,
+      "grad_norm": 7.566086459376941,
+      "learning_rate": 6.325980585613594e-09,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3703,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 2.0,
+      "rewards/rejected": -9.75,
+      "step": 25130
+    },
+    {
+      "epoch": 0.9357726452141222,
+      "grad_norm": 7.768196780379803,
+      "learning_rate": 6.2535734444914004e-09,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3673,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": -9.6875,
+      "step": 25140
+    },
+    {
+      "epoch": 0.936144869814446,
+      "grad_norm": 8.055386654010553,
+      "learning_rate": 6.181577829289936e-09,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3787,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -9.5,
+      "step": 25150
+    },
+    {
+      "epoch": 0.9365170944147698,
+      "grad_norm": 8.773668401241743,
+      "learning_rate": 6.109993861561969e-09,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.4049,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.625,
+      "step": 25160
+    },
+    {
+      "epoch": 0.9368893190150938,
+      "grad_norm": 6.528191170735717,
+      "learning_rate": 6.0388216621652945e-09,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3944,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.5625,
+      "step": 25170
+    },
+    {
+      "epoch": 0.9372615436154176,
+      "grad_norm": 7.945368788731117,
+      "learning_rate": 5.968061351262599e-09,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.3756,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.125,
+      "step": 25180
+    },
+    {
+      "epoch": 0.9376337682157414,
+      "grad_norm": 9.538435349616632,
+      "learning_rate": 5.897713048320985e-09,
+      "logits/chosen": -3.609375,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4049,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.5625,
+      "step": 25190
+    },
+    {
+      "epoch": 0.9380059928160652,
+      "grad_norm": 7.283961316472517,
+      "learning_rate": 5.827776872112111e-09,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3711,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.5,
+      "step": 25200
+    },
+    {
+      "epoch": 0.938378217416389,
+      "grad_norm": 6.890310256572558,
+      "learning_rate": 5.758252940711666e-09,
+      "logits/chosen": -3.640625,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3675,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.5,
+      "step": 25210
+    },
+    {
+      "epoch": 0.9387504420167129,
+      "grad_norm": 7.2407951668217425,
+      "learning_rate": 5.689141371499479e-09,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3884,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.5,
+      "step": 25220
+    },
+    {
+      "epoch": 0.9391226666170367,
+      "grad_norm": 6.070852889925006,
+      "learning_rate": 5.6204422811591546e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3521,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.3125,
+      "step": 25230
+    },
+    {
+      "epoch": 0.9394948912173605,
+      "grad_norm": 8.514178948364906,
+      "learning_rate": 5.5521557856778035e-09,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -992.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3903,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -8.0,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.625,
+      "step": 25240
+    },
+    {
+      "epoch": 0.9398671158176843,
+      "grad_norm": 7.275877498281245,
+      "learning_rate": 5.484282000346007e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3822,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.3125,
+      "step": 25250
+    },
+    {
+      "epoch": 0.9402393404180083,
+      "grad_norm": 7.244057043784297,
+      "learning_rate": 5.416821039757491e-09,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -1032.0,
+      "logps/rejected": -1152.0,
+      "loss": 0.3752,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -8.25,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -9.6875,
+      "step": 25260
+    },
+    {
+      "epoch": 0.9406115650183321,
+      "grad_norm": 6.236855043596449,
+      "learning_rate": 5.34977301780909e-09,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3784,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.5390625,
+      "rewards/rejected": -9.3125,
+      "step": 25270
+    },
+    {
+      "epoch": 0.9409837896186559,
+      "grad_norm": 7.603446749171045,
+      "learning_rate": 5.283138047700392e-09,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3677,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.375,
+      "step": 25280
+    },
+    {
+      "epoch": 0.9413560142189797,
+      "grad_norm": 7.913301335245522,
+      "learning_rate": 5.216916241933683e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1168.0,
+      "loss": 0.3801,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 2.046875,
+      "rewards/rejected": -9.875,
+      "step": 25290
+    },
+    {
+      "epoch": 0.9417282388193036,
+      "grad_norm": 9.191383995070396,
+      "learning_rate": 5.151107712313529e-09,
+      "logits/chosen": -3.625,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3781,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.5625,
+      "step": 25300
+    },
+    {
+      "epoch": 0.9421004634196274,
+      "grad_norm": 8.237004236431517,
+      "learning_rate": 5.085712569946915e-09,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.421875,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.4061,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -9.375,
+      "step": 25310
+    },
+    {
+      "epoch": 0.9424726880199512,
+      "grad_norm": 7.546572287719575,
+      "learning_rate": 5.020730925242828e-09,
+      "logits/chosen": -3.96875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3962,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.5,
+      "step": 25320
+    },
+    {
+      "epoch": 0.942844912620275,
+      "grad_norm": 8.52329649689671,
+      "learning_rate": 4.956162887912146e-09,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3899,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.5,
+      "step": 25330
+    },
+    {
+      "epoch": 0.9432171372205989,
+      "grad_norm": 9.012531388577552,
+      "learning_rate": 4.89200856696742e-09,
+      "logits/chosen": -4.0,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.4091,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -9.375,
+      "step": 25340
+    },
+    {
+      "epoch": 0.9435893618209228,
+      "grad_norm": 9.169900512268091,
+      "learning_rate": 4.828268070722674e-09,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1152.0,
+      "loss": 0.377,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.75,
+      "step": 25350
+    },
+    {
+      "epoch": 0.9439615864212466,
+      "grad_norm": 9.00945638081036,
+      "learning_rate": 4.764941506793324e-09,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3673,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 2.0625,
+      "rewards/rejected": -9.75,
+      "step": 25360
+    },
+    {
+      "epoch": 0.9443338110215704,
+      "grad_norm": 7.250554197229469,
+      "learning_rate": 4.702028982095901e-09,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.4058,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.4375,
+      "step": 25370
+    },
+    {
+      "epoch": 0.9447060356218943,
+      "grad_norm": 7.248450395478969,
+      "learning_rate": 4.639530602847913e-09,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.4013,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.4375,
+      "step": 25380
+    },
+    {
+      "epoch": 0.9450782602222181,
+      "grad_norm": 7.633810984411593,
+      "learning_rate": 4.57744647456762e-09,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3692,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.3125,
+      "step": 25390
+    },
+    {
+      "epoch": 0.9454504848225419,
+      "grad_norm": 8.336438292462582,
+      "learning_rate": 4.515776702073926e-09,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.394,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.0,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.75,
+      "step": 25400
+    },
+    {
+      "epoch": 0.9458227094228657,
+      "grad_norm": 8.285698119956546,
+      "learning_rate": 4.454521389486099e-09,
+      "logits/chosen": -3.640625,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3806,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -9.375,
+      "step": 25410
+    },
+    {
+      "epoch": 0.9461949340231895,
+      "grad_norm": 8.28420116918595,
+      "learning_rate": 4.393680640223718e-09,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3803,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.5625,
+      "step": 25420
+    },
+    {
+      "epoch": 0.9465671586235135,
+      "grad_norm": 8.465246251637536,
+      "learning_rate": 4.333254557006477e-09,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3704,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 2.015625,
+      "rewards/rejected": -9.5625,
+      "step": 25430
+    },
+    {
+      "epoch": 0.9469393832238373,
+      "grad_norm": 7.224155767896263,
+      "learning_rate": 4.273243241853852e-09,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3865,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.5,
+      "step": 25440
+    },
+    {
+      "epoch": 0.9473116078241611,
+      "grad_norm": 8.004760148096517,
+      "learning_rate": 4.213646796085157e-09,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3972,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.90625,
+      "rewards/rejected": -9.5625,
+      "step": 25450
+    },
+    {
+      "epoch": 0.947683832424485,
+      "grad_norm": 8.778924617314685,
+      "learning_rate": 4.154465320319184e-09,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3692,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.4375,
+      "step": 25460
+    },
+    {
+      "epoch": 0.9480560570248088,
+      "grad_norm": 6.82027651671271,
+      "learning_rate": 4.095698914474255e-09,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3616,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.921875,
+      "rewards/rejected": -9.5625,
+      "step": 25470
+    },
+    {
+      "epoch": 0.9484282816251326,
+      "grad_norm": 9.475251845700777,
+      "learning_rate": 4.037347677767728e-09,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.4001,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.8671875,
+      "rewards/rejected": -9.625,
+      "step": 25480
+    },
+    {
+      "epoch": 0.9488005062254564,
+      "grad_norm": 8.704643368113006,
+      "learning_rate": 3.97941170871613e-09,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3724,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -9.5625,
+      "step": 25490
+    },
+    {
+      "epoch": 0.9491727308257802,
+      "grad_norm": 8.138239216401093,
+      "learning_rate": 3.921891105134884e-09,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.84375,
+      "logps/chosen": -1004.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3761,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -9.3125,
+      "step": 25500
+    },
+    {
+      "epoch": 0.9495449554261041,
+      "grad_norm": 8.072424717729907,
+      "learning_rate": 3.864785964138112e-09,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3825,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.5625,
+      "step": 25510
+    },
+    {
+      "epoch": 0.949917180026428,
+      "grad_norm": 8.015566055455706,
+      "learning_rate": 3.80809638213847e-09,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3623,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.5,
+      "step": 25520
+    },
+    {
+      "epoch": 0.9502894046267518,
+      "grad_norm": 7.740809910212882,
+      "learning_rate": 3.751822454847037e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3675,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.8671875,
+      "rewards/rejected": -9.5,
+      "step": 25530
+    },
+    {
+      "epoch": 0.9506616292270756,
+      "grad_norm": 7.034224908861516,
+      "learning_rate": 3.6959642772730893e-09,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3863,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -8.0,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -9.5625,
+      "step": 25540
+    },
+    {
+      "epoch": 0.9510338538273995,
+      "grad_norm": 8.296773412644795,
+      "learning_rate": 3.6405219437240785e-09,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3685,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.375,
+      "step": 25550
+    },
+    {
+      "epoch": 0.9514060784277233,
+      "grad_norm": 7.6369514816938535,
+      "learning_rate": 3.5854955478052672e-09,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3957,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.6328125,
+      "rewards/rejected": -9.1875,
+      "step": 25560
+    },
+    {
+      "epoch": 0.9517783030280471,
+      "grad_norm": 7.495784635748868,
+      "learning_rate": 3.5308851824197296e-09,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.5,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3732,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.5,
+      "step": 25570
+    },
+    {
+      "epoch": 0.9521505276283709,
+      "grad_norm": 7.806614104980573,
+      "learning_rate": 3.476690939768101e-09,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3773,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.75,
+      "step": 25580
+    },
+    {
+      "epoch": 0.9525227522286948,
+      "grad_norm": 7.836814804627553,
+      "learning_rate": 3.4229129113484968e-09,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3901,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.625,
+      "step": 25590
+    },
+    {
+      "epoch": 0.9528949768290186,
+      "grad_norm": 6.980650345662951,
+      "learning_rate": 3.3695511879562877e-09,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.37,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.5625,
+      "step": 25600
+    },
+    {
+      "epoch": 0.9532672014293425,
+      "grad_norm": 8.900378271132563,
+      "learning_rate": 3.316605859684074e-09,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3992,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.375,
+      "step": 25610
+    },
+    {
+      "epoch": 0.9536394260296663,
+      "grad_norm": 7.901574362846449,
+      "learning_rate": 3.264077015921268e-09,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3797,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.3125,
+      "step": 25620
+    },
+    {
+      "epoch": 0.9540116506299902,
+      "grad_norm": 7.3257724612832416,
+      "learning_rate": 3.21196474535429e-09,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1160.0,
+      "loss": 0.3739,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 2.0,
+      "rewards/rejected": -9.875,
+      "step": 25630
+    },
+    {
+      "epoch": 0.954383875230314,
+      "grad_norm": 6.7874370008127345,
+      "learning_rate": 3.1602691359661758e-09,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3606,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.5,
+      "step": 25640
+    },
+    {
+      "epoch": 0.9547560998306378,
+      "grad_norm": 7.381654221756642,
+      "learning_rate": 3.108990275036444e-09,
+      "logits/chosen": -3.640625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3886,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.90625,
+      "rewards/rejected": -9.625,
+      "step": 25650
+    },
+    {
+      "epoch": 0.9551283244309616,
+      "grad_norm": 9.18251277547178,
+      "learning_rate": 3.0581282491411176e-09,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.75,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3855,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -8.0625,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.8125,
+      "step": 25660
+    },
+    {
+      "epoch": 0.9555005490312855,
+      "grad_norm": 7.317491219413723,
+      "learning_rate": 3.0076831441523677e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3794,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.625,
+      "step": 25670
+    },
+    {
+      "epoch": 0.9558727736316093,
+      "grad_norm": 8.822584395363911,
+      "learning_rate": 2.957655045238455e-09,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3923,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.5078125,
+      "rewards/rejected": -9.3125,
+      "step": 25680
+    },
+    {
+      "epoch": 0.9562449982319331,
+      "grad_norm": 9.52031695832083,
+      "learning_rate": 2.9080440368636773e-09,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.4156,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.6015625,
+      "rewards/rejected": -9.5625,
+      "step": 25690
+    },
+    {
+      "epoch": 0.956617222832257,
+      "grad_norm": 8.16522125589425,
+      "learning_rate": 2.8588502027881156e-09,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3717,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.640625,
+      "rewards/rejected": -9.25,
+      "step": 25700
+    },
+    {
+      "epoch": 0.9569894474325809,
+      "grad_norm": 9.69540881989094,
+      "learning_rate": 2.8100736260674442e-09,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3786,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.3125,
+      "step": 25710
+    },
+    {
+      "epoch": 0.9573616720329047,
+      "grad_norm": 6.74493207484038,
+      "learning_rate": 2.761714389052955e-09,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3839,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -9.3125,
+      "step": 25720
+    },
+    {
+      "epoch": 0.9577338966332285,
+      "grad_norm": 8.176889945201019,
+      "learning_rate": 2.713772573391282e-09,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3788,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.9765625,
+      "rewards/rejected": -9.625,
+      "step": 25730
+    },
+    {
+      "epoch": 0.9581061212335523,
+      "grad_norm": 8.062601603222628,
+      "learning_rate": 2.6662482600243452e-09,
+      "logits/chosen": -3.609375,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4082,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.4375,
+      "step": 25740
+    },
+    {
+      "epoch": 0.9584783458338761,
+      "grad_norm": 6.993259073569726,
+      "learning_rate": 2.619141529189184e-09,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3939,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.4375,
+      "step": 25750
+    },
+    {
+      "epoch": 0.9588505704342,
+      "grad_norm": 7.967946801585202,
+      "learning_rate": 2.572452460417762e-09,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3937,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.5,
+      "step": 25760
+    },
+    {
+      "epoch": 0.9592227950345238,
+      "grad_norm": 8.581037306013657,
+      "learning_rate": 2.526181132536942e-09,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1160.0,
+      "loss": 0.3659,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.5625,
+      "step": 25770
+    },
+    {
+      "epoch": 0.9595950196348476,
+      "grad_norm": 7.593190323321908,
+      "learning_rate": 2.480327623668288e-09,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3744,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.5,
+      "step": 25780
+    },
+    {
+      "epoch": 0.9599672442351715,
+      "grad_norm": 7.798552310304482,
+      "learning_rate": 2.434892011227929e-09,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -996.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3847,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.9453125,
+      "rewards/rejected": -9.75,
+      "step": 25790
+    },
+    {
+      "epoch": 0.9603394688354954,
+      "grad_norm": 10.116244172551605,
+      "learning_rate": 2.389874371926448e-09,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4119,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.53125,
+      "rewards/rejected": -9.3125,
+      "step": 25800
+    },
+    {
+      "epoch": 0.9607116934358192,
+      "grad_norm": 7.719807534308138,
+      "learning_rate": 2.345274781768769e-09,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3676,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.71875,
+      "rewards/rejected": -9.5,
+      "step": 25810
+    },
+    {
+      "epoch": 0.961083918036143,
+      "grad_norm": 7.738893376286991,
+      "learning_rate": 2.3010933160539927e-09,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.4013,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -9.625,
+      "step": 25820
+    },
+    {
+      "epoch": 0.9614561426364668,
+      "grad_norm": 8.963374675001797,
+      "learning_rate": 2.2573300493752835e-09,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3896,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.5546875,
+      "rewards/rejected": -9.375,
+      "step": 25830
+    },
+    {
+      "epoch": 0.9618283672367907,
+      "grad_norm": 7.226494130739676,
+      "learning_rate": 2.213985055619788e-09,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3785,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.375,
+      "step": 25840
+    },
+    {
+      "epoch": 0.9622005918371145,
+      "grad_norm": 8.417548661889851,
+      "learning_rate": 2.171058407968357e-09,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3951,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.4375,
+      "step": 25850
+    },
+    {
+      "epoch": 0.9625728164374383,
+      "grad_norm": 7.305723917301859,
+      "learning_rate": 2.128550178895655e-09,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -988.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.374,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.75,
+      "step": 25860
+    },
+    {
+      "epoch": 0.9629450410377622,
+      "grad_norm": 7.879032513305666,
+      "learning_rate": 2.086460440169857e-09,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3913,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.5,
+      "step": 25870
+    },
+    {
+      "epoch": 0.9633172656380861,
+      "grad_norm": 11.534620433635054,
+      "learning_rate": 2.044789262852592e-09,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4003,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": -9.4375,
+      "step": 25880
+    },
+    {
+      "epoch": 0.9636894902384099,
+      "grad_norm": 7.331810182404094,
+      "learning_rate": 2.003536717298804e-09,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.36,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.5,
+      "step": 25890
+    },
+    {
+      "epoch": 0.9640617148387337,
+      "grad_norm": 8.61588283703836,
+      "learning_rate": 1.9627028731566685e-09,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.4013,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.9296875,
+      "rewards/rejected": -9.5625,
+      "step": 25900
+    },
+    {
+      "epoch": 0.9644339394390575,
+      "grad_norm": 8.846961450421611,
+      "learning_rate": 1.922287799367456e-09,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -1000.0,
+      "logps/rejected": -1168.0,
+      "loss": 0.3828,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": -9.8125,
+      "step": 25910
+    },
+    {
+      "epoch": 0.9648061640393814,
+      "grad_norm": 6.995577685406074,
+      "learning_rate": 1.882291564165389e-09,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3566,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.9296875,
+      "rewards/rejected": -9.75,
+      "step": 25920
+    },
+    {
+      "epoch": 0.9651783886397052,
+      "grad_norm": 9.382213783387089,
+      "learning_rate": 1.8427142350775638e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.4013,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.84375,
+      "rewards/rejected": -9.75,
+      "step": 25930
+    },
+    {
+      "epoch": 0.965550613240029,
+      "grad_norm": 7.987110342805769,
+      "learning_rate": 1.803555878923807e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3901,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.4375,
+      "step": 25940
+    },
+    {
+      "epoch": 0.9659228378403528,
+      "grad_norm": 7.945906471987939,
+      "learning_rate": 1.7648165618166233e-09,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3913,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.6875,
+      "rewards/rejected": -9.5,
+      "step": 25950
+    },
+    {
+      "epoch": 0.9662950624406768,
+      "grad_norm": 7.025577573931613,
+      "learning_rate": 1.726496349160944e-09,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -920.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3416,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -7.40625,
+      "rewards/margins": 2.171875,
+      "rewards/rejected": -9.625,
+      "step": 25960
+    },
+    {
+      "epoch": 0.9666672870410006,
+      "grad_norm": 8.6813760592304,
+      "learning_rate": 1.688595305654239e-09,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3868,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.375,
+      "step": 25970
+    },
+    {
+      "epoch": 0.9670395116413244,
+      "grad_norm": 6.809003511614786,
+      "learning_rate": 1.651113495286155e-09,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3897,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.625,
+      "step": 25980
+    },
+    {
+      "epoch": 0.9674117362416482,
+      "grad_norm": 8.258772799954308,
+      "learning_rate": 1.6140509813386273e-09,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.453125,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3866,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.375,
+      "step": 25990
+    },
+    {
+      "epoch": 0.967783960841972,
+      "grad_norm": 8.085420707648852,
+      "learning_rate": 1.5774078263856571e-09,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3866,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.5,
+      "step": 26000
+    },
+    {
+      "epoch": 0.9681561854422959,
+      "grad_norm": 6.763752401290458,
+      "learning_rate": 1.5411840922931184e-09,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3653,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.875,
+      "rewards/rejected": -9.625,
+      "step": 26010
+    },
+    {
+      "epoch": 0.9685284100426197,
+      "grad_norm": 8.873395725235099,
+      "learning_rate": 1.5053798402189788e-09,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3893,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.375,
+      "step": 26020
+    },
+    {
+      "epoch": 0.9689006346429435,
+      "grad_norm": 7.638329481683679,
+      "learning_rate": 1.4699951306127722e-09,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -1004.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3621,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -8.0625,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.8125,
+      "step": 26030
+    },
+    {
+      "epoch": 0.9692728592432673,
+      "grad_norm": 8.548017444344138,
+      "learning_rate": 1.4350300232158218e-09,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3928,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.4375,
+      "step": 26040
+    },
+    {
+      "epoch": 0.9696450838435913,
+      "grad_norm": 8.345502480858306,
+      "learning_rate": 1.400484577060962e-09,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3772,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.5,
+      "step": 26050
+    },
+    {
+      "epoch": 0.9700173084439151,
+      "grad_norm": 6.655085833674345,
+      "learning_rate": 1.3663588504725098e-09,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3745,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.25,
+      "step": 26060
+    },
+    {
+      "epoch": 0.9703895330442389,
+      "grad_norm": 9.511087938778354,
+      "learning_rate": 1.3326529010662112e-09,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3809,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.9296875,
+      "rewards/rejected": -9.4375,
+      "step": 26070
+    },
+    {
+      "epoch": 0.9707617576445627,
+      "grad_norm": 8.245052236908169,
+      "learning_rate": 1.2993667857489898e-09,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.36,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": -9.625,
+      "step": 26080
+    },
+    {
+      "epoch": 0.9711339822448866,
+      "grad_norm": 7.044550269147712,
+      "learning_rate": 1.2665005607190581e-09,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3812,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.375,
+      "step": 26090
+    },
+    {
+      "epoch": 0.9715062068452104,
+      "grad_norm": 7.635425409795397,
+      "learning_rate": 1.2340542814655853e-09,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -992.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.368,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.671875,
+      "rewards/rejected": -9.5625,
+      "step": 26100
+    },
+    {
+      "epoch": 0.9718784314455342,
+      "grad_norm": 8.719924367688913,
+      "learning_rate": 1.2020280027688622e-09,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3911,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -9.375,
+      "step": 26110
+    },
+    {
+      "epoch": 0.972250656045858,
+      "grad_norm": 7.164402845771902,
+      "learning_rate": 1.1704217786999703e-09,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.46875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3701,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.5625,
+      "step": 26120
+    },
+    {
+      "epoch": 0.9726228806461819,
+      "grad_norm": 7.87262105810357,
+      "learning_rate": 1.139235662620891e-09,
+      "logits/chosen": -3.984375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -1004.0,
+      "logps/rejected": -1152.0,
+      "loss": 0.4072,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.0625,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.625,
+      "step": 26130
+    },
+    {
+      "epoch": 0.9729951052465058,
+      "grad_norm": 8.144870765894575,
+      "learning_rate": 1.1084697071842008e-09,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3756,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.375,
+      "step": 26140
+    },
+    {
+      "epoch": 0.9733673298468296,
+      "grad_norm": 6.937050152498703,
+      "learning_rate": 1.0781239643332384e-09,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1152.0,
+      "loss": 0.3759,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.6875,
+      "step": 26150
+    },
+    {
+      "epoch": 0.9737395544471534,
+      "grad_norm": 9.082579862245037,
+      "learning_rate": 1.0481984853017988e-09,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3746,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.609375,
+      "rewards/rejected": -9.375,
+      "step": 26160
+    },
+    {
+      "epoch": 0.9741117790474773,
+      "grad_norm": 8.27083146515157,
+      "learning_rate": 1.0186933206141612e-09,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3678,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.5,
+      "step": 26170
+    },
+    {
+      "epoch": 0.9744840036478011,
+      "grad_norm": 7.107041321925005,
+      "learning_rate": 9.89608520084978e-10,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -1008.0,
+      "logps/rejected": -1152.0,
+      "loss": 0.4007,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -8.0,
+      "rewards/margins": 1.8046875,
+      "rewards/rejected": -9.8125,
+      "step": 26180
+    },
+    {
+      "epoch": 0.9748562282481249,
+      "grad_norm": 8.50234389762464,
+      "learning_rate": 9.609441328191082e-10,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3862,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.5,
+      "step": 26190
+    },
+    {
+      "epoch": 0.9752284528484487,
+      "grad_norm": 7.991625998355685,
+      "learning_rate": 9.327002072117563e-10,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.421875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.388,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.5625,
+      "step": 26200
+    },
+    {
+      "epoch": 0.9756006774487725,
+      "grad_norm": 7.61531207710337,
+      "learning_rate": 9.048767909480837e-10,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3741,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.375,
+      "step": 26210
+    },
+    {
+      "epoch": 0.9759729020490965,
+      "grad_norm": 8.957474840452447,
+      "learning_rate": 8.774739310034585e-10,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3625,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.5625,
+      "step": 26220
+    },
+    {
+      "epoch": 0.9763451266494203,
+      "grad_norm": 7.349552090097092,
+      "learning_rate": 8.504916736430667e-10,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.53125,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.4034,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.9296875,
+      "rewards/rejected": -9.8125,
+      "step": 26230
+    },
+    {
+      "epoch": 0.9767173512497441,
+      "grad_norm": 7.213014655405053,
+      "learning_rate": 8.239300644220237e-10,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -1012.0,
+      "logps/rejected": -1160.0,
+      "loss": 0.3794,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.8515625,
+      "rewards/rejected": -9.75,
+      "step": 26240
+    },
+    {
+      "epoch": 0.977089575850068,
+      "grad_norm": 6.697929968556677,
+      "learning_rate": 7.977891481852906e-10,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3999,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.7109375,
+      "rewards/rejected": -9.6875,
+      "step": 26250
+    },
+    {
+      "epoch": 0.9774618004503918,
+      "grad_norm": 8.565907787681383,
+      "learning_rate": 7.720689690674798e-10,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.379,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.859375,
+      "rewards/rejected": -9.375,
+      "step": 26260
+    },
+    {
+      "epoch": 0.9778340250507156,
+      "grad_norm": 8.59511364501124,
+      "learning_rate": 7.467695704929666e-10,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3682,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.5625,
+      "step": 26270
+    },
+    {
+      "epoch": 0.9782062496510394,
+      "grad_norm": 8.250858173852171,
+      "learning_rate": 7.218909951755836e-10,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.75,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.4063,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.375,
+      "step": 26280
+    },
+    {
+      "epoch": 0.9785784742513632,
+      "grad_norm": 8.450769404262243,
+      "learning_rate": 6.974332851187592e-10,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3986,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.7890625,
+      "rewards/rejected": -9.5,
+      "step": 26290
+    },
+    {
+      "epoch": 0.9789506988516871,
+      "grad_norm": 7.956833651297262,
+      "learning_rate": 6.733964816153237e-10,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -988.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3676,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -8.0,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.6875,
+      "step": 26300
+    },
+    {
+      "epoch": 0.979322923452011,
+      "grad_norm": 8.702286729175416,
+      "learning_rate": 6.497806252474814e-10,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -928.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3837,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.25,
+      "step": 26310
+    },
+    {
+      "epoch": 0.9796951480523348,
+      "grad_norm": 7.602147763743852,
+      "learning_rate": 6.265857558867271e-10,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.40625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.369,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.65625,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.375,
+      "step": 26320
+    },
+    {
+      "epoch": 0.9800673726526586,
+      "grad_norm": 7.003905768878151,
+      "learning_rate": 6.038119126938191e-10,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3887,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.375,
+      "step": 26330
+    },
+    {
+      "epoch": 0.9804395972529825,
+      "grad_norm": 8.172243776991232,
+      "learning_rate": 5.814591341186392e-10,
+      "logits/chosen": -3.734375,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.382,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.6953125,
+      "rewards/rejected": -9.5,
+      "step": 26340
+    },
+    {
+      "epoch": 0.9808118218533063,
+      "grad_norm": 8.412479608639437,
+      "learning_rate": 5.595274579002219e-10,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3821,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.828125,
+      "rewards/rejected": -9.5625,
+      "step": 26350
+    },
+    {
+      "epoch": 0.9811840464536301,
+      "grad_norm": 9.828988978201002,
+      "learning_rate": 5.380169210666141e-10,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.4118,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.625,
+      "step": 26360
+    },
+    {
+      "epoch": 0.9815562710539539,
+      "grad_norm": 8.205889761169583,
+      "learning_rate": 5.169275599347933e-10,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -1008.0,
+      "logps/rejected": -1152.0,
+      "loss": 0.3924,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -8.0,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.6875,
+      "step": 26370
+    },
+    {
+      "epoch": 0.9819284956542778,
+      "grad_norm": 7.088840004209817,
+      "learning_rate": 4.96259410110722e-10,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3533,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.78125,
+      "rewards/rejected": -9.5625,
+      "step": 26380
+    },
+    {
+      "epoch": 0.9823007202546016,
+      "grad_norm": 9.365059660968761,
+      "learning_rate": 4.760125064891818e-10,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -1000.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3741,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": -9.6875,
+      "step": 26390
+    },
+    {
+      "epoch": 0.9826729448549255,
+      "grad_norm": 8.742787351403228,
+      "learning_rate": 4.561868832537452e-10,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.385,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.9453125,
+      "rewards/rejected": -9.625,
+      "step": 26400
+    },
+    {
+      "epoch": 0.9830451694552493,
+      "grad_norm": 8.124395148854394,
+      "learning_rate": 4.367825738767206e-10,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -924.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3806,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.5,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.1875,
+      "step": 26410
+    },
+    {
+      "epoch": 0.9834173940555732,
+      "grad_norm": 6.030875182374471,
+      "learning_rate": 4.1779961111915176e-10,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3741,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -9.3125,
+      "step": 26420
+    },
+    {
+      "epoch": 0.983789618655897,
+      "grad_norm": 7.203717583031877,
+      "learning_rate": 3.9923802703059616e-10,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.78125,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3748,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": -9.5,
+      "step": 26430
+    },
+    {
+      "epoch": 0.9841618432562208,
+      "grad_norm": 7.372912886531599,
+      "learning_rate": 3.810978529493192e-10,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3724,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.625,
+      "step": 26440
+    },
+    {
+      "epoch": 0.9845340678565446,
+      "grad_norm": 9.426056803322183,
+      "learning_rate": 3.633791195019886e-10,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3893,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.625,
+      "rewards/rejected": -9.4375,
+      "step": 26450
+    },
+    {
+      "epoch": 0.9849062924568684,
+      "grad_norm": 8.294701337525558,
+      "learning_rate": 3.4608185660378595e-10,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -944.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3694,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.625,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.4375,
+      "step": 26460
+    },
+    {
+      "epoch": 0.9852785170571923,
+      "grad_norm": 8.926083775976789,
+      "learning_rate": 3.292060934582952e-10,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.4062,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.59375,
+      "rewards/rejected": -9.3125,
+      "step": 26470
+    },
+    {
+      "epoch": 0.9856507416575161,
+      "grad_norm": 7.703145819604227,
+      "learning_rate": 3.127518585575306e-10,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3828,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.890625,
+      "rewards/rejected": -9.625,
+      "step": 26480
+    },
+    {
+      "epoch": 0.98602296625784,
+      "grad_norm": 8.101749726553084,
+      "learning_rate": 2.9671917968171476e-10,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -992.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.379,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.96875,
+      "rewards/margins": 1.65625,
+      "rewards/rejected": -9.625,
+      "step": 26490
+    },
+    {
+      "epoch": 0.9863951908581639,
+      "grad_norm": 7.395431479810991,
+      "learning_rate": 2.8110808389944506e-10,
+      "logits/chosen": -3.9375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -964.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3849,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.9375,
+      "rewards/rejected": -9.625,
+      "step": 26500
+    },
+    {
+      "epoch": 0.9867674154584877,
+      "grad_norm": 7.100411645891299,
+      "learning_rate": 2.6591859756749934e-10,
+      "logits/chosen": -3.90625,
+      "logits/rejected": -3.828125,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.4007,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.4375,
+      "step": 26510
+    },
+    {
+      "epoch": 0.9871396400588115,
+      "grad_norm": 9.680375208847861,
+      "learning_rate": 2.511507463307805e-10,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3839,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.734375,
+      "rewards/rejected": -9.5625,
+      "step": 26520
+    },
+    {
+      "epoch": 0.9875118646591353,
+      "grad_norm": 7.258127167075642,
+      "learning_rate": 2.3680455512242736e-10,
+      "logits/chosen": -3.71875,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3927,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.7421875,
+      "rewards/rejected": -9.5,
+      "step": 26530
+    },
+    {
+      "epoch": 0.9878840892594591,
+      "grad_norm": 7.101665376958755,
+      "learning_rate": 2.2288004816364836e-10,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -1000.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.378,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.96875,
+      "rewards/margins": 1.5234375,
+      "rewards/rejected": -9.5,
+      "step": 26540
+    },
+    {
+      "epoch": 0.988256313859783,
+      "grad_norm": 6.759720674510336,
+      "learning_rate": 2.0937724896369358e-10,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3864,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.7265625,
+      "rewards/rejected": -9.625,
+      "step": 26550
+    },
+    {
+      "epoch": 0.9886285384601068,
+      "grad_norm": 8.087977294537376,
+      "learning_rate": 1.9629618031977159e-10,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.640625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3668,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.75,
+      "rewards/rejected": -9.4375,
+      "step": 26560
+    },
+    {
+      "epoch": 0.9890007630604306,
+      "grad_norm": 7.0346301508083755,
+      "learning_rate": 1.8363686431718817e-10,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -980.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.368,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.9375,
+      "rewards/margins": 1.6796875,
+      "rewards/rejected": -9.5625,
+      "step": 26570
+    },
+    {
+      "epoch": 0.9893729876607545,
+      "grad_norm": 9.18932553593361,
+      "learning_rate": 1.713993223290966e-10,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1144.0,
+      "loss": 0.3683,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.796875,
+      "rewards/rejected": -9.6875,
+      "step": 26580
+    },
+    {
+      "epoch": 0.9897452122610784,
+      "grad_norm": 8.17021118499696,
+      "learning_rate": 1.5958357501658082e-10,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.515625,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3604,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.8984375,
+      "rewards/rejected": -9.5625,
+      "step": 26590
+    },
+    {
+      "epoch": 0.9901174368614022,
+      "grad_norm": 7.8419687425859514,
+      "learning_rate": 1.4818964232859997e-10,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.765625,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3609,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.9140625,
+      "rewards/rejected": -9.5,
+      "step": 26600
+    },
+    {
+      "epoch": 0.990489661461726,
+      "grad_norm": 7.7097494221947125,
+      "learning_rate": 1.3721754350196068e-10,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -932.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.3736,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.5625,
+      "rewards/margins": 1.7734375,
+      "rewards/rejected": -9.3125,
+      "step": 26610
+    },
+    {
+      "epoch": 0.9908618860620498,
+      "grad_norm": 7.307867888252128,
+      "learning_rate": 1.2666729706120593e-10,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -988.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.3824,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -9.375,
+      "step": 26620
+    },
+    {
+      "epoch": 0.9912341106623737,
+      "grad_norm": 8.737926345280744,
+      "learning_rate": 1.1653892081875393e-10,
+      "logits/chosen": -3.796875,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1072.0,
+      "loss": 0.4016,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -9.1875,
+      "step": 26630
+    },
+    {
+      "epoch": 0.9916063352626975,
+      "grad_norm": 7.361966512705138,
+      "learning_rate": 1.0683243187467605e-10,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3613,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.703125,
+      "rewards/rejected": -9.4375,
+      "step": 26640
+    },
+    {
+      "epoch": 0.9919785598630213,
+      "grad_norm": 8.367082026222384,
+      "learning_rate": 9.754784661680781e-11,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3808,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -9.5625,
+      "step": 26650
+    },
+    {
+      "epoch": 0.9923507844633452,
+      "grad_norm": 7.587400315305485,
+      "learning_rate": 8.86851807206379e-11,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.484375,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3781,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.78125,
+      "rewards/margins": 1.6171875,
+      "rewards/rejected": -9.4375,
+      "step": 26660
+    },
+    {
+      "epoch": 0.9927230090636691,
+      "grad_norm": 8.357062950161398,
+      "learning_rate": 8.024444914933592e-11,
+      "logits/chosen": -3.6875,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -936.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3817,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -7.46875,
+      "rewards/margins": 2.03125,
+      "rewards/rejected": -9.5,
+      "step": 26670
+    },
+    {
+      "epoch": 0.9930952336639929,
+      "grad_norm": 6.956484332419937,
+      "learning_rate": 7.222566615369685e-11,
+      "logits/chosen": -3.703125,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1080.0,
+      "loss": 0.3831,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.4765625,
+      "rewards/rejected": -9.1875,
+      "step": 26680
+    },
+    {
+      "epoch": 0.9934674582643167,
+      "grad_norm": 7.987048267727548,
+      "learning_rate": 6.462884527208557e-11,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.71875,
+      "logps/chosen": -960.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3818,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -9.25,
+      "step": 26690
+    },
+    {
+      "epoch": 0.9938396828646405,
+      "grad_norm": 8.219012769301218,
+      "learning_rate": 5.745399933054784e-11,
+      "logits/chosen": -3.890625,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3899,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -9.375,
+      "step": 26700
+    },
+    {
+      "epoch": 0.9942119074649644,
+      "grad_norm": 7.202948773839784,
+      "learning_rate": 5.0701140442588333e-11,
+      "logits/chosen": -3.78125,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3624,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.75,
+      "rewards/margins": 1.7578125,
+      "rewards/rejected": -9.5,
+      "step": 26710
+    },
+    {
+      "epoch": 0.9945841320652882,
+      "grad_norm": 10.050238710877242,
+      "learning_rate": 4.437028000933707e-11,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.734375,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3961,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.6484375,
+      "rewards/rejected": -9.375,
+      "step": 26720
+    },
+    {
+      "epoch": 0.994956356665612,
+      "grad_norm": 8.338915990686202,
+      "learning_rate": 3.846142871938296e-11,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.625,
+      "logps/chosen": -948.0,
+      "logps/rejected": -1088.0,
+      "loss": 0.4126,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.5,
+      "rewards/rejected": -9.1875,
+      "step": 26730
+    },
+    {
+      "epoch": 0.9953285812659358,
+      "grad_norm": 7.43525476756313,
+      "learning_rate": 3.2974596548884795e-11,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3608,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.6875,
+      "rewards/margins": 1.8359375,
+      "rewards/rejected": -9.5,
+      "step": 26740
+    },
+    {
+      "epoch": 0.9957008058662598,
+      "grad_norm": 7.504889509854151,
+      "learning_rate": 2.790979276143246e-11,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.703125,
+      "logps/chosen": -988.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3774,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.578125,
+      "rewards/rejected": -9.5,
+      "step": 26750
+    },
+    {
+      "epoch": 0.9960730304665836,
+      "grad_norm": 7.8583811465473685,
+      "learning_rate": 2.3267025908130232e-11,
+      "logits/chosen": -3.75,
+      "logits/rejected": -3.609375,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.389,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -7.53125,
+      "rewards/margins": 1.9765625,
+      "rewards/rejected": -9.5,
+      "step": 26760
+    },
+    {
+      "epoch": 0.9964452550669074,
+      "grad_norm": 7.737852756401967,
+      "learning_rate": 1.9046303827569e-11,
+      "logits/chosen": -3.859375,
+      "logits/rejected": -3.59375,
+      "logps/chosen": -940.0,
+      "logps/rejected": -1104.0,
+      "loss": 0.366,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.8828125,
+      "rewards/rejected": -9.5,
+      "step": 26770
+    },
+    {
+      "epoch": 0.9968174796672312,
+      "grad_norm": 7.547117337051925,
+      "learning_rate": 1.524763364565973e-11,
+      "logits/chosen": -3.84375,
+      "logits/rejected": -3.546875,
+      "logps/chosen": -972.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.3758,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.625,
+      "step": 26780
+    },
+    {
+      "epoch": 0.997189704267555,
+      "grad_norm": 7.564753887340131,
+      "learning_rate": 1.1871021775911039e-11,
+      "logits/chosen": -3.875,
+      "logits/rejected": -3.65625,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1136.0,
+      "loss": 0.4028,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -7.8125,
+      "rewards/margins": 1.8203125,
+      "rewards/rejected": -9.625,
+      "step": 26790
+    },
+    {
+      "epoch": 0.9975619288678789,
+      "grad_norm": 7.1182855851862055,
+      "learning_rate": 8.916473919151624e-12,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.5625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1120.0,
+      "loss": 0.3697,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -7.71875,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.5,
+      "step": 26800
+    },
+    {
+      "epoch": 0.9979341534682027,
+      "grad_norm": 6.857767271478771,
+      "learning_rate": 6.38399506364129e-12,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.578125,
+      "logps/chosen": -956.0,
+      "logps/rejected": -1128.0,
+      "loss": 0.3677,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.765625,
+      "rewards/rejected": -9.625,
+      "step": 26810
+    },
+    {
+      "epoch": 0.9983063780685265,
+      "grad_norm": 8.576975690718706,
+      "learning_rate": 4.27358948507095e-12,
+      "logits/chosen": -3.671875,
+      "logits/rejected": -3.6875,
+      "logps/chosen": -976.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3854,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -7.90625,
+      "rewards/margins": 1.5625,
+      "rewards/rejected": -9.4375,
+      "step": 26820
+    },
+    {
+      "epoch": 0.9986786026688503,
+      "grad_norm": 7.790291258127137,
+      "learning_rate": 2.5852607465071118e-12,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.671875,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3911,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -7.875,
+      "rewards/margins": 1.5859375,
+      "rewards/rejected": -9.4375,
+      "step": 26830
+    },
+    {
+      "epoch": 0.9990508272691743,
+      "grad_norm": 8.0689377246478,
+      "learning_rate": 1.3190116984196365e-12,
+      "logits/chosen": -3.921875,
+      "logits/rejected": -3.75,
+      "logps/chosen": -968.0,
+      "logps/rejected": -1112.0,
+      "loss": 0.3842,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -7.84375,
+      "rewards/margins": 1.6640625,
+      "rewards/rejected": -9.5,
+      "step": 26840
+    },
+    {
+      "epoch": 0.9994230518694981,
+      "grad_norm": 9.268939312547884,
+      "learning_rate": 4.748444786539796e-13,
+      "logits/chosen": -3.828125,
+      "logits/rejected": -3.8125,
+      "logps/chosen": -984.0,
+      "logps/rejected": -1168.0,
+      "loss": 0.3913,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -7.96875,
+      "rewards/margins": 1.8125,
+      "rewards/rejected": -9.8125,
+      "step": 26850
+    },
+    {
+      "epoch": 0.9997952764698219,
+      "grad_norm": 8.988014038511007,
+      "learning_rate": 5.2760512486704765e-14,
+      "logits/chosen": -3.8125,
+      "logits/rejected": -3.625,
+      "logps/chosen": -952.0,
+      "logps/rejected": -1096.0,
+      "loss": 0.3867,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -7.59375,
+      "rewards/margins": 1.546875,
+      "rewards/rejected": -9.125,
+      "step": 26860
+    },
+    {
+      "epoch": 0.9999813887699838,
+      "step": 26865,
+      "total_flos": 0.0,
+      "train_loss": 0.0,
+      "train_runtime": 18.8042,
+      "train_samples_per_second": 182872.522,
+      "train_steps_per_second": 1428.667
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 26865,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 5000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}