{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.2548582351067219,
  "eval_steps": 500,
  "global_step": 600,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.002123818625889349,
      "grad_norm": 0.2956613302230835,
      "learning_rate": 4.99996440845116e-06,
      "logits/chosen": -1.6179815530776978,
      "logits/rejected": -1.6184433698654175,
      "logps/chosen": -0.3400000035762787,
      "logps/rejected": -0.4482549726963043,
      "loss": 0.3949,
      "num_input_tokens_seen": 882592,
      "odds_ratio_loss": 0.5491451621055603,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.03400000184774399,
      "rewards/margins": 0.010825497098267078,
      "rewards/rejected": -0.04482549428939819,
      "sft_loss": 0.3400000035762787,
      "step": 5,
      "train_runtime": 926.9801,
      "train_tokens_per_second": 952.115
    },
    {
      "epoch": 0.004247637251778698,
      "grad_norm": 0.14733675122261047,
      "learning_rate": 4.9998198195208405e-06,
      "logits/chosen": -1.6484524011611938,
      "logits/rejected": -1.6493781805038452,
      "logps/chosen": -0.269551157951355,
      "logps/rejected": -0.383524626493454,
      "loss": 0.3223,
      "num_input_tokens_seen": 1645664,
      "odds_ratio_loss": 0.5276888608932495,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.026955116540193558,
      "rewards/margins": 0.011397343128919601,
      "rewards/rejected": -0.03835245966911316,
      "sft_loss": 0.269551157951355,
      "step": 10,
      "train_runtime": 1644.9794,
      "train_tokens_per_second": 1000.416
    },
    {
      "epoch": 0.0063714558776680474,
      "grad_norm": 0.2993604242801666,
      "learning_rate": 4.999564015164962e-06,
      "logits/chosen": -1.663969874382019,
      "logits/rejected": -1.6640106439590454,
      "logps/chosen": -0.28227946162223816,
      "logps/rejected": -0.3938451409339905,
      "loss": 0.3343,
      "num_input_tokens_seen": 2413232,
      "odds_ratio_loss": 0.5201923251152039,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.028227943927049637,
      "rewards/margins": 0.011156569235026836,
      "rewards/rejected": -0.03938451409339905,
      "sft_loss": 0.28227946162223816,
      "step": 15,
      "train_runtime": 2389.6145,
      "train_tokens_per_second": 1009.883
    },
    {
      "epoch": 0.008495274503557396,
      "grad_norm": 0.3080098032951355,
      "learning_rate": 4.999197006764101e-06,
      "logits/chosen": -1.6573028564453125,
      "logits/rejected": -1.6576881408691406,
      "logps/chosen": -0.2723864018917084,
      "logps/rejected": -0.4457150399684906,
      "loss": 0.3192,
      "num_input_tokens_seen": 3078144,
      "odds_ratio_loss": 0.4678315222263336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.027238640934228897,
      "rewards/margins": 0.01733286678791046,
      "rewards/rejected": -0.04457150772213936,
      "sft_loss": 0.2723864018917084,
      "step": 20,
      "train_runtime": 2958.789,
      "train_tokens_per_second": 1040.339
    },
    {
      "epoch": 0.010619093129446745,
      "grad_norm": 0.39248254895210266,
      "learning_rate": 4.9987188106462314e-06,
      "logits/chosen": -1.6139488220214844,
      "logits/rejected": -1.6125049591064453,
      "logps/chosen": -0.2793450951576233,
      "logps/rejected": -0.4063512980937958,
      "loss": 0.3303,
      "num_input_tokens_seen": 3873104,
      "odds_ratio_loss": 0.5098921656608582,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.02793451026082039,
      "rewards/margins": 0.012700623832643032,
      "rewards/rejected": -0.040635135024785995,
      "sft_loss": 0.2793450951576233,
      "step": 25,
      "train_runtime": 3762.846,
      "train_tokens_per_second": 1029.302
    },
    {
      "epoch": 0.012742911755336095,
      "grad_norm": 0.32270482182502747,
      "learning_rate": 4.998129448086002e-06,
      "logits/chosen": -1.6860077381134033,
      "logits/rejected": -1.6875889301300049,
      "logps/chosen": -0.2858825922012329,
      "logps/rejected": -0.4011116027832031,
      "loss": 0.3381,
      "num_input_tokens_seen": 4767296,
      "odds_ratio_loss": 0.522192656993866,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -0.02858825959265232,
      "rewards/margins": 0.01152290403842926,
      "rewards/rejected": -0.04011116549372673,
      "sft_loss": 0.2858825922012329,
      "step": 30,
      "train_runtime": 4649.677,
      "train_tokens_per_second": 1025.296
    },
    {
      "epoch": 0.014866730381225443,
      "grad_norm": 0.14581075310707092,
      "learning_rate": 4.997428945303785e-06,
      "logits/chosen": -1.71160888671875,
      "logits/rejected": -1.7108162641525269,
      "logps/chosen": -0.26841896772384644,
      "logps/rejected": -0.39266031980514526,
      "loss": 0.3182,
      "num_input_tokens_seen": 5641760,
      "odds_ratio_loss": 0.49825096130371094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.026841897517442703,
      "rewards/margins": 0.012424135580658913,
      "rewards/rejected": -0.03926603123545647,
      "sft_loss": 0.26841896772384644,
      "step": 35,
      "train_runtime": 5539.828,
      "train_tokens_per_second": 1018.4
    },
    {
      "epoch": 0.016990549007114792,
      "grad_norm": 0.18738733232021332,
      "learning_rate": 4.996617333464512e-06,
      "logits/chosen": -1.6659491062164307,
      "logits/rejected": -1.6651753187179565,
      "logps/chosen": -0.23961809277534485,
      "logps/rejected": -0.3842446804046631,
      "loss": 0.2859,
      "num_input_tokens_seen": 6314672,
      "odds_ratio_loss": 0.4625028669834137,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.023961810395121574,
      "rewards/margins": 0.014462659135460854,
      "rewards/rejected": -0.03842446953058243,
      "sft_loss": 0.23961809277534485,
      "step": 40,
      "train_runtime": 6146.4803,
      "train_tokens_per_second": 1027.364
    },
    {
      "epoch": 0.019114367633004142,
      "grad_norm": 0.31590959429740906,
      "learning_rate": 4.995694648676288e-06,
      "logits/chosen": -1.6645311117172241,
      "logits/rejected": -1.6616137027740479,
      "logps/chosen": -0.26000967621803284,
      "logps/rejected": -0.37502923607826233,
      "loss": 0.3112,
      "num_input_tokens_seen": 7113840,
      "odds_ratio_loss": 0.5120108723640442,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.026000967249274254,
      "rewards/margins": 0.011501955799758434,
      "rewards/rejected": -0.03750292584300041,
      "sft_loss": 0.26000967621803284,
      "step": 45,
      "train_runtime": 6944.7438,
      "train_tokens_per_second": 1024.349
    },
    {
      "epoch": 0.02123818625889349,
      "grad_norm": 0.28091415762901306,
      "learning_rate": 4.994660931988786e-06,
      "logits/chosen": -1.7028652429580688,
      "logits/rejected": -1.7032212018966675,
      "logps/chosen": -0.25074753165245056,
      "logps/rejected": -0.3659655451774597,
      "loss": 0.3014,
      "num_input_tokens_seen": 7859776,
      "odds_ratio_loss": 0.50666743516922,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.025074753910303116,
      "rewards/margins": 0.011521801352500916,
      "rewards/rejected": -0.03659655526280403,
      "sft_loss": 0.25074753165245056,
      "step": 50,
      "train_runtime": 7660.6272,
      "train_tokens_per_second": 1025.996
    },
    {
      "epoch": 0.02336200488478284,
      "grad_norm": 0.18218040466308594,
      "learning_rate": 4.993516229391414e-06,
      "logits/chosen": -1.6127599477767944,
      "logits/rejected": -1.6123024225234985,
      "logps/chosen": -0.22657275199890137,
      "logps/rejected": -0.38914796710014343,
      "loss": 0.271,
      "num_input_tokens_seen": 8517696,
      "odds_ratio_loss": 0.4439874291419983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.022657275199890137,
      "rewards/margins": 0.016257524490356445,
      "rewards/rejected": -0.03891479969024658,
      "sft_loss": 0.22657275199890137,
      "step": 55,
      "train_runtime": 8258.8771,
      "train_tokens_per_second": 1031.338
    },
    {
      "epoch": 0.02548582351067219,
      "grad_norm": 0.232417494058609,
      "learning_rate": 4.99226059181128e-06,
      "logits/chosen": -1.6862449645996094,
      "logits/rejected": -1.686137080192566,
      "logps/chosen": -0.21866658329963684,
      "logps/rejected": -0.3603269159793854,
      "loss": 0.265,
      "num_input_tokens_seen": 9308976,
      "odds_ratio_loss": 0.46296849846839905,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.021866658702492714,
      "rewards/margins": 0.014166036620736122,
      "rewards/rejected": -0.03603269159793854,
      "sft_loss": 0.21866658329963684,
      "step": 60,
      "train_runtime": 9018.5713,
      "train_tokens_per_second": 1032.201
    },
    {
      "epoch": 0.027609642136561537,
      "grad_norm": 0.19058002531528473,
      "learning_rate": 4.9908940751109145e-06,
      "logits/chosen": -1.6789970397949219,
      "logits/rejected": -1.6768442392349243,
      "logps/chosen": -0.24218253791332245,
      "logps/rejected": -0.3783145844936371,
      "loss": 0.2913,
      "num_input_tokens_seen": 10182832,
      "odds_ratio_loss": 0.49102774262428284,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -0.024218251928687096,
      "rewards/margins": 0.013613204471766949,
      "rewards/rejected": -0.03783145546913147,
      "sft_loss": 0.24218253791332245,
      "step": 65,
      "train_runtime": 9890.8858,
      "train_tokens_per_second": 1029.517
    },
    {
      "epoch": 0.029733460762450887,
      "grad_norm": 0.20720285177230835,
      "learning_rate": 4.989416740085796e-06,
      "logits/chosen": -1.5963817834854126,
      "logits/rejected": -1.5952785015106201,
      "logps/chosen": -0.21330037713050842,
      "logps/rejected": -0.36663225293159485,
      "loss": 0.2584,
      "num_input_tokens_seen": 10828992,
      "odds_ratio_loss": 0.451129287481308,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.021330038085579872,
      "rewards/margins": 0.015333187766373158,
      "rewards/rejected": -0.036663226783275604,
      "sft_loss": 0.21330037713050842,
      "step": 70,
      "train_runtime": 10456.9744,
      "train_tokens_per_second": 1035.576
    },
    {
      "epoch": 0.03185727938834024,
      "grad_norm": 0.260476291179657,
      "learning_rate": 4.987828652461637e-06,
      "logits/chosen": -1.6634387969970703,
      "logits/rejected": -1.6630572080612183,
      "logps/chosen": -0.2052687704563141,
      "logps/rejected": -0.34149646759033203,
      "loss": 0.2518,
      "num_input_tokens_seen": 11523968,
      "odds_ratio_loss": 0.4651528000831604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.020526880398392677,
      "rewards/margins": 0.0136227672919631,
      "rewards/rejected": -0.0341496467590332,
      "sft_loss": 0.2052687704563141,
      "step": 75,
      "train_runtime": 11059.2326,
      "train_tokens_per_second": 1042.022
    },
    {
      "epoch": 0.033981098014229584,
      "grad_norm": 0.24337738752365112,
      "learning_rate": 4.9861298828914685e-06,
      "logits/chosen": -1.719187617301941,
      "logits/rejected": -1.7172062397003174,
      "logps/chosen": -0.22513428330421448,
      "logps/rejected": -0.3598342537879944,
      "loss": 0.2708,
      "num_input_tokens_seen": 12241712,
      "odds_ratio_loss": 0.45616158843040466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.022513430565595627,
      "rewards/margins": 0.013469995930790901,
      "rewards/rejected": -0.03598342463374138,
      "sft_loss": 0.22513428330421448,
      "step": 80,
      "train_runtime": 11705.8865,
      "train_tokens_per_second": 1045.774
    },
    {
      "epoch": 0.03610491664011893,
      "grad_norm": 0.35554245114326477,
      "learning_rate": 4.984320506952487e-06,
      "logits/chosen": -1.6329272985458374,
      "logits/rejected": -1.6302893161773682,
      "logps/chosen": -0.22967509925365448,
      "logps/rejected": -0.3732440769672394,
      "loss": 0.2748,
      "num_input_tokens_seen": 12878720,
      "odds_ratio_loss": 0.45122718811035156,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.022967511788010597,
      "rewards/margins": 0.01435689628124237,
      "rewards/rejected": -0.03732440993189812,
      "sft_loss": 0.22967509925365448,
      "step": 85,
      "train_runtime": 12262.6005,
      "train_tokens_per_second": 1050.244
    },
    {
      "epoch": 0.038228735266008285,
      "grad_norm": 0.22221030294895172,
      "learning_rate": 4.982400605142702e-06,
      "logits/chosen": -1.7374569177627563,
      "logits/rejected": -1.7368042469024658,
      "logps/chosen": -0.2531585395336151,
      "logps/rejected": -0.3707667887210846,
      "loss": 0.3014,
      "num_input_tokens_seen": 13763456,
      "odds_ratio_loss": 0.4821968972682953,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.02531585469841957,
      "rewards/margins": 0.011760825291275978,
      "rewards/rejected": -0.0370766818523407,
      "sft_loss": 0.2531585395336151,
      "step": 90,
      "train_runtime": 13167.1088,
      "train_tokens_per_second": 1045.291
    },
    {
      "epoch": 0.04035255389189763,
      "grad_norm": 0.19162172079086304,
      "learning_rate": 4.980370262877348e-06,
      "logits/chosen": -1.6875009536743164,
      "logits/rejected": -1.6881672143936157,
      "logps/chosen": -0.27708110213279724,
      "logps/rejected": -0.40693196654319763,
      "loss": 0.3269,
      "num_input_tokens_seen": 14663824,
      "odds_ratio_loss": 0.4980649948120117,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.027708111330866814,
      "rewards/margins": 0.012985085137188435,
      "rewards/rejected": -0.040693193674087524,
      "sft_loss": 0.27708110213279724,
      "step": 95,
      "train_runtime": 14096.3295,
      "train_tokens_per_second": 1040.258
    },
    {
      "epoch": 0.04247637251778698,
      "grad_norm": 0.268886536359787,
      "learning_rate": 4.978229570485085e-06,
      "logits/chosen": -1.6658505201339722,
      "logits/rejected": -1.6657627820968628,
      "logps/chosen": -0.25221019983291626,
      "logps/rejected": -0.3684101104736328,
      "loss": 0.303,
      "num_input_tokens_seen": 15450672,
      "odds_ratio_loss": 0.5082995295524597,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.025221019983291626,
      "rewards/margins": 0.011619992554187775,
      "rewards/rejected": -0.0368410162627697,
      "sft_loss": 0.25221019983291626,
      "step": 100,
      "train_runtime": 14873.5745,
      "train_tokens_per_second": 1038.8
    },
    {
      "epoch": 0.04460019114367633,
      "grad_norm": 0.2696066200733185,
      "learning_rate": 4.975978623203986e-06,
      "logits/chosen": -1.6801649332046509,
      "logits/rejected": -1.6801986694335938,
      "logps/chosen": -0.28334546089172363,
      "logps/rejected": -0.380205363035202,
      "loss": 0.3388,
      "num_input_tokens_seen": 16393248,
      "odds_ratio_loss": 0.5550203323364258,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -0.028334546834230423,
      "rewards/margins": 0.009685988537967205,
      "rewards/rejected": -0.038020532578229904,
      "sft_loss": 0.28334546089172363,
      "step": 105,
      "train_runtime": 15896.301,
      "train_tokens_per_second": 1031.262
    },
    {
      "epoch": 0.04672400976956568,
      "grad_norm": 0.17821939289569855,
      "learning_rate": 4.97361752117729e-06,
      "logits/chosen": -1.7047245502471924,
      "logits/rejected": -1.7054890394210815,
      "logps/chosen": -0.24090777337551117,
      "logps/rejected": -0.3428797423839569,
      "loss": 0.2919,
      "num_input_tokens_seen": 17278976,
      "odds_ratio_loss": 0.5095131397247314,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.024090778082609177,
      "rewards/margins": 0.01019719522446394,
      "rewards/rejected": -0.03428797051310539,
      "sft_loss": 0.24090777337551117,
      "step": 110,
      "train_runtime": 16808.2548,
      "train_tokens_per_second": 1028.005
    },
    {
      "epoch": 0.048847828395455026,
      "grad_norm": 0.3716052770614624,
      "learning_rate": 4.971146369448957e-06,
      "logits/chosen": -1.6462633609771729,
      "logits/rejected": -1.6481584310531616,
      "logps/chosen": -0.2574033737182617,
      "logps/rejected": -0.36456406116485596,
      "loss": 0.3092,
      "num_input_tokens_seen": 18180752,
      "odds_ratio_loss": 0.5177726149559021,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02574033848941326,
      "rewards/margins": 0.010716068558394909,
      "rewards/rejected": -0.036456406116485596,
      "sft_loss": 0.2574033737182617,
      "step": 115,
      "train_runtime": 17737.668,
      "train_tokens_per_second": 1024.98
    },
    {
      "epoch": 0.05097164702134438,
      "grad_norm": 0.26968955993652344,
      "learning_rate": 4.968565277958985e-06,
      "logits/chosen": -1.6227775812149048,
      "logits/rejected": -1.6210949420928955,
      "logps/chosen": -0.21240372955799103,
      "logps/rejected": -0.35389187932014465,
      "loss": 0.2575,
      "num_input_tokens_seen": 18900960,
      "odds_ratio_loss": 0.4509609341621399,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.021240374073386192,
      "rewards/margins": 0.014148813672363758,
      "rewards/rejected": -0.035389192402362823,
      "sft_loss": 0.21240372955799103,
      "step": 120,
      "train_runtime": 18378.9739,
      "train_tokens_per_second": 1028.401
    },
    {
      "epoch": 0.053095465647233726,
      "grad_norm": 0.30487748980522156,
      "learning_rate": 4.965874361538526e-06,
      "logits/chosen": -1.6385345458984375,
      "logits/rejected": -1.6367794275283813,
      "logps/chosen": -0.2303926944732666,
      "logps/rejected": -0.38706523180007935,
      "loss": 0.2756,
      "num_input_tokens_seen": 19735680,
      "odds_ratio_loss": 0.4518728256225586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.02303926832973957,
      "rewards/margins": 0.015667254105210304,
      "rewards/rejected": -0.038706522434949875,
      "sft_loss": 0.2303926944732666,
      "step": 125,
      "train_runtime": 19231.3507,
      "train_tokens_per_second": 1026.224
    },
    {
      "epoch": 0.05521928427312307,
      "grad_norm": 0.26881685853004456,
      "learning_rate": 4.963073739904775e-06,
      "logits/chosen": -1.6806081533432007,
      "logits/rejected": -1.680883765220642,
      "logps/chosen": -0.2508854568004608,
      "logps/rejected": -0.3582703769207001,
      "loss": 0.3006,
      "num_input_tokens_seen": 20576544,
      "odds_ratio_loss": 0.49716052412986755,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -0.02508854679763317,
      "rewards/margins": 0.010738492012023926,
      "rewards/rejected": -0.03582703694701195,
      "sft_loss": 0.2508854568004608,
      "step": 130,
      "train_runtime": 20070.7378,
      "train_tokens_per_second": 1025.201
    },
    {
      "epoch": 0.05734310289901243,
      "grad_norm": 0.24270352721214294,
      "learning_rate": 4.960163537655641e-06,
      "logits/chosen": -1.5967674255371094,
      "logits/rejected": -1.5967588424682617,
      "logps/chosen": -0.2557952105998993,
      "logps/rejected": -0.36130109429359436,
      "loss": 0.3075,
      "num_input_tokens_seen": 21460736,
      "odds_ratio_loss": 0.5174564719200134,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -0.02557951770722866,
      "rewards/margins": 0.010550587438046932,
      "rewards/rejected": -0.03613010793924332,
      "sft_loss": 0.2557952105998993,
      "step": 135,
      "train_runtime": 20983.4171,
      "train_tokens_per_second": 1022.747
    },
    {
      "epoch": 0.059466921524901774,
      "grad_norm": 0.2963540554046631,
      "learning_rate": 4.95714388426421e-06,
      "logits/chosen": -1.6365562677383423,
      "logits/rejected": -1.6379492282867432,
      "logps/chosen": -0.19694402813911438,
      "logps/rejected": -0.3430638015270233,
      "loss": 0.2424,
      "num_input_tokens_seen": 22201264,
      "odds_ratio_loss": 0.4547400176525116,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.019694402813911438,
      "rewards/margins": 0.014611977152526379,
      "rewards/rejected": -0.03430637717247009,
      "sft_loss": 0.19694402813911438,
      "step": 140,
      "train_runtime": 21666.0456,
      "train_tokens_per_second": 1024.703
    },
    {
      "epoch": 0.06159074015079112,
      "grad_norm": 0.23768247663974762,
      "learning_rate": 4.954014914072978e-06,
      "logits/chosen": -1.5674035549163818,
      "logits/rejected": -1.5670559406280518,
      "logps/chosen": -0.24238739907741547,
      "logps/rejected": -0.3893781900405884,
      "loss": 0.2875,
      "num_input_tokens_seen": 22959440,
      "odds_ratio_loss": 0.45113325119018555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.024238741025328636,
      "rewards/margins": 0.014699079096317291,
      "rewards/rejected": -0.03893781825900078,
      "sft_loss": 0.24238739907741547,
      "step": 145,
      "train_runtime": 22389.9176,
      "train_tokens_per_second": 1025.437
    },
    {
      "epoch": 0.06371455877668047,
      "grad_norm": 0.24912331998348236,
      "learning_rate": 4.950776766287879e-06,
      "logits/chosen": -1.6102631092071533,
      "logits/rejected": -1.6071560382843018,
      "logps/chosen": -0.15143337845802307,
      "logps/rejected": -0.29899343848228455,
      "loss": 0.1908,
      "num_input_tokens_seen": 23710976,
      "odds_ratio_loss": 0.3936986029148102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.015143337659537792,
      "rewards/margins": 0.014756006188690662,
      "rewards/rejected": -0.029899343848228455,
      "sft_loss": 0.15143337845802307,
      "step": 150,
      "train_runtime": 23104.5065,
      "train_tokens_per_second": 1026.249
    },
    {
      "epoch": 0.06583837740256981,
      "grad_norm": 0.28810468316078186,
      "learning_rate": 4.94742958497209e-06,
      "logits/chosen": -1.6544857025146484,
      "logits/rejected": -1.653502106666565,
      "logps/chosen": -0.2111881971359253,
      "logps/rejected": -0.3713727593421936,
      "loss": 0.2548,
      "num_input_tokens_seen": 24441344,
      "odds_ratio_loss": 0.43608471751213074,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -0.021118823438882828,
      "rewards/margins": 0.0160184558480978,
      "rewards/rejected": -0.03713727742433548,
      "sft_loss": 0.2111881971359253,
      "step": 155,
      "train_runtime": 23797.0514,
      "train_tokens_per_second": 1027.074
    },
    {
      "epoch": 0.06796219602845917,
      "grad_norm": 0.28899532556533813,
      "learning_rate": 4.943973519039619e-06,
      "logits/chosen": -1.6064504384994507,
      "logits/rejected": -1.6060974597930908,
      "logps/chosen": -0.24232283234596252,
      "logps/rejected": -0.3618694245815277,
      "loss": 0.2915,
      "num_input_tokens_seen": 25287392,
      "odds_ratio_loss": 0.49199333786964417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.024232283234596252,
      "rewards/margins": 0.0119546577334404,
      "rewards/rejected": -0.03618694096803665,
      "sft_loss": 0.24232283234596252,
      "step": 160,
      "train_runtime": 24656.5138,
      "train_tokens_per_second": 1025.587
    },
    {
      "epoch": 0.07008601465434852,
      "grad_norm": 0.3305506408214569,
      "learning_rate": 4.9404087222486874e-06,
      "logits/chosen": -1.5842187404632568,
      "logits/rejected": -1.5849140882492065,
      "logps/chosen": -0.2113833874464035,
      "logps/rejected": -0.37160906195640564,
      "loss": 0.2561,
      "num_input_tokens_seen": 26077376,
      "odds_ratio_loss": 0.4470787048339844,
      "rewards/accuracies": 0.9000000357627869,
      "rewards/chosen": -0.02113833837211132,
      "rewards/margins": 0.016022570431232452,
      "rewards/rejected": -0.037160903215408325,
      "sft_loss": 0.2113833874464035,
      "step": 165,
      "train_runtime": 25419.927,
      "train_tokens_per_second": 1025.864
    },
    {
      "epoch": 0.07220983328023786,
      "grad_norm": 0.2550567388534546,
      "learning_rate": 4.936735353194882e-06,
      "logits/chosen": -1.5790501832962036,
      "logits/rejected": -1.5791125297546387,
      "logps/chosen": -0.19251468777656555,
      "logps/rejected": -0.35775133967399597,
      "loss": 0.2345,
      "num_input_tokens_seen": 26841968,
      "odds_ratio_loss": 0.42013540863990784,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.019251469522714615,
      "rewards/margins": 0.016523664817214012,
      "rewards/rejected": -0.03577513247728348,
      "sft_loss": 0.19251468777656555,
      "step": 170,
      "train_runtime": 26159.9335,
      "train_tokens_per_second": 1026.072
    },
    {
      "epoch": 0.07433365190612722,
      "grad_norm": 0.2912465035915375,
      "learning_rate": 4.932953575304102e-06,
      "logits/chosen": -1.6077922582626343,
      "logits/rejected": -1.6070846319198608,
      "logps/chosen": -0.20563630759716034,
      "logps/rejected": -0.3200438618659973,
      "loss": 0.2531,
      "num_input_tokens_seen": 27600752,
      "odds_ratio_loss": 0.4742105007171631,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.020563630387187004,
      "rewards/margins": 0.011440754868090153,
      "rewards/rejected": -0.03200438618659973,
      "sft_loss": 0.20563630759716034,
      "step": 175,
      "train_runtime": 26893.0439,
      "train_tokens_per_second": 1026.316
    },
    {
      "epoch": 0.07645747053201657,
      "grad_norm": 0.3313763737678528,
      "learning_rate": 4.929063556825288e-06,
      "logits/chosen": -1.6549072265625,
      "logits/rejected": -1.6553726196289062,
      "logps/chosen": -0.22688885033130646,
      "logps/rejected": -0.36104461550712585,
      "loss": 0.2742,
      "num_input_tokens_seen": 28445728,
      "odds_ratio_loss": 0.4726777970790863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.022688886150717735,
      "rewards/margins": 0.01341557502746582,
      "rewards/rejected": -0.036104459315538406,
      "sft_loss": 0.22688885033130646,
      "step": 180,
      "train_runtime": 27731.9166,
      "train_tokens_per_second": 1025.74
    },
    {
      "epoch": 0.07858128915790591,
      "grad_norm": 0.29398900270462036,
      "learning_rate": 4.925065470822938e-06,
      "logits/chosen": -1.6529582738876343,
      "logits/rejected": -1.6513477563858032,
      "logps/chosen": -0.21773914992809296,
      "logps/rejected": -0.34818795323371887,
      "loss": 0.265,
      "num_input_tokens_seen": 29289168,
      "odds_ratio_loss": 0.4724561870098114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.021773917600512505,
      "rewards/margins": 0.013044880703091621,
      "rewards/rejected": -0.034818798303604126,
      "sft_loss": 0.21773914992809296,
      "step": 185,
      "train_runtime": 28572.7248,
      "train_tokens_per_second": 1025.074
    },
    {
      "epoch": 0.08070510778379526,
      "grad_norm": 0.3113972544670105,
      "learning_rate": 4.920959495169406e-06,
      "logits/chosen": -1.6174672842025757,
      "logits/rejected": -1.6192734241485596,
      "logps/chosen": -0.23038557171821594,
      "logps/rejected": -0.37209945917129517,
      "loss": 0.2776,
      "num_input_tokens_seen": 30115728,
      "odds_ratio_loss": 0.47191864252090454,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.023038554936647415,
      "rewards/margins": 0.014171388931572437,
      "rewards/rejected": -0.037209946662187576,
      "sft_loss": 0.23038557171821594,
      "step": 190,
      "train_runtime": 29357.982,
      "train_tokens_per_second": 1025.811
    },
    {
      "epoch": 0.08282892640968462,
      "grad_norm": 0.2816208004951477,
      "learning_rate": 4.91674581253699e-06,
      "logits/chosen": -1.55691397190094,
      "logits/rejected": -1.5565577745437622,
      "logps/chosen": -0.2311859130859375,
      "logps/rejected": -0.3674752414226532,
      "loss": 0.2773,
      "num_input_tokens_seen": 30938640,
      "odds_ratio_loss": 0.46152350306510925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.02311859093606472,
      "rewards/margins": 0.01362893171608448,
      "rewards/rejected": -0.0367475263774395,
      "sft_loss": 0.2311859130859375,
      "step": 195,
      "train_runtime": 30189.5574,
      "train_tokens_per_second": 1024.813
    },
    {
      "epoch": 0.08495274503557396,
      "grad_norm": 0.35654670000076294,
      "learning_rate": 4.912424610389802e-06,
      "logits/chosen": -1.7251243591308594,
      "logits/rejected": -1.721566081047058,
      "logps/chosen": -0.2660072445869446,
      "logps/rejected": -0.42824801802635193,
      "loss": 0.3096,
      "num_input_tokens_seen": 31851168,
      "odds_ratio_loss": 0.43630552291870117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.026600724086165428,
      "rewards/margins": 0.016224076971411705,
      "rewards/rejected": -0.04282480105757713,
      "sft_loss": 0.2660072445869446,
      "step": 200,
      "train_runtime": 31137.0192,
      "train_tokens_per_second": 1022.936
    },
    {
      "epoch": 0.08707656366146331,
      "grad_norm": 0.33661603927612305,
      "learning_rate": 4.9079960809754334e-06,
      "logits/chosen": -1.6197242736816406,
      "logits/rejected": -1.6200474500656128,
      "logps/chosen": -0.2601544260978699,
      "logps/rejected": -0.4248271882534027,
      "loss": 0.3043,
      "num_input_tokens_seen": 32722880,
      "odds_ratio_loss": 0.4414806365966797,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.026015443727374077,
      "rewards/margins": 0.01646728254854679,
      "rewards/rejected": -0.042482730001211166,
      "sft_loss": 0.2601544260978699,
      "step": 205,
      "train_runtime": 31973.7173,
      "train_tokens_per_second": 1023.431
    },
    {
      "epoch": 0.08920038228735266,
      "grad_norm": 0.36740103363990784,
      "learning_rate": 4.903460421316397e-06,
      "logits/chosen": -1.5472924709320068,
      "logits/rejected": -1.5462303161621094,
      "logps/chosen": -0.17049634456634521,
      "logps/rejected": -0.2507079541683197,
      "loss": 0.2202,
      "num_input_tokens_seen": 33349408,
      "odds_ratio_loss": 0.49721336364746094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01704963482916355,
      "rewards/margins": 0.0080211590975523,
      "rewards/rejected": -0.0250707920640707,
      "sft_loss": 0.17049634456634521,
      "step": 210,
      "train_runtime": 32480.6435,
      "train_tokens_per_second": 1026.747
    },
    {
      "epoch": 0.091324200913242,
      "grad_norm": 0.27315446734428406,
      "learning_rate": 4.8988178332013645e-06,
      "logits/chosen": -1.5653256177902222,
      "logits/rejected": -1.561923623085022,
      "logps/chosen": -0.21049056947231293,
      "logps/rejected": -0.36978989839553833,
      "loss": 0.2521,
      "num_input_tokens_seen": 34069152,
      "odds_ratio_loss": 0.41596508026123047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.021049058064818382,
      "rewards/margins": 0.0159299299120903,
      "rewards/rejected": -0.036978986114263535,
      "sft_loss": 0.21049056947231293,
      "step": 215,
      "train_runtime": 33173.1098,
      "train_tokens_per_second": 1027.011
    },
    {
      "epoch": 0.09344801953913136,
      "grad_norm": 0.23859930038452148,
      "learning_rate": 4.894068523176187e-06,
      "logits/chosen": -1.6060088872909546,
      "logits/rejected": -1.6064884662628174,
      "logps/chosen": -0.2257494032382965,
      "logps/rejected": -0.3585110306739807,
      "loss": 0.2738,
      "num_input_tokens_seen": 34955728,
      "odds_ratio_loss": 0.48070240020751953,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.02257493883371353,
      "rewards/margins": 0.013276161625981331,
      "rewards/rejected": -0.03585110232234001,
      "sft_loss": 0.2257494032382965,
      "step": 220,
      "train_runtime": 34069.7362,
      "train_tokens_per_second": 1026.005
    },
    {
      "epoch": 0.09557183816502071,
      "grad_norm": 0.2715383768081665,
      "learning_rate": 4.889212702534707e-06,
      "logits/chosen": -1.5576510429382324,
      "logits/rejected": -1.5570205450057983,
      "logps/chosen": -0.17858575284481049,
      "logps/rejected": -0.327460378408432,
      "loss": 0.2215,
      "num_input_tokens_seen": 35665600,
      "odds_ratio_loss": 0.4286881387233734,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01785857416689396,
      "rewards/margins": 0.014887464232742786,
      "rewards/rejected": -0.03274604305624962,
      "sft_loss": 0.17858575284481049,
      "step": 225,
      "train_runtime": 34721.9249,
      "train_tokens_per_second": 1027.178
    },
    {
      "epoch": 0.09769565679091005,
      "grad_norm": 0.49879419803619385,
      "learning_rate": 4.884250587309357e-06,
      "logits/chosen": -1.5645736455917358,
      "logits/rejected": -1.5642821788787842,
      "logps/chosen": -0.20932011306285858,
      "logps/rejected": -0.4076058864593506,
      "loss": 0.2482,
      "num_input_tokens_seen": 36454592,
      "odds_ratio_loss": 0.3891282379627228,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.020932013168931007,
      "rewards/margins": 0.01982857845723629,
      "rewards/rejected": -0.040760595351457596,
      "sft_loss": 0.20932011306285858,
      "step": 230,
      "train_runtime": 35478.8293,
      "train_tokens_per_second": 1027.503
    },
    {
      "epoch": 0.0998194754167994,
      "grad_norm": 0.2571529150009155,
      "learning_rate": 4.8791823982615525e-06,
      "logits/chosen": -1.5819326639175415,
      "logits/rejected": -1.5807644128799438,
      "logps/chosen": -0.18989567458629608,
      "logps/rejected": -0.3264922797679901,
      "loss": 0.2351,
      "num_input_tokens_seen": 37219184,
      "odds_ratio_loss": 0.4521309435367584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.018989568576216698,
      "rewards/margins": 0.013659658841788769,
      "rewards/rejected": -0.03264923021197319,
      "sft_loss": 0.18989567458629608,
      "step": 235,
      "train_runtime": 36201.5029,
      "train_tokens_per_second": 1028.112
    },
    {
      "epoch": 0.10194329404268876,
      "grad_norm": 0.42164161801338196,
      "learning_rate": 4.874008360871862e-06,
      "logits/chosen": -1.530084490776062,
      "logits/rejected": -1.528136968612671,
      "logps/chosen": -0.1685694456100464,
      "logps/rejected": -0.3498329818248749,
      "loss": 0.2073,
      "num_input_tokens_seen": 37894368,
      "odds_ratio_loss": 0.3874427080154419,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.016856946051120758,
      "rewards/margins": 0.0181263517588377,
      "rewards/rejected": -0.03498329967260361,
      "sft_loss": 0.1685694456100464,
      "step": 240,
      "train_runtime": 36810.8785,
      "train_tokens_per_second": 1029.434
    },
    {
      "epoch": 0.1040671126685781,
      "grad_norm": 0.3284667432308197,
      "learning_rate": 4.868728705329988e-06,
      "logits/chosen": -1.5562241077423096,
      "logits/rejected": -1.5540558099746704,
      "logps/chosen": -0.23364387452602386,
      "logps/rejected": -0.41561222076416016,
      "loss": 0.275,
      "num_input_tokens_seen": 38615440,
      "odds_ratio_loss": 0.41352373361587524,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.023364387452602386,
      "rewards/margins": 0.0181968342512846,
      "rewards/rejected": -0.041561223566532135,
      "sft_loss": 0.23364387452602386,
      "step": 245,
      "train_runtime": 37505.4295,
      "train_tokens_per_second": 1029.596
    },
    {
      "epoch": 0.10619093129446745,
      "grad_norm": 0.268734335899353,
      "learning_rate": 4.863343666524512e-06,
      "logits/chosen": -1.62189519405365,
      "logits/rejected": -1.6187324523925781,
      "logps/chosen": -0.1755245476961136,
      "logps/rejected": -0.3848912715911865,
      "loss": 0.2094,
      "num_input_tokens_seen": 39312240,
      "odds_ratio_loss": 0.3384332060813904,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.017552455887198448,
      "rewards/margins": 0.020936671644449234,
      "rewards/rejected": -0.03848912939429283,
      "sft_loss": 0.1755245476961136,
      "step": 250,
      "train_runtime": 38135.5717,
      "train_tokens_per_second": 1030.855
    },
    {
      "epoch": 0.1083147499203568,
      "grad_norm": 0.31579649448394775,
      "learning_rate": 4.857853484032452e-06,
      "logits/chosen": -1.5422351360321045,
      "logits/rejected": -1.540584921836853,
      "logps/chosen": -0.20063064992427826,
      "logps/rejected": -0.3748759627342224,
      "loss": 0.2429,
      "num_input_tokens_seen": 40036048,
      "odds_ratio_loss": 0.42222413420677185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.020063063129782677,
      "rewards/margins": 0.017424535006284714,
      "rewards/rejected": -0.03748759627342224,
      "sft_loss": 0.20063064992427826,
      "step": 255,
      "train_runtime": 38815.5146,
      "train_tokens_per_second": 1031.444
    },
    {
      "epoch": 0.11043856854624615,
      "grad_norm": 0.3133474886417389,
      "learning_rate": 4.85225840210861e-06,
      "logits/chosen": -1.5848644971847534,
      "logits/rejected": -1.5845024585723877,
      "logps/chosen": -0.20163094997406006,
      "logps/rejected": -0.3724139332771301,
      "loss": 0.2446,
      "num_input_tokens_seen": 40807328,
      "odds_ratio_loss": 0.430143266916275,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.020163094624876976,
      "rewards/margins": 0.017078299075365067,
      "rewards/rejected": -0.03724139556288719,
      "sft_loss": 0.20163094997406006,
      "step": 260,
      "train_runtime": 39555.1144,
      "train_tokens_per_second": 1031.657
    },
    {
      "epoch": 0.1125623871721355,
      "grad_norm": 0.29493072628974915,
      "learning_rate": 4.846558669674688e-06,
      "logits/chosen": -1.557217001914978,
      "logits/rejected": -1.5541645288467407,
      "logps/chosen": -0.19392022490501404,
      "logps/rejected": -0.39551058411598206,
      "loss": 0.2319,
      "num_input_tokens_seen": 41557968,
      "odds_ratio_loss": 0.3794221580028534,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.019392022863030434,
      "rewards/margins": 0.0201590396463871,
      "rewards/rejected": -0.039551060646772385,
      "sft_loss": 0.19392022490501404,
      "step": 265,
      "train_runtime": 40239.6601,
      "train_tokens_per_second": 1032.761
    },
    {
      "epoch": 0.11468620579802485,
      "grad_norm": 0.27749863266944885,
      "learning_rate": 4.840754540308232e-06,
      "logits/chosen": -1.6163939237594604,
      "logits/rejected": -1.6133354902267456,
      "logps/chosen": -0.20039153099060059,
      "logps/rejected": -0.3801855444908142,
      "loss": 0.2409,
      "num_input_tokens_seen": 42309424,
      "odds_ratio_loss": 0.4050087034702301,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.020039156079292297,
      "rewards/margins": 0.017979398369789124,
      "rewards/rejected": -0.03801855444908142,
      "sft_loss": 0.20039153099060059,
      "step": 270,
      "train_runtime": 40927.2194,
      "train_tokens_per_second": 1033.772
    },
    {
      "epoch": 0.1168100244239142,
      "grad_norm": 0.46233096718788147,
      "learning_rate": 4.834846272231339e-06,
      "logits/chosen": -1.5639489889144897,
      "logits/rejected": -1.5624293088912964,
      "logps/chosen": -0.24991865456104279,
      "logps/rejected": -0.4742889404296875,
      "loss": 0.2873,
      "num_input_tokens_seen": 43172032,
      "odds_ratio_loss": 0.3739108443260193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.024991866201162338,
      "rewards/margins": 0.022437024861574173,
      "rewards/rejected": -0.04742889106273651,
      "sft_loss": 0.24991865456104279,
      "step": 275,
      "train_runtime": 41829.3681,
      "train_tokens_per_second": 1032.099
    },
    {
      "epoch": 0.11893384304980355,
      "grad_norm": 0.4262549579143524,
      "learning_rate": 4.828834128299173e-06,
      "logits/chosen": -1.5840650796890259,
      "logits/rejected": -1.5827009677886963,
      "logps/chosen": -0.1959383487701416,
      "logps/rejected": -0.4240526854991913,
      "loss": 0.2316,
      "num_input_tokens_seen": 43948032,
      "odds_ratio_loss": 0.3563404083251953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01959383487701416,
      "rewards/margins": 0.02281143330037594,
      "rewards/rejected": -0.04240526631474495,
      "sft_loss": 0.1959383487701416,
      "step": 280,
      "train_runtime": 42585.7044,
      "train_tokens_per_second": 1031.99
    },
    {
      "epoch": 0.1210576616756929,
      "grad_norm": 0.42866721749305725,
      "learning_rate": 4.8227183759882705e-06,
      "logits/chosen": -1.594415545463562,
      "logits/rejected": -1.5925581455230713,
      "logps/chosen": -0.20694172382354736,
      "logps/rejected": -0.38142308592796326,
      "loss": 0.2467,
      "num_input_tokens_seen": 44737744,
      "odds_ratio_loss": 0.3977479040622711,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.020694173872470856,
      "rewards/margins": 0.01744813285768032,
      "rewards/rejected": -0.03814230486750603,
      "sft_loss": 0.20694172382354736,
      "step": 285,
      "train_runtime": 43270.3104,
      "train_tokens_per_second": 1033.913
    },
    {
      "epoch": 0.12318148030158224,
      "grad_norm": 0.3096730411052704,
      "learning_rate": 4.8164992873846385e-06,
      "logits/chosen": -1.6240776777267456,
      "logits/rejected": -1.6202102899551392,
      "logps/chosen": -0.19814078509807587,
      "logps/rejected": -0.34014254808425903,
      "loss": 0.2421,
      "num_input_tokens_seen": 45544192,
      "odds_ratio_loss": 0.439925879240036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.019814081490039825,
      "rewards/margins": 0.014200172387063503,
      "rewards/rejected": -0.0340142548084259,
      "sft_loss": 0.19814078509807587,
      "step": 290,
      "train_runtime": 44031.8453,
      "train_tokens_per_second": 1034.347
    },
    {
      "epoch": 0.12530529892747158,
      "grad_norm": 0.39139634370803833,
      "learning_rate": 4.810177139171653e-06,
      "logits/chosen": -1.4523519277572632,
      "logits/rejected": -1.4513581991195679,
      "logps/chosen": -0.19744689762592316,
      "logps/rejected": -0.37059417366981506,
      "loss": 0.2381,
      "num_input_tokens_seen": 46181408,
      "odds_ratio_loss": 0.40675726532936096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.019744688645005226,
      "rewards/margins": 0.01731473207473755,
      "rewards/rejected": -0.037059422582387924,
      "sft_loss": 0.19744689762592316,
      "step": 295,
      "train_runtime": 44601.9308,
      "train_tokens_per_second": 1035.413
    },
    {
      "epoch": 0.12742911755336095,
      "grad_norm": 0.35217106342315674,
      "learning_rate": 4.803752212617747e-06,
      "logits/chosen": -1.5504106283187866,
      "logits/rejected": -1.547971487045288,
      "logps/chosen": -0.15978465974330902,
      "logps/rejected": -0.35288044810295105,
      "loss": 0.1975,
      "num_input_tokens_seen": 46916416,
      "odds_ratio_loss": 0.3774333894252777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.015978466719388962,
      "rewards/margins": 0.019309580326080322,
      "rewards/rejected": -0.035288047045469284,
      "sft_loss": 0.15978465974330902,
      "step": 300,
      "train_runtime": 45262.0651,
      "train_tokens_per_second": 1036.55
    },
    {
      "epoch": 0.1295529361792503,
      "grad_norm": 0.3365028500556946,
      "learning_rate": 4.7972247935638995e-06,
      "logits/chosen": -1.5341777801513672,
      "logits/rejected": -1.5325621366500854,
      "logps/chosen": -0.20445671677589417,
      "logps/rejected": -0.4159996211528778,
      "loss": 0.2408,
      "num_input_tokens_seen": 47649168,
      "odds_ratio_loss": 0.3637700080871582,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.020445672795176506,
      "rewards/margins": 0.021154293790459633,
      "rewards/rejected": -0.04159996658563614,
      "sft_loss": 0.20445671677589417,
      "step": 305,
      "train_runtime": 45974.466,
      "train_tokens_per_second": 1036.427
    },
    {
      "epoch": 0.13167675480513963,
      "grad_norm": 0.3367343544960022,
      "learning_rate": 4.790595172410914e-06,
      "logits/chosen": -1.5029903650283813,
      "logits/rejected": -1.5012362003326416,
      "logps/chosen": -0.17400263249874115,
      "logps/rejected": -0.43273279070854187,
      "loss": 0.2062,
      "num_input_tokens_seen": 48263392,
      "odds_ratio_loss": 0.32218560576438904,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.017400262877345085,
      "rewards/margins": 0.025873014703392982,
      "rewards/rejected": -0.04327327758073807,
      "sft_loss": 0.17400263249874115,
      "step": 310,
      "train_runtime": 46490.0475,
      "train_tokens_per_second": 1038.145
    },
    {
      "epoch": 0.133800573431029,
      "grad_norm": 0.8648568391799927,
      "learning_rate": 4.783863644106502e-06,
      "logits/chosen": -1.538582682609558,
      "logits/rejected": -1.5359286069869995,
      "logps/chosen": -0.21271610260009766,
      "logps/rejected": -0.47346386313438416,
      "loss": 0.2455,
      "num_input_tokens_seen": 48933168,
      "odds_ratio_loss": 0.3282425105571747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.021271610632538795,
      "rewards/margins": 0.02607477270066738,
      "rewards/rejected": -0.047346390783786774,
      "sft_loss": 0.21271610260009766,
      "step": 315,
      "train_runtime": 47098.524,
      "train_tokens_per_second": 1038.953
    },
    {
      "epoch": 0.13592439205691834,
      "grad_norm": 0.31725844740867615,
      "learning_rate": 4.7770305081321645e-06,
      "logits/chosen": -1.5916751623153687,
      "logits/rejected": -1.5888961553573608,
      "logps/chosen": -0.24918365478515625,
      "logps/rejected": -0.4175334870815277,
      "loss": 0.2911,
      "num_input_tokens_seen": 49742688,
      "odds_ratio_loss": 0.4194348454475403,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.024918366223573685,
      "rewards/margins": 0.016834987327456474,
      "rewards/rejected": -0.04175335168838501,
      "sft_loss": 0.24918365478515625,
      "step": 320,
      "train_runtime": 47878.5942,
      "train_tokens_per_second": 1038.934
    },
    {
      "epoch": 0.13804821068280768,
      "grad_norm": 0.2416473627090454,
      "learning_rate": 4.770096068489861e-06,
      "logits/chosen": -1.505483627319336,
      "logits/rejected": -1.5024555921554565,
      "logps/chosen": -0.20475201308727264,
      "logps/rejected": -0.42144569754600525,
      "loss": 0.2429,
      "num_input_tokens_seen": 50486880,
      "odds_ratio_loss": 0.3811642825603485,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.020475205034017563,
      "rewards/margins": 0.02166936732828617,
      "rewards/rejected": -0.04214457422494888,
      "sft_loss": 0.20475201308727264,
      "step": 325,
      "train_runtime": 48553.8858,
      "train_tokens_per_second": 1039.811
    },
    {
      "epoch": 0.14017202930869704,
      "grad_norm": 0.3809424042701721,
      "learning_rate": 4.763060633688488e-06,
      "logits/chosen": -1.5287418365478516,
      "logits/rejected": -1.5293445587158203,
      "logps/chosen": -0.21139581501483917,
      "logps/rejected": -0.4373548626899719,
      "loss": 0.2501,
      "num_input_tokens_seen": 51203488,
      "odds_ratio_loss": 0.38707247376441956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.021139580756425858,
      "rewards/margins": 0.022595908492803574,
      "rewards/rejected": -0.04373548552393913,
      "sft_loss": 0.21139581501483917,
      "step": 330,
      "train_runtime": 49216.0357,
      "train_tokens_per_second": 1040.382
    },
    {
      "epoch": 0.14229584793458638,
      "grad_norm": 0.30999433994293213,
      "learning_rate": 4.7559245167301535e-06,
      "logits/chosen": -1.537314534187317,
      "logits/rejected": -1.53397536277771,
      "logps/chosen": -0.18174372613430023,
      "logps/rejected": -0.4075315594673157,
      "loss": 0.2192,
      "num_input_tokens_seen": 52016432,
      "odds_ratio_loss": 0.37452277541160583,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.018174372613430023,
      "rewards/margins": 0.022578781470656395,
      "rewards/rejected": -0.04075315594673157,
      "sft_loss": 0.18174372613430023,
      "step": 335,
      "train_runtime": 50004.3935,
      "train_tokens_per_second": 1040.237
    },
    {
      "epoch": 0.14441966656047572,
      "grad_norm": 0.38702282309532166,
      "learning_rate": 4.748688035096255e-06,
      "logits/chosen": -1.5276494026184082,
      "logits/rejected": -1.5261821746826172,
      "logps/chosen": -0.21979044377803802,
      "logps/rejected": -0.4520311951637268,
      "loss": 0.2564,
      "num_input_tokens_seen": 52856320,
      "odds_ratio_loss": 0.3660522401332855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.021979045122861862,
      "rewards/margins": 0.023224076256155968,
      "rewards/rejected": -0.04520311579108238,
      "sft_loss": 0.21979044377803802,
      "step": 340,
      "train_runtime": 50856.5833,
      "train_tokens_per_second": 1039.321
    },
    {
      "epoch": 0.1465434851863651,
      "grad_norm": 0.35227009654045105,
      "learning_rate": 4.741351510733349e-06,
      "logits/chosen": -1.6234554052352905,
      "logits/rejected": -1.619990587234497,
      "logps/chosen": -0.22717244923114777,
      "logps/rejected": -0.415462464094162,
      "loss": 0.2681,
      "num_input_tokens_seen": 53746512,
      "odds_ratio_loss": 0.40907421708106995,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.022717244923114777,
      "rewards/margins": 0.018829001113772392,
      "rewards/rejected": -0.04154624417424202,
      "sft_loss": 0.22717244923114777,
      "step": 345,
      "train_runtime": 51755.8205,
      "train_tokens_per_second": 1038.463
    },
    {
      "epoch": 0.14866730381225443,
      "grad_norm": 0.3836142122745514,
      "learning_rate": 4.73391527003883e-06,
      "logits/chosen": -1.465417742729187,
      "logits/rejected": -1.4578847885131836,
      "logps/chosen": -0.22221946716308594,
      "logps/rejected": -0.48223400115966797,
      "loss": 0.2567,
      "num_input_tokens_seen": 54517248,
      "odds_ratio_loss": 0.3446839451789856,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.022221947088837624,
      "rewards/margins": 0.02600145898759365,
      "rewards/rejected": -0.048223406076431274,
      "sft_loss": 0.22221946716308594,
      "step": 350,
      "train_runtime": 52527.1446,
      "train_tokens_per_second": 1037.887
    },
    {
      "epoch": 0.15079112243814377,
      "grad_norm": 0.5152142643928528,
      "learning_rate": 4.726379643846412e-06,
      "logits/chosen": -1.4976345300674438,
      "logits/rejected": -1.4955178499221802,
      "logps/chosen": -0.20917537808418274,
      "logps/rejected": -0.39172467589378357,
      "loss": 0.2492,
      "num_input_tokens_seen": 55313248,
      "odds_ratio_loss": 0.4004102349281311,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.020917538553476334,
      "rewards/margins": 0.01825493387877941,
      "rewards/rejected": -0.039172470569610596,
      "sft_loss": 0.20917537808418274,
      "step": 355,
      "train_runtime": 53285.0776,
      "train_tokens_per_second": 1038.063
    },
    {
      "epoch": 0.15291494106403314,
      "grad_norm": 0.35300472378730774,
      "learning_rate": 4.7187449674114085e-06,
      "logits/chosen": -1.6178301572799683,
      "logits/rejected": -1.6141846179962158,
      "logps/chosen": -0.2544923722743988,
      "logps/rejected": -0.48757272958755493,
      "loss": 0.2921,
      "num_input_tokens_seen": 56199456,
      "odds_ratio_loss": 0.37635698914527893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.02544924058020115,
      "rewards/margins": 0.023308034986257553,
      "rewards/rejected": -0.04875727370381355,
      "sft_loss": 0.2544923722743988,
      "step": 360,
      "train_runtime": 54178.4692,
      "train_tokens_per_second": 1037.302
    },
    {
      "epoch": 0.15503875968992248,
      "grad_norm": 0.4215538799762726,
      "learning_rate": 4.711011580395813e-06,
      "logits/chosen": -1.5603975057601929,
      "logits/rejected": -1.5581893920898438,
      "logps/chosen": -0.2834070324897766,
      "logps/rejected": -0.49649810791015625,
      "loss": 0.3247,
      "num_input_tokens_seen": 57097712,
      "odds_ratio_loss": 0.41326332092285156,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.02834070287644863,
      "rewards/margins": 0.021309109404683113,
      "rewards/rejected": -0.04964981600642204,
      "sft_loss": 0.2834070324897766,
      "step": 365,
      "train_runtime": 55034.2466,
      "train_tokens_per_second": 1037.494
    },
    {
      "epoch": 0.15716257831581182,
      "grad_norm": 0.42323988676071167,
      "learning_rate": 4.703179826853195e-06,
      "logits/chosen": -1.4333704710006714,
      "logits/rejected": -1.4315990209579468,
      "logps/chosen": -0.19110994040966034,
      "logps/rejected": -0.4354076385498047,
      "loss": 0.2267,
      "num_input_tokens_seen": 57789760,
      "odds_ratio_loss": 0.35592490434646606,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.019110996276140213,
      "rewards/margins": 0.024429772049188614,
      "rewards/rejected": -0.04354076832532883,
      "sft_loss": 0.19110994040966034,
      "step": 370,
      "train_runtime": 55699.4786,
      "train_tokens_per_second": 1037.528
    },
    {
      "epoch": 0.15928639694170119,
      "grad_norm": 0.40263572335243225,
      "learning_rate": 4.695250055213387e-06,
      "logits/chosen": -1.5139859914779663,
      "logits/rejected": -1.5134934186935425,
      "logps/chosen": -0.23862476646900177,
      "logps/rejected": -0.4360879957675934,
      "loss": 0.2791,
      "num_input_tokens_seen": 58535536,
      "odds_ratio_loss": 0.4044667184352875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.023862475529313087,
      "rewards/margins": 0.0197463296353817,
      "rewards/rejected": -0.04360880330204964,
      "sft_loss": 0.23862476646900177,
      "step": 375,
      "train_runtime": 56411.8333,
      "train_tokens_per_second": 1037.646
    },
    {
      "epoch": 0.16141021556759053,
      "grad_norm": 0.36219680309295654,
      "learning_rate": 4.687222618266987e-06,
      "logits/chosen": -1.5795234441757202,
      "logits/rejected": -1.573940634727478,
      "logps/chosen": -0.18536217510700226,
      "logps/rejected": -0.4107401967048645,
      "loss": 0.2216,
      "num_input_tokens_seen": 59337168,
      "odds_ratio_loss": 0.36286160349845886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.018536219373345375,
      "rewards/margins": 0.022537803277373314,
      "rewards/rejected": -0.04107401892542839,
      "sft_loss": 0.18536217510700226,
      "step": 380,
      "train_runtime": 57171.7422,
      "train_tokens_per_second": 1037.876
    },
    {
      "epoch": 0.16353403419347987,
      "grad_norm": 0.3343680799007416,
      "learning_rate": 4.67909787314966e-06,
      "logits/chosen": -1.4570516347885132,
      "logits/rejected": -1.4535053968429565,
      "logps/chosen": -0.2062748223543167,
      "logps/rejected": -0.40731826424598694,
      "loss": 0.2474,
      "num_input_tokens_seen": 60081040,
      "odds_ratio_loss": 0.410898894071579,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.02062748186290264,
      "rewards/margins": 0.020104344934225082,
      "rewards/rejected": -0.040731824934482574,
      "sft_loss": 0.2062748223543167,
      "step": 385,
      "train_runtime": 57825.0127,
      "train_tokens_per_second": 1039.015
    },
    {
      "epoch": 0.16565785281936923,
      "grad_norm": 0.4492781460285187,
      "learning_rate": 4.670876181326253e-06,
      "logits/chosen": -1.538793921470642,
      "logits/rejected": -1.535956621170044,
      "logps/chosen": -0.1623261421918869,
      "logps/rejected": -0.43555983901023865,
      "loss": 0.1939,
      "num_input_tokens_seen": 60755744,
      "odds_ratio_loss": 0.3162221610546112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01623261533677578,
      "rewards/margins": 0.027323368936777115,
      "rewards/rejected": -0.043555986136198044,
      "sft_loss": 0.1623261421918869,
      "step": 390,
      "train_runtime": 58412.1702,
      "train_tokens_per_second": 1040.121
    },
    {
      "epoch": 0.16778167144525857,
      "grad_norm": 1.2000969648361206,
      "learning_rate": 4.6625579085747096e-06,
      "logits/chosen": -1.4861072301864624,
      "logits/rejected": -1.48383367061615,
      "logps/chosen": -0.17792180180549622,
      "logps/rejected": -0.4890073835849762,
      "loss": 0.2088,
      "num_input_tokens_seen": 61431360,
      "odds_ratio_loss": 0.3083958029747009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01779218018054962,
      "rewards/margins": 0.031108561903238297,
      "rewards/rejected": -0.04890074208378792,
      "sft_loss": 0.17792180180549622,
      "step": 395,
      "train_runtime": 59028.1496,
      "train_tokens_per_second": 1040.713
    },
    {
      "epoch": 0.1699054900711479,
      "grad_norm": 0.33823323249816895,
      "learning_rate": 4.6541434249698e-06,
      "logits/chosen": -1.4362635612487793,
      "logits/rejected": -1.4307713508605957,
      "logps/chosen": -0.1838473081588745,
      "logps/rejected": -0.36607468128204346,
      "loss": 0.2232,
      "num_input_tokens_seen": 62182272,
      "odds_ratio_loss": 0.39401254057884216,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01838473044335842,
      "rewards/margins": 0.018222734332084656,
      "rewards/rejected": -0.036607466638088226,
      "sft_loss": 0.1838473081588745,
      "step": 400,
      "train_runtime": 59758.104,
      "train_tokens_per_second": 1040.566
    },
    {
      "epoch": 0.17202930869703728,
      "grad_norm": 0.28551188111305237,
      "learning_rate": 4.645633104866655e-06,
      "logits/chosen": -1.5525025129318237,
      "logits/rejected": -1.5476278066635132,
      "logps/chosen": -0.22308234870433807,
      "logps/rejected": -0.41523274779319763,
      "loss": 0.2631,
      "num_input_tokens_seen": 63022976,
      "odds_ratio_loss": 0.4005754590034485,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.022308234125375748,
      "rewards/margins": 0.019215045496821404,
      "rewards/rejected": -0.0415232814848423,
      "sft_loss": 0.22308234870433807,
      "step": 405,
      "train_runtime": 60610.894,
      "train_tokens_per_second": 1039.796
    },
    {
      "epoch": 0.17415312732292662,
      "grad_norm": 0.3130952715873718,
      "learning_rate": 4.637027326884111e-06,
      "logits/chosen": -1.511374831199646,
      "logits/rejected": -1.5063308477401733,
      "logps/chosen": -0.21574239432811737,
      "logps/rejected": -0.44365617632865906,
      "loss": 0.2548,
      "num_input_tokens_seen": 63671152,
      "odds_ratio_loss": 0.3902105987071991,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.021574238315224648,
      "rewards/margins": 0.02279137633740902,
      "rewards/rejected": -0.04436561465263367,
      "sft_loss": 0.21574239432811737,
      "step": 410,
      "train_runtime": 61168.6494,
      "train_tokens_per_second": 1040.912
    },
    {
      "epoch": 0.17627694594881596,
      "grad_norm": 0.5503313541412354,
      "learning_rate": 4.628326473887865e-06,
      "logits/chosen": -1.5144814252853394,
      "logits/rejected": -1.5087536573410034,
      "logps/chosen": -0.2391294240951538,
      "logps/rejected": -0.5166025161743164,
      "loss": 0.2765,
      "num_input_tokens_seen": 64493712,
      "odds_ratio_loss": 0.37384283542633057,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.02391294203698635,
      "rewards/margins": 0.02774731256067753,
      "rewards/rejected": -0.05166025832295418,
      "sft_loss": 0.2391294240951538,
      "step": 415,
      "train_runtime": 61963.2935,
      "train_tokens_per_second": 1040.837
    },
    {
      "epoch": 0.17840076457470533,
      "grad_norm": 0.38913479447364807,
      "learning_rate": 4.619530932973446e-06,
      "logits/chosen": -1.5765724182128906,
      "logits/rejected": -1.572107195854187,
      "logps/chosen": -0.24681340157985687,
      "logps/rejected": -0.49377235770225525,
      "loss": 0.2852,
      "num_input_tokens_seen": 65387984,
      "odds_ratio_loss": 0.38363930583000183,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.024681339040398598,
      "rewards/margins": 0.024695897474884987,
      "rewards/rejected": -0.04937724024057388,
      "sft_loss": 0.24681340157985687,
      "step": 420,
      "train_runtime": 62884.0441,
      "train_tokens_per_second": 1039.818
    },
    {
      "epoch": 0.18052458320059467,
      "grad_norm": 0.30293014645576477,
      "learning_rate": 4.610641095448984e-06,
      "logits/chosen": -1.458439588546753,
      "logits/rejected": -1.451570749282837,
      "logps/chosen": -0.17603813111782074,
      "logps/rejected": -0.37730008363723755,
      "loss": 0.2144,
      "num_input_tokens_seen": 66072208,
      "odds_ratio_loss": 0.38355544209480286,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.017603812739253044,
      "rewards/margins": 0.02012619376182556,
      "rewards/rejected": -0.037730008363723755,
      "sft_loss": 0.17603813111782074,
      "step": 425,
      "train_runtime": 63510.6455,
      "train_tokens_per_second": 1040.333
    },
    {
      "epoch": 0.182648401826484,
      "grad_norm": 0.39900699257850647,
      "learning_rate": 4.6016573568178105e-06,
      "logits/chosen": -1.4609564542770386,
      "logits/rejected": -1.4594862461090088,
      "logps/chosen": -0.1564684808254242,
      "logps/rejected": -0.5182979106903076,
      "loss": 0.1837,
      "num_input_tokens_seen": 66642608,
      "odds_ratio_loss": 0.27226004004478455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01564684882760048,
      "rewards/margins": 0.0361829437315464,
      "rewards/rejected": -0.05182979255914688,
      "sft_loss": 0.1564684808254242,
      "step": 430,
      "train_runtime": 63968.3408,
      "train_tokens_per_second": 1041.806
    },
    {
      "epoch": 0.18477222045237338,
      "grad_norm": 0.6248931884765625,
      "learning_rate": 4.592580116760858e-06,
      "logits/chosen": -1.5748494863510132,
      "logits/rejected": -1.574177861213684,
      "logps/chosen": -0.2759421765804291,
      "logps/rejected": -0.5249406099319458,
      "loss": 0.3151,
      "num_input_tokens_seen": 67524544,
      "odds_ratio_loss": 0.3916897773742676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.027594221755862236,
      "rewards/margins": 0.024899842217564583,
      "rewards/rejected": -0.05249406024813652,
      "sft_loss": 0.2759421765804291,
      "step": 435,
      "train_runtime": 64868.8848,
      "train_tokens_per_second": 1040.939
    },
    {
      "epoch": 0.18689603907826272,
      "grad_norm": 0.6337681412696838,
      "learning_rate": 4.583409779118876e-06,
      "logits/chosen": -1.4300365447998047,
      "logits/rejected": -1.428247332572937,
      "logps/chosen": -0.21333053708076477,
      "logps/rejected": -0.5245060920715332,
      "loss": 0.2453,
      "num_input_tokens_seen": 68211456,
      "odds_ratio_loss": 0.31961479783058167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.021333051845431328,
      "rewards/margins": 0.031117552891373634,
      "rewards/rejected": -0.05245060846209526,
      "sft_loss": 0.21333053708076477,
      "step": 440,
      "train_runtime": 65516.1551,
      "train_tokens_per_second": 1041.139
    },
    {
      "epoch": 0.18901985770415206,
      "grad_norm": 0.4188646078109741,
      "learning_rate": 4.574146751874473e-06,
      "logits/chosen": -1.5008445978164673,
      "logits/rejected": -1.4969948530197144,
      "logps/chosen": -0.16804303228855133,
      "logps/rejected": -0.5000943541526794,
      "loss": 0.1979,
      "num_input_tokens_seen": 68890384,
      "odds_ratio_loss": 0.2989177703857422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.016804303973913193,
      "rewards/margins": 0.03320513665676117,
      "rewards/rejected": -0.05000944063067436,
      "sft_loss": 0.16804303228855133,
      "step": 445,
      "train_runtime": 66130.8719,
      "train_tokens_per_second": 1041.728
    },
    {
      "epoch": 0.19114367633004142,
      "grad_norm": 0.4648490846157074,
      "learning_rate": 4.564791447133956e-06,
      "logits/chosen": -1.47642183303833,
      "logits/rejected": -1.4757719039916992,
      "logps/chosen": -0.22468982636928558,
      "logps/rejected": -0.5398010611534119,
      "loss": 0.2582,
      "num_input_tokens_seen": 69575168,
      "odds_ratio_loss": 0.3350237309932709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.022468984127044678,
      "rewards/margins": 0.03151111677289009,
      "rewards/rejected": -0.05398010089993477,
      "sft_loss": 0.22468982636928558,
      "step": 450,
      "train_runtime": 66765.2455,
      "train_tokens_per_second": 1042.087
    },
    {
      "epoch": 0.19326749495593076,
      "grad_norm": 0.39382052421569824,
      "learning_rate": 4.555344281109001e-06,
      "logits/chosen": -1.4993313550949097,
      "logits/rejected": -1.4935619831085205,
      "logps/chosen": -0.20984461903572083,
      "logps/rejected": -0.5598115921020508,
      "loss": 0.2443,
      "num_input_tokens_seen": 70369312,
      "odds_ratio_loss": 0.3441499173641205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.020984461531043053,
      "rewards/margins": 0.034996699541807175,
      "rewards/rejected": -0.05598115921020508,
      "sft_loss": 0.20984461903572083,
      "step": 455,
      "train_runtime": 67537.0837,
      "train_tokens_per_second": 1041.936
    },
    {
      "epoch": 0.1953913135818201,
      "grad_norm": 0.4138486683368683,
      "learning_rate": 4.545805674098136e-06,
      "logits/chosen": -1.5723291635513306,
      "logits/rejected": -1.569050669670105,
      "logps/chosen": -0.22112448513507843,
      "logps/rejected": -0.48221856355667114,
      "loss": 0.2573,
      "num_input_tokens_seen": 71141408,
      "odds_ratio_loss": 0.36147814989089966,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.022112449631094933,
      "rewards/margins": 0.026109402999281883,
      "rewards/rejected": -0.048221852630376816,
      "sft_loss": 0.22112448513507843,
      "step": 460,
      "train_runtime": 68267.9601,
      "train_tokens_per_second": 1042.091
    },
    {
      "epoch": 0.19751513220770947,
      "grad_norm": 0.38689109683036804,
      "learning_rate": 4.536176050468042e-06,
      "logits/chosen": -1.4762446880340576,
      "logits/rejected": -1.4698266983032227,
      "logps/chosen": -0.2256724089384079,
      "logps/rejected": -0.5008609890937805,
      "loss": 0.2598,
      "num_input_tokens_seen": 71914576,
      "odds_ratio_loss": 0.34122857451438904,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.02256724052131176,
      "rewards/margins": 0.027518857270479202,
      "rewards/rejected": -0.05008609965443611,
      "sft_loss": 0.2256724089384079,
      "step": 465,
      "train_runtime": 69033.6971,
      "train_tokens_per_second": 1041.731
    },
    {
      "epoch": 0.1996389508335988,
      "grad_norm": 0.35828182101249695,
      "learning_rate": 4.52645583863467e-06,
      "logits/chosen": -1.4943541288375854,
      "logits/rejected": -1.4925615787506104,
      "logps/chosen": -0.140981525182724,
      "logps/rejected": -0.4300973415374756,
      "loss": 0.1729,
      "num_input_tokens_seen": 72562432,
      "odds_ratio_loss": 0.31914106011390686,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0140981525182724,
      "rewards/margins": 0.028911584988236427,
      "rewards/rejected": -0.04300973564386368,
      "sft_loss": 0.140981525182724,
      "step": 470,
      "train_runtime": 69586.7703,
      "train_tokens_per_second": 1042.762
    },
    {
      "epoch": 0.20176276945948815,
      "grad_norm": 0.490843266248703,
      "learning_rate": 4.516645471044188e-06,
      "logits/chosen": -1.5066670179367065,
      "logits/rejected": -1.503004550933838,
      "logps/chosen": -0.22511906921863556,
      "logps/rejected": -0.5109016299247742,
      "loss": 0.2606,
      "num_input_tokens_seen": 73322048,
      "odds_ratio_loss": 0.3551252484321594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.022511905059218407,
      "rewards/margins": 0.028578251600265503,
      "rewards/rejected": -0.05109015852212906,
      "sft_loss": 0.22511906921863556,
      "step": 475,
      "train_runtime": 70312.7719,
      "train_tokens_per_second": 1042.798
    },
    {
      "epoch": 0.20388658808537752,
      "grad_norm": 0.38893041014671326,
      "learning_rate": 4.506745384153734e-06,
      "logits/chosen": -1.5081838369369507,
      "logits/rejected": -1.5050203800201416,
      "logps/chosen": -0.16327860951423645,
      "logps/rejected": -0.4427843987941742,
      "loss": 0.1961,
      "num_input_tokens_seen": 74058176,
      "odds_ratio_loss": 0.3279661238193512,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.016327861696481705,
      "rewards/margins": 0.027950583025813103,
      "rewards/rejected": -0.04427844285964966,
      "sft_loss": 0.16327860951423645,
      "step": 480,
      "train_runtime": 70977.6036,
      "train_tokens_per_second": 1043.402
    },
    {
      "epoch": 0.20601040671126686,
      "grad_norm": 0.4462454617023468,
      "learning_rate": 4.496756018412002e-06,
      "logits/chosen": -1.529907464981079,
      "logits/rejected": -1.524214506149292,
      "logps/chosen": -0.19855861365795135,
      "logps/rejected": -0.47792139649391174,
      "loss": 0.2304,
      "num_input_tokens_seen": 74883088,
      "odds_ratio_loss": 0.31843897700309753,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.019855860620737076,
      "rewards/margins": 0.02793627791106701,
      "rewards/rejected": -0.047792140394449234,
      "sft_loss": 0.19855861365795135,
      "step": 485,
      "train_runtime": 71800.0988,
      "train_tokens_per_second": 1042.939
    },
    {
      "epoch": 0.2081342253371562,
      "grad_norm": 0.3345007300376892,
      "learning_rate": 4.486677818239647e-06,
      "logits/chosen": -1.503440260887146,
      "logits/rejected": -1.4981225728988647,
      "logps/chosen": -0.19868183135986328,
      "logps/rejected": -0.5078573822975159,
      "loss": 0.2324,
      "num_input_tokens_seen": 75590736,
      "odds_ratio_loss": 0.3367151618003845,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.019868183881044388,
      "rewards/margins": 0.030917558819055557,
      "rewards/rejected": -0.050785746425390244,
      "sft_loss": 0.19868183135986328,
      "step": 490,
      "train_runtime": 72433.4928,
      "train_tokens_per_second": 1043.588
    },
    {
      "epoch": 0.21025804396304557,
      "grad_norm": 0.3422100245952606,
      "learning_rate": 4.476511232009515e-06,
      "logits/chosen": -1.4880125522613525,
      "logits/rejected": -1.4798282384872437,
      "logps/chosen": -0.22778700292110443,
      "logps/rejected": -0.5316972732543945,
      "loss": 0.2617,
      "num_input_tokens_seen": 76418896,
      "odds_ratio_loss": 0.33909526467323303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.022778699174523354,
      "rewards/margins": 0.030391031876206398,
      "rewards/rejected": -0.05316972732543945,
      "sft_loss": 0.22778700292110443,
      "step": 495,
      "train_runtime": 73238.1536,
      "train_tokens_per_second": 1043.43
    },
    {
      "epoch": 0.2123818625889349,
      "grad_norm": 0.31238770484924316,
      "learning_rate": 4.466256712026688e-06,
      "logits/chosen": -1.5578067302703857,
      "logits/rejected": -1.55014169216156,
      "logps/chosen": -0.1827155202627182,
      "logps/rejected": -0.5963243842124939,
      "loss": 0.2133,
      "num_input_tokens_seen": 77321888,
      "odds_ratio_loss": 0.30620744824409485,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01827155239880085,
      "rewards/margins": 0.04136088490486145,
      "rewards/rejected": -0.05963243171572685,
      "sft_loss": 0.1827155202627182,
      "step": 500,
      "train_runtime": 74168.1836,
      "train_tokens_per_second": 1042.521
    },
    {
      "epoch": 0.21450568121482425,
      "grad_norm": 0.46370604634284973,
      "learning_rate": 4.455914714508369e-06,
      "logits/chosen": -1.5513077974319458,
      "logits/rejected": -1.5444039106369019,
      "logps/chosen": -0.16203901171684265,
      "logps/rejected": -0.4189639985561371,
      "loss": 0.1976,
      "num_input_tokens_seen": 78057168,
      "odds_ratio_loss": 0.35596761107444763,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.016203900799155235,
      "rewards/margins": 0.025692496448755264,
      "rewards/rejected": -0.04189639911055565,
      "sft_loss": 0.16203901171684265,
      "step": 505,
      "train_runtime": 74834.2668,
      "train_tokens_per_second": 1043.067
    },
    {
      "epoch": 0.2166294998407136,
      "grad_norm": 0.5425517559051514,
      "learning_rate": 4.44548569956358e-06,
      "logits/chosen": -1.576695442199707,
      "logits/rejected": -1.5731325149536133,
      "logps/chosen": -0.18903768062591553,
      "logps/rejected": -0.4708414077758789,
      "loss": 0.2232,
      "num_input_tokens_seen": 78898912,
      "odds_ratio_loss": 0.34121793508529663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.018903767690062523,
      "rewards/margins": 0.02818037010729313,
      "rewards/rejected": -0.04708414152264595,
      "sft_loss": 0.18903768062591553,
      "step": 510,
      "train_runtime": 75658.6636,
      "train_tokens_per_second": 1042.827
    },
    {
      "epoch": 0.21875331846660295,
      "grad_norm": 0.5388215780258179,
      "learning_rate": 4.4349701311726965e-06,
      "logits/chosen": -1.4953768253326416,
      "logits/rejected": -1.490883469581604,
      "logps/chosen": -0.19652894139289856,
      "logps/rejected": -0.4889468848705292,
      "loss": 0.23,
      "num_input_tokens_seen": 79668240,
      "odds_ratio_loss": 0.33440372347831726,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.019652891904115677,
      "rewards/margins": 0.02924179472029209,
      "rewards/rejected": -0.04889468848705292,
      "sft_loss": 0.19652894139289856,
      "step": 515,
      "train_runtime": 76386.4848,
      "train_tokens_per_second": 1042.963
    },
    {
      "epoch": 0.2208771370924923,
      "grad_norm": 0.46711787581443787,
      "learning_rate": 4.424368477166801e-06,
      "logits/chosen": -1.4765347242355347,
      "logits/rejected": -1.4677412509918213,
      "logps/chosen": -0.1996721625328064,
      "logps/rejected": -0.5868034362792969,
      "loss": 0.2314,
      "num_input_tokens_seen": 80476688,
      "odds_ratio_loss": 0.31734487414360046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0199672169983387,
      "rewards/margins": 0.03871312737464905,
      "rewards/rejected": -0.05868034437298775,
      "sft_loss": 0.1996721625328064,
      "step": 520,
      "train_runtime": 77161.1405,
      "train_tokens_per_second": 1042.969
    },
    {
      "epoch": 0.22300095571838166,
      "grad_norm": 0.35093003511428833,
      "learning_rate": 4.413681209206872e-06,
      "logits/chosen": -1.5613449811935425,
      "logits/rejected": -1.5583739280700684,
      "logps/chosen": -0.2297806292772293,
      "logps/rejected": -0.5119220018386841,
      "loss": 0.2639,
      "num_input_tokens_seen": 81415232,
      "odds_ratio_loss": 0.34125277400016785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.02297806553542614,
      "rewards/margins": 0.028214136138558388,
      "rewards/rejected": -0.05119220167398453,
      "sft_loss": 0.2297806292772293,
      "step": 525,
      "train_runtime": 78152.9966,
      "train_tokens_per_second": 1041.742
    },
    {
      "epoch": 0.225124774344271,
      "grad_norm": 0.45800235867500305,
      "learning_rate": 4.4029088027628e-06,
      "logits/chosen": -1.5018181800842285,
      "logits/rejected": -1.5004240274429321,
      "logps/chosen": -0.21481457352638245,
      "logps/rejected": -0.5728682279586792,
      "loss": 0.2438,
      "num_input_tokens_seen": 82150448,
      "odds_ratio_loss": 0.29013198614120483,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.021481456235051155,
      "rewards/margins": 0.035805363208055496,
      "rewards/rejected": -0.0572868175804615,
      "sft_loss": 0.21481457352638245,
      "step": 530,
      "train_runtime": 78862.5439,
      "train_tokens_per_second": 1041.692
    },
    {
      "epoch": 0.22724859297016034,
      "grad_norm": 0.587579071521759,
      "learning_rate": 4.392051737092231e-06,
      "logits/chosen": -1.5634466409683228,
      "logits/rejected": -1.5582525730133057,
      "logps/chosen": -0.23821821808815002,
      "logps/rejected": -0.5162791609764099,
      "loss": 0.273,
      "num_input_tokens_seen": 83041456,
      "odds_ratio_loss": 0.3475506007671356,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.023821821436285973,
      "rewards/margins": 0.02780608832836151,
      "rewards/rejected": -0.05162791535258293,
      "sft_loss": 0.23821821808815002,
      "step": 535,
      "train_runtime": 79731.4888,
      "train_tokens_per_second": 1041.514
    },
    {
      "epoch": 0.2293724115960497,
      "grad_norm": 0.2632409632205963,
      "learning_rate": 4.381110495219252e-06,
      "logits/chosen": -1.5422700643539429,
      "logits/rejected": -1.5351835489273071,
      "logps/chosen": -0.21786637604236603,
      "logps/rejected": -0.500041663646698,
      "loss": 0.2533,
      "num_input_tokens_seen": 83890976,
      "odds_ratio_loss": 0.35384315252304077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.021786635741591454,
      "rewards/margins": 0.028217529878020287,
      "rewards/rejected": -0.05000416561961174,
      "sft_loss": 0.21786637604236603,
      "step": 540,
      "train_runtime": 80580.1214,
      "train_tokens_per_second": 1041.088
    },
    {
      "epoch": 0.23149623022193905,
      "grad_norm": 0.4494136869907379,
      "learning_rate": 4.370085563912892e-06,
      "logits/chosen": -1.51874840259552,
      "logits/rejected": -1.508168339729309,
      "logps/chosen": -0.20429356396198273,
      "logps/rejected": -0.48666268587112427,
      "loss": 0.2408,
      "num_input_tokens_seen": 84781984,
      "odds_ratio_loss": 0.3650580048561096,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.020429356023669243,
      "rewards/margins": 0.028236908838152885,
      "rewards/rejected": -0.04866626486182213,
      "sft_loss": 0.20429356396198273,
      "step": 545,
      "train_runtime": 81483.2332,
      "train_tokens_per_second": 1040.484
    },
    {
      "epoch": 0.2336200488478284,
      "grad_norm": 0.36119934916496277,
      "learning_rate": 4.358977433665471e-06,
      "logits/chosen": -1.5111784934997559,
      "logits/rejected": -1.505872130393982,
      "logps/chosen": -0.21618981659412384,
      "logps/rejected": -0.49542656540870667,
      "loss": 0.2511,
      "num_input_tokens_seen": 85453456,
      "odds_ratio_loss": 0.34916362166404724,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.021618982776999474,
      "rewards/margins": 0.027923673391342163,
      "rewards/rejected": -0.049542661756277084,
      "sft_loss": 0.21618981659412384,
      "step": 550,
      "train_runtime": 82072.357,
      "train_tokens_per_second": 1041.197
    },
    {
      "epoch": 0.23574386747371776,
      "grad_norm": 0.45707690715789795,
      "learning_rate": 4.347786598670783e-06,
      "logits/chosen": -1.490849256515503,
      "logits/rejected": -1.4699293375015259,
      "logps/chosen": -0.287718266248703,
      "logps/rejected": -0.5819355249404907,
      "loss": 0.3247,
      "num_input_tokens_seen": 86275024,
      "odds_ratio_loss": 0.3699973523616791,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.02877183072268963,
      "rewards/margins": 0.029421720653772354,
      "rewards/rejected": -0.058193545788526535,
      "sft_loss": 0.287718266248703,
      "step": 555,
      "train_runtime": 82819.6272,
      "train_tokens_per_second": 1041.722
    },
    {
      "epoch": 0.2378676860996071,
      "grad_norm": 0.412557989358902,
      "learning_rate": 4.336513556802101e-06,
      "logits/chosen": -1.532092571258545,
      "logits/rejected": -1.5297744274139404,
      "logps/chosen": -0.20612530410289764,
      "logps/rejected": -0.570284903049469,
      "loss": 0.2358,
      "num_input_tokens_seen": 87072768,
      "odds_ratio_loss": 0.29628607630729675,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.020612528547644615,
      "rewards/margins": 0.036415960639715195,
      "rewards/rejected": -0.05702848359942436,
      "sft_loss": 0.20612530410289764,
      "step": 560,
      "train_runtime": 83591.1379,
      "train_tokens_per_second": 1041.651
    },
    {
      "epoch": 0.23999150472549644,
      "grad_norm": 0.5773653388023376,
      "learning_rate": 4.325158809590028e-06,
      "logits/chosen": -1.4484983682632446,
      "logits/rejected": -1.443122386932373,
      "logps/chosen": -0.2014315128326416,
      "logps/rejected": -0.4770578444004059,
      "loss": 0.2343,
      "num_input_tokens_seen": 87758432,
      "odds_ratio_loss": 0.3283328115940094,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.02014315314590931,
      "rewards/margins": 0.02756263129413128,
      "rewards/rejected": -0.04770578071475029,
      "sft_loss": 0.2014315128326416,
      "step": 565,
      "train_runtime": 84208.2102,
      "train_tokens_per_second": 1042.16
    },
    {
      "epoch": 0.2421153233513858,
      "grad_norm": 0.4461754262447357,
      "learning_rate": 4.313722862200194e-06,
      "logits/chosen": -1.4757493734359741,
      "logits/rejected": -1.4763233661651611,
      "logps/chosen": -0.20637857913970947,
      "logps/rejected": -0.7389758229255676,
      "loss": 0.2329,
      "num_input_tokens_seen": 88551648,
      "odds_ratio_loss": 0.2655881345272064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.020637856796383858,
      "rewards/margins": 0.05325973033905029,
      "rewards/rejected": -0.0738975927233696,
      "sft_loss": 0.20637857913970947,
      "step": 570,
      "train_runtime": 85007.5288,
      "train_tokens_per_second": 1041.692
    },
    {
      "epoch": 0.24423914197727514,
      "grad_norm": 0.29294300079345703,
      "learning_rate": 4.3022062234107655e-06,
      "logits/chosen": -1.4706867933273315,
      "logits/rejected": -1.4620670080184937,
      "logps/chosen": -0.19137442111968994,
      "logps/rejected": -0.5528604388237,
      "loss": 0.2205,
      "num_input_tokens_seen": 89428384,
      "odds_ratio_loss": 0.29116758704185486,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.019137442111968994,
      "rewards/margins": 0.03614859655499458,
      "rewards/rejected": -0.05528603866696358,
      "sft_loss": 0.19137442111968994,
      "step": 575,
      "train_runtime": 85895.3396,
      "train_tokens_per_second": 1041.132
    },
    {
      "epoch": 0.24636296060316448,
      "grad_norm": 0.4504767060279846,
      "learning_rate": 4.290609405589827e-06,
      "logits/chosen": -1.4801222085952759,
      "logits/rejected": -1.4758281707763672,
      "logps/chosen": -0.22295509278774261,
      "logps/rejected": -0.4333480894565582,
      "loss": 0.2632,
      "num_input_tokens_seen": 90212256,
      "odds_ratio_loss": 0.40224844217300415,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.022295504808425903,
      "rewards/margins": 0.02103930339217186,
      "rewards/rejected": -0.04333480820059776,
      "sft_loss": 0.22295509278774261,
      "step": 580,
      "train_runtime": 86672.0197,
      "train_tokens_per_second": 1040.846
    },
    {
      "epoch": 0.24848677922905385,
      "grad_norm": 0.373007595539093,
      "learning_rate": 4.278932924672573e-06,
      "logits/chosen": -1.4207862615585327,
      "logits/rejected": -1.4173411130905151,
      "logps/chosen": -0.1737605780363083,
      "logps/rejected": -0.5204710364341736,
      "loss": 0.2024,
      "num_input_tokens_seen": 90945504,
      "odds_ratio_loss": 0.2865773141384125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01737605594098568,
      "rewards/margins": 0.03467104956507683,
      "rewards/rejected": -0.052047111093997955,
      "sft_loss": 0.1737605780363083,
      "step": 585,
      "train_runtime": 87357.6827,
      "train_tokens_per_second": 1041.07
    },
    {
      "epoch": 0.25061059785494316,
      "grad_norm": 0.47345107793807983,
      "learning_rate": 4.267177300138362e-06,
      "logits/chosen": -1.4976738691329956,
      "logits/rejected": -1.4955440759658813,
      "logps/chosen": -0.19565899670124054,
      "logps/rejected": -0.553374707698822,
      "loss": 0.2259,
      "num_input_tokens_seen": 91751680,
      "odds_ratio_loss": 0.3026733994483948,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.019565900787711143,
      "rewards/margins": 0.03577156737446785,
      "rewards/rejected": -0.055337466299533844,
      "sft_loss": 0.19565899670124054,
      "step": 590,
      "train_runtime": 88126.6526,
      "train_tokens_per_second": 1041.134
    },
    {
      "epoch": 0.25273441648083256,
      "grad_norm": 0.3928413391113281,
      "learning_rate": 4.255343054987601e-06,
      "logits/chosen": -1.4836921691894531,
      "logits/rejected": -1.477555751800537,
      "logps/chosen": -0.24060700833797455,
      "logps/rejected": -0.547676682472229,
      "loss": 0.2767,
      "num_input_tokens_seen": 92582192,
      "odds_ratio_loss": 0.36110439896583557,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.024060698226094246,
      "rewards/margins": 0.030706966295838356,
      "rewards/rejected": -0.0547676682472229,
      "sft_loss": 0.24060700833797455,
      "step": 595,
      "train_runtime": 88914.2593,
      "train_tokens_per_second": 1041.252
    },
    {
      "epoch": 0.2548582351067219,
      "grad_norm": 0.49117347598075867,
      "learning_rate": 4.243430715718481e-06,
      "logits/chosen": -1.4262853860855103,
      "logits/rejected": -1.4173415899276733,
      "logps/chosen": -0.20252223312854767,
      "logps/rejected": -0.5903180241584778,
      "loss": 0.2322,
      "num_input_tokens_seen": 93338608,
      "odds_ratio_loss": 0.29716578125953674,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.020252224057912827,
      "rewards/margins": 0.03877957537770271,
      "rewards/rejected": -0.05903180316090584,
      "sft_loss": 0.20252223312854767,
      "step": 600,
      "train_runtime": 89637.0392,
      "train_tokens_per_second": 1041.295
    }
  ],
  "logging_steps": 5,
  "max_steps": 2355,
  "num_input_tokens_seen": 93338608,
  "num_train_epochs": 1,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 7.940393577912238e+18,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}