{
  "best_metric": 1.0906352996826172,
  "best_model_checkpoint": "models/llama3.2-3b-orpo-vanilla/checkpoint-10000",
  "epoch": 0.9999812632328418,
  "eval_steps": 5000,
  "global_step": 26685,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0003747353431638905,
      "grad_norm": 15.947562873068343,
      "learning_rate": 8e-07,
      "log_odds_chosen": 0.654064953327179,
      "log_odds_ratio": -0.606738269329071,
      "logits/chosen": -1.093164086341858,
      "logits/rejected": -1.0947265625,
      "logps/chosen": -2.1949219703674316,
      "logps/rejected": -2.807421922683716,
      "loss": 2.4499,
      "nll_loss": 2.3636717796325684,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21955566108226776,
      "rewards/margins": 0.0612640380859375,
      "rewards/rejected": -0.28071290254592896,
      "step": 10
    },
    {
      "epoch": 0.000749470686327781,
      "grad_norm": 8.574923100513368,
      "learning_rate": 1.6e-06,
      "log_odds_chosen": 0.4022216796875,
      "log_odds_ratio": -0.69384765625,
      "logits/chosen": -1.1767578125,
      "logits/rejected": -1.160742163658142,
      "logps/chosen": -2.328125,
      "logps/rejected": -2.6781249046325684,
      "loss": 2.2314,
      "nll_loss": 2.145703077316284,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23281249403953552,
      "rewards/margins": 0.03490600734949112,
      "rewards/rejected": -0.2676757872104645,
      "step": 20
    },
    {
      "epoch": 0.0011242060294916715,
      "grad_norm": 3.0863067046666828,
      "learning_rate": 2.4e-06,
      "log_odds_chosen": 0.37806397676467896,
      "log_odds_ratio": -0.661425769329071,
      "logits/chosen": -1.361328125,
      "logits/rejected": -1.33203125,
      "logps/chosen": -1.7742187976837158,
      "logps/rejected": -2.115234375,
      "loss": 1.8711,
      "nll_loss": 1.827734351158142,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.177490234375,
      "rewards/margins": 0.03405151516199112,
      "rewards/rejected": -0.21147461235523224,
      "step": 30
    },
    {
      "epoch": 0.001498941372655562,
      "grad_norm": 2.521355163289798,
      "learning_rate": 3.2e-06,
      "log_odds_chosen": 0.2977539002895355,
      "log_odds_ratio": -0.6693359613418579,
      "logits/chosen": -1.154882788658142,
      "logits/rejected": -1.1580078601837158,
      "logps/chosen": -1.5261719226837158,
      "logps/rejected": -1.7666015625,
      "loss": 1.6765,
      "nll_loss": 1.552343726158142,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.15239258110523224,
      "rewards/margins": 0.02413635328412056,
      "rewards/rejected": -0.1766357421875,
      "step": 40
    },
    {
      "epoch": 0.0018736767158194524,
      "grad_norm": 2.3577795238172334,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.4524902403354645,
      "log_odds_ratio": -0.584179699420929,
      "logits/chosen": -1.1287109851837158,
      "logits/rejected": -1.048437476158142,
      "logps/chosen": -1.3828125,
      "logps/rejected": -1.740625023841858,
      "loss": 1.601,
      "nll_loss": 1.4406249523162842,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.13825683295726776,
      "rewards/margins": 0.03587036207318306,
      "rewards/rejected": -0.174072265625,
      "step": 50
    },
    {
      "epoch": 0.002248412058983343,
      "grad_norm": 2.4050083050174327,
      "learning_rate": 4.8e-06,
      "log_odds_chosen": 0.3501525819301605,
      "log_odds_ratio": -0.6087890863418579,
      "logits/chosen": -1.0193359851837158,
      "logits/rejected": -0.989453136920929,
      "logps/chosen": -1.365234375,
      "logps/rejected": -1.644140601158142,
      "loss": 1.5768,
      "nll_loss": 1.5324218273162842,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.13657227158546448,
      "rewards/margins": 0.02790222130715847,
      "rewards/rejected": -0.16450195014476776,
      "step": 60
    },
    {
      "epoch": 0.0026231474021472336,
      "grad_norm": 2.490536967687583,
      "learning_rate": 5.6e-06,
      "log_odds_chosen": 0.4223876893520355,
      "log_odds_ratio": -0.5965820550918579,
      "logits/chosen": -1.1091797351837158,
      "logits/rejected": -1.0451171398162842,
      "logps/chosen": -1.2785155773162842,
      "logps/rejected": -1.620703101158142,
      "loss": 1.537,
      "nll_loss": 1.4445312023162842,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.12788085639476776,
      "rewards/margins": 0.03421325609087944,
      "rewards/rejected": -0.16206054389476776,
      "step": 70
    },
    {
      "epoch": 0.002997882745311124,
      "grad_norm": 2.1576490507819033,
      "learning_rate": 6.4e-06,
      "log_odds_chosen": 0.21810302138328552,
      "log_odds_ratio": -0.6728515625,
      "logits/chosen": -1.1623046398162842,
      "logits/rejected": -1.1437499523162842,
      "logps/chosen": -1.278906226158142,
      "logps/rejected": -1.451757788658142,
      "loss": 1.4786,
      "nll_loss": 1.431249976158142,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.1278076171875,
      "rewards/margins": 0.01741790771484375,
      "rewards/rejected": -0.14516600966453552,
      "step": 80
    },
    {
      "epoch": 0.0033726180884750146,
      "grad_norm": 2.4593856177519555,
      "learning_rate": 7.2e-06,
      "log_odds_chosen": 0.21759644150733948,
      "log_odds_ratio": -0.6640625,
      "logits/chosen": -1.1853516101837158,
      "logits/rejected": -1.1466796398162842,
      "logps/chosen": -1.224023461341858,
      "logps/rejected": -1.404882788658142,
      "loss": 1.4774,
      "nll_loss": 1.3464844226837158,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.12241210788488388,
      "rewards/margins": 0.01810913160443306,
      "rewards/rejected": -0.14052733778953552,
      "step": 90
    },
    {
      "epoch": 0.003747353431638905,
      "grad_norm": 2.219899533858184,
      "learning_rate": 8e-06,
      "log_odds_chosen": 0.2554931640625,
      "log_odds_ratio": -0.6883789300918579,
      "logits/chosen": -1.1384766101837158,
      "logits/rejected": -1.1027343273162842,
      "logps/chosen": -1.1847655773162842,
      "logps/rejected": -1.391210913658142,
      "loss": 1.4138,
      "nll_loss": 1.3527343273162842,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.11850585788488388,
      "rewards/margins": 0.02066192589700222,
      "rewards/rejected": -0.13916015625,
      "step": 100
    },
    {
      "epoch": 0.004122088774802795,
      "grad_norm": 1.852579200867629,
      "learning_rate": 7.627700713964738e-06,
      "log_odds_chosen": 0.25755006074905396,
      "log_odds_ratio": -0.640625,
      "logits/chosen": -1.126953125,
      "logits/rejected": -1.0734374523162842,
      "logps/chosen": -1.110937476158142,
      "logps/rejected": -1.309960961341858,
      "loss": 1.3919,
      "nll_loss": 1.373046875,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.111083984375,
      "rewards/margins": 0.01993102952837944,
      "rewards/rejected": -0.13095703721046448,
      "step": 110
    },
    {
      "epoch": 0.004496824117966686,
      "grad_norm": 1.593936462008365,
      "learning_rate": 7.3029674334022146e-06,
      "log_odds_chosen": 0.20570068061351776,
      "log_odds_ratio": -0.6670898199081421,
      "logits/chosen": -1.1064453125,
      "logits/rejected": -1.05859375,
      "logps/chosen": -1.0958983898162842,
      "logps/rejected": -1.245703101158142,
      "loss": 1.3777,
      "nll_loss": 1.3087890148162842,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.10952148586511612,
      "rewards/margins": 0.01487884484231472,
      "rewards/rejected": -0.12446288764476776,
      "step": 120
    },
    {
      "epoch": 0.0048715594611305766,
      "grad_norm": 2.1893982742060714,
      "learning_rate": 7.016464154456233e-06,
      "log_odds_chosen": 0.28863525390625,
      "log_odds_ratio": -0.646484375,
      "logits/chosen": -1.0574219226837158,
      "logits/rejected": -1.017578125,
      "logps/chosen": -1.0056641101837158,
      "logps/rejected": -1.229101538658142,
      "loss": 1.3646,
      "nll_loss": 1.319921851158142,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1005859375,
      "rewards/margins": 0.02226715162396431,
      "rewards/rejected": -0.1229248046875,
      "step": 130
    },
    {
      "epoch": 0.005246294804294467,
      "grad_norm": 1.5010589207313372,
      "learning_rate": 6.7612340378281325e-06,
      "log_odds_chosen": 0.19410400092601776,
      "log_odds_ratio": -0.6649414300918579,
      "logits/chosen": -1.009179711341858,
      "logits/rejected": -0.966015636920929,
      "logps/chosen": -1.0439453125,
      "logps/rejected": -1.1943359375,
      "loss": 1.3018,
      "nll_loss": 1.322656273841858,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.10434570163488388,
      "rewards/margins": 0.015139770694077015,
      "rewards/rejected": -0.11953125149011612,
      "step": 140
    },
    {
      "epoch": 0.005621030147458357,
      "grad_norm": 1.459953413851116,
      "learning_rate": 6.531972647421809e-06,
      "log_odds_chosen": 0.23885497450828552,
      "log_odds_ratio": -0.660449206829071,
      "logits/chosen": -1.020898461341858,
      "logits/rejected": -1.0001952648162842,
      "logps/chosen": -1.0382812023162842,
      "logps/rejected": -1.235742211341858,
      "loss": 1.3189,
      "nll_loss": 1.289453148841858,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1038818359375,
      "rewards/margins": 0.01963195763528347,
      "rewards/rejected": -0.12351074069738388,
      "step": 150
    },
    {
      "epoch": 0.005995765490622248,
      "grad_norm": 1.8650435820092022,
      "learning_rate": 6.3245553203367575e-06,
      "log_odds_chosen": 0.3742431700229645,
      "log_odds_ratio": -0.615039050579071,
      "logits/chosen": -0.945117175579071,
      "logits/rejected": -0.877734363079071,
      "logps/chosen": -0.9857422113418579,
      "logps/rejected": -1.2919921875,
      "loss": 1.3173,
      "nll_loss": 1.3076171875,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.09855957329273224,
      "rewards/margins": 0.03058319166302681,
      "rewards/rejected": -0.12917479872703552,
      "step": 160
    },
    {
      "epoch": 0.0063705008337861385,
      "grad_norm": 1.5339619556767616,
      "learning_rate": 6.135719910778963e-06,
      "log_odds_chosen": 0.2978149354457855,
      "log_odds_ratio": -0.6446288824081421,
      "logits/chosen": -0.887499988079071,
      "logits/rejected": -0.8353515863418579,
      "logps/chosen": -0.9712890386581421,
      "logps/rejected": -1.1984374523162842,
      "loss": 1.2965,
      "nll_loss": 1.162695288658142,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.09714355319738388,
      "rewards/margins": 0.02264404296875,
      "rewards/rejected": -0.11984863132238388,
      "step": 170
    },
    {
      "epoch": 0.006745236176950029,
      "grad_norm": 1.8025966705267202,
      "learning_rate": 5.962847939999439e-06,
      "log_odds_chosen": 0.26649171113967896,
      "log_odds_ratio": -0.66796875,
      "logits/chosen": -0.9453125,
      "logits/rejected": -0.8892577886581421,
      "logps/chosen": -0.9580078125,
      "logps/rejected": -1.1531250476837158,
      "loss": 1.2829,
      "nll_loss": 1.2109375,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.09572754055261612,
      "rewards/margins": 0.01952667161822319,
      "rewards/rejected": -0.11533202975988388,
      "step": 180
    },
    {
      "epoch": 0.00711997152011392,
      "grad_norm": 1.8335803791368985,
      "learning_rate": 5.803810000880094e-06,
      "log_odds_chosen": 0.2651611268520355,
      "log_odds_ratio": -0.642382800579071,
      "logits/chosen": -0.9560546875,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.942578136920929,
      "logps/rejected": -1.141015648841858,
      "loss": 1.2414,
      "nll_loss": 1.1466796398162842,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.09423828125,
      "rewards/margins": 0.01982879638671875,
      "rewards/rejected": -0.11411132663488388,
      "step": 190
    },
    {
      "epoch": 0.00749470686327781,
      "grad_norm": 1.632003980060777,
      "learning_rate": 5.65685424949238e-06,
      "log_odds_chosen": 0.3052307069301605,
      "log_odds_ratio": -0.6454101800918579,
      "logits/chosen": -0.961718738079071,
      "logits/rejected": -0.904101550579071,
      "logps/chosen": -0.907421886920929,
      "logps/rejected": -1.130859375,
      "loss": 1.2415,
      "nll_loss": 1.1828124523162842,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.09072265774011612,
      "rewards/margins": 0.02233123779296875,
      "rewards/rejected": -0.113037109375,
      "step": 200
    },
    {
      "epoch": 0.007869442206441701,
      "grad_norm": 1.5666596825663737,
      "learning_rate": 5.5205244747388325e-06,
      "log_odds_chosen": 0.28504639863967896,
      "log_odds_ratio": -0.640332043170929,
      "logits/chosen": -0.9390624761581421,
      "logits/rejected": -0.889843761920929,
      "logps/chosen": -0.9599609375,
      "logps/rejected": -1.1591796875,
      "loss": 1.2385,
      "nll_loss": 1.237890601158142,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.09602050483226776,
      "rewards/margins": 0.02000274695456028,
      "rewards/rejected": -0.11601562798023224,
      "step": 210
    },
    {
      "epoch": 0.00824417754960559,
      "grad_norm": 1.481150575009195,
      "learning_rate": 5.393598899705936e-06,
      "log_odds_chosen": 0.23063965141773224,
      "log_odds_ratio": -0.66943359375,
      "logits/chosen": -0.9779297113418579,
      "logits/rejected": -0.9117187261581421,
      "logps/chosen": -0.890820324420929,
      "logps/rejected": -1.071874976158142,
      "loss": 1.2353,
      "nll_loss": 1.120507836341858,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.08906249701976776,
      "rewards/margins": 0.018155669793486595,
      "rewards/rejected": -0.1072998046875,
      "step": 220
    },
    {
      "epoch": 0.008618912892769481,
      "grad_norm": 1.454025949871822,
      "learning_rate": 5.275043787166296e-06,
      "log_odds_chosen": 0.2831787168979645,
      "log_odds_ratio": -0.635546863079071,
      "logits/chosen": -0.9742187261581421,
      "logits/rejected": -0.9371093511581421,
      "logps/chosen": -0.9164062738418579,
      "logps/rejected": -1.122656226158142,
      "loss": 1.2125,
      "nll_loss": 1.1785156726837158,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.09162597358226776,
      "rewards/margins": 0.02065277099609375,
      "rewards/rejected": -0.11228027194738388,
      "step": 230
    },
    {
      "epoch": 0.008993648235933372,
      "grad_norm": 1.4895735212646553,
      "learning_rate": 5.163977794943223e-06,
      "log_odds_chosen": 0.3818725645542145,
      "log_odds_ratio": -0.6475585699081421,
      "logits/chosen": -0.944531261920929,
      "logits/rejected": -0.9068359136581421,
      "logps/chosen": -0.8958984613418579,
      "logps/rejected": -1.173437476158142,
      "loss": 1.1955,
      "nll_loss": 1.1552734375,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08957519382238388,
      "rewards/margins": 0.02775115892291069,
      "rewards/rejected": -0.11738280951976776,
      "step": 240
    },
    {
      "epoch": 0.009368383579097262,
      "grad_norm": 1.4401104748304867,
      "learning_rate": 5.059644256269407e-06,
      "log_odds_chosen": 0.3070312440395355,
      "log_odds_ratio": -0.6292968988418579,
      "logits/chosen": -0.9068359136581421,
      "logits/rejected": -0.871874988079071,
      "logps/chosen": -0.827343761920929,
      "logps/rejected": -1.034570336341858,
      "loss": 1.2147,
      "nll_loss": 1.1462891101837158,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08277587592601776,
      "rewards/margins": 0.02069854736328125,
      "rewards/rejected": -0.10349120944738388,
      "step": 250
    },
    {
      "epoch": 0.009743118922261153,
      "grad_norm": 1.3172076160474726,
      "learning_rate": 4.961389383568338e-06,
      "log_odds_chosen": 0.24478760361671448,
      "log_odds_ratio": -0.6695312261581421,
      "logits/chosen": -0.870312511920929,
      "logits/rejected": -0.811328113079071,
      "logps/chosen": -0.8246093988418579,
      "logps/rejected": -1.001953125,
      "loss": 1.1985,
      "nll_loss": 1.165429711341858,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.08247070014476776,
      "rewards/margins": 0.01765899732708931,
      "rewards/rejected": -0.10014648735523224,
      "step": 260
    },
    {
      "epoch": 0.010117854265425044,
      "grad_norm": 1.3101096685391262,
      "learning_rate": 4.8686449556014755e-06,
      "log_odds_chosen": 0.28321534395217896,
      "log_odds_ratio": -0.659960925579071,
      "logits/chosen": -0.864453136920929,
      "logits/rejected": -0.8705078363418579,
      "logps/chosen": -0.8310546875,
      "logps/rejected": -1.046484351158142,
      "loss": 1.1739,
      "nll_loss": 1.064550757408142,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08310546725988388,
      "rewards/margins": 0.021508026868104935,
      "rewards/rejected": -0.10463867336511612,
      "step": 270
    },
    {
      "epoch": 0.010492589608588935,
      "grad_norm": 1.5471970503663046,
      "learning_rate": 4.780914437337574e-06,
      "log_odds_chosen": 0.19062499701976776,
      "log_odds_ratio": -0.687207043170929,
      "logits/chosen": -0.944531261920929,
      "logits/rejected": -0.90234375,
      "logps/chosen": -0.8578125238418579,
      "logps/rejected": -1.0148437023162842,
      "loss": 1.1805,
      "nll_loss": 1.072656273841858,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08579101413488388,
      "rewards/margins": 0.01566619798541069,
      "rewards/rejected": -0.10148926079273224,
      "step": 280
    },
    {
      "epoch": 0.010867324951752825,
      "grad_norm": 1.3516633096108386,
      "learning_rate": 4.697761756117627e-06,
      "log_odds_chosen": 0.1553955078125,
      "log_odds_ratio": -0.6905273199081421,
      "logits/chosen": -0.9898437261581421,
      "logits/rejected": -0.9330078363418579,
      "logps/chosen": -0.798632800579071,
      "logps/rejected": -0.9154297113418579,
      "loss": 1.1868,
      "nll_loss": 1.0597655773162842,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07982178032398224,
      "rewards/margins": 0.011699676513671875,
      "rewards/rejected": -0.09150390326976776,
      "step": 290
    },
    {
      "epoch": 0.011242060294916714,
      "grad_norm": 1.4101628661970425,
      "learning_rate": 4.618802153517006e-06,
      "log_odds_chosen": 0.2838989198207855,
      "log_odds_ratio": -0.662890613079071,
      "logits/chosen": -0.9251953363418579,
      "logits/rejected": -0.8960937261581421,
      "logps/chosen": -0.834765613079071,
      "logps/rejected": -1.048828125,
      "loss": 1.197,
      "nll_loss": 1.160742163658142,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08352050930261612,
      "rewards/margins": 0.0213470458984375,
      "rewards/rejected": -0.10488281399011612,
      "step": 300
    },
    {
      "epoch": 0.011616795638080605,
      "grad_norm": 1.359075362398179,
      "learning_rate": 4.543694673976518e-06,
      "log_odds_chosen": 0.4194091856479645,
      "log_odds_ratio": -0.5997070074081421,
      "logits/chosen": -0.9457031488418579,
      "logits/rejected": -0.830859363079071,
      "logps/chosen": -0.804492175579071,
      "logps/rejected": -1.1005859375,
      "loss": 1.2131,
      "nll_loss": 1.055078148841858,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.08052978664636612,
      "rewards/margins": 0.02959594689309597,
      "rewards/rejected": -0.11000976711511612,
      "step": 310
    },
    {
      "epoch": 0.011991530981244496,
      "grad_norm": 2.7696423637952705,
      "learning_rate": 4.472135954999579e-06,
      "log_odds_chosen": 0.34801024198532104,
      "log_odds_ratio": -0.6297851800918579,
      "logits/chosen": -0.8392578363418579,
      "logits/rejected": -0.810546875,
      "logps/chosen": -0.841992199420929,
      "logps/rejected": -1.0880858898162842,
      "loss": 1.2213,
      "nll_loss": 1.1837890148162842,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.08419189602136612,
      "rewards/margins": 0.02464446984231472,
      "rewards/rejected": -0.10895995795726776,
      "step": 320
    },
    {
      "epoch": 0.012366266324408386,
      "grad_norm": 1.2956165247930922,
      "learning_rate": 4.403855060505443e-06,
      "log_odds_chosen": 0.3248535096645355,
      "log_odds_ratio": -0.664355456829071,
      "logits/chosen": -0.8486328125,
      "logits/rejected": -0.817187488079071,
      "logps/chosen": -0.8564453125,
      "logps/rejected": -1.085351586341858,
      "loss": 1.1958,
      "nll_loss": 1.155664086341858,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.085693359375,
      "rewards/margins": 0.02290649339556694,
      "rewards/rejected": -0.1085205078125,
      "step": 330
    },
    {
      "epoch": 0.012741001667572277,
      "grad_norm": 1.3660986388202874,
      "learning_rate": 4.338609156373123e-06,
      "log_odds_chosen": 0.324951171875,
      "log_odds_ratio": -0.626953125,
      "logits/chosen": -0.888476550579071,
      "logits/rejected": -0.834765613079071,
      "logps/chosen": -0.8021484613418579,
      "logps/rejected": -1.03125,
      "loss": 1.1743,
      "nll_loss": 1.1554687023162842,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.08023681491613388,
      "rewards/margins": 0.022937774658203125,
      "rewards/rejected": -0.10312499850988388,
      "step": 340
    },
    {
      "epoch": 0.013115737010736168,
      "grad_norm": 1.3074385998767046,
      "learning_rate": 4.27617987059879e-06,
      "log_odds_chosen": 0.2587646543979645,
      "log_odds_ratio": -0.6724609136581421,
      "logits/chosen": -0.9013671875,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.8306640386581421,
      "logps/rejected": -1.001953125,
      "loss": 1.1839,
      "nll_loss": 1.1845703125,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08305664360523224,
      "rewards/margins": 0.01712646521627903,
      "rewards/rejected": -0.10017089545726776,
      "step": 350
    },
    {
      "epoch": 0.013490472353900058,
      "grad_norm": 1.3413936378871691,
      "learning_rate": 4.216370213557839e-06,
      "log_odds_chosen": 0.32062989473342896,
      "log_odds_ratio": -0.6429687738418579,
      "logits/chosen": -0.889453113079071,
      "logits/rejected": -0.865429699420929,
      "logps/chosen": -0.8216797113418579,
      "logps/rejected": -1.0382812023162842,
      "loss": 1.1823,
      "nll_loss": 1.1277344226837158,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.08212890475988388,
      "rewards/margins": 0.02170104905962944,
      "rewards/rejected": -0.10385742038488388,
      "step": 360
    },
    {
      "epoch": 0.01386520769706395,
      "grad_norm": 1.3765239530954547,
      "learning_rate": 4.15900195928029e-06,
      "log_odds_chosen": 0.23958739638328552,
      "log_odds_ratio": -0.6693359613418579,
      "logits/chosen": -0.8824218511581421,
      "logits/rejected": -0.8343750238418579,
      "logps/chosen": -0.7925781011581421,
      "logps/rejected": -0.9560546875,
      "loss": 1.177,
      "nll_loss": 1.093359351158142,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07926025241613388,
      "rewards/margins": 0.0164031982421875,
      "rewards/rejected": -0.09566650539636612,
      "step": 370
    },
    {
      "epoch": 0.01423994304022784,
      "grad_norm": 1.3250918396523346,
      "learning_rate": 4.103913408340617e-06,
      "log_odds_chosen": 0.22230224311351776,
      "log_odds_ratio": -0.6712890863418579,
      "logits/chosen": -0.8882812261581421,
      "logits/rejected": -0.848828136920929,
      "logps/chosen": -0.773632824420929,
      "logps/rejected": -0.9351562261581421,
      "loss": 1.1903,
      "nll_loss": 1.0888671875,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07734374701976776,
      "rewards/margins": 0.01611633226275444,
      "rewards/rejected": -0.093505859375,
      "step": 380
    },
    {
      "epoch": 0.014614678383391729,
      "grad_norm": 1.305674287412372,
      "learning_rate": 4.050957468334666e-06,
      "log_odds_chosen": 0.21367187798023224,
      "log_odds_ratio": -0.6958984136581421,
      "logits/chosen": -0.885546863079071,
      "logits/rejected": -0.8544921875,
      "logps/chosen": -0.787109375,
      "logps/rejected": -0.9613281488418579,
      "loss": 1.1596,
      "nll_loss": 1.074609398841858,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07880859076976776,
      "rewards/margins": 0.017321014776825905,
      "rewards/rejected": -0.09608153998851776,
      "step": 390
    },
    {
      "epoch": 0.01498941372655562,
      "grad_norm": 1.2971858314266453,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.18096923828125,
      "log_odds_ratio": -0.6927734613418579,
      "logits/chosen": -0.905468761920929,
      "logits/rejected": -0.888867199420929,
      "logps/chosen": -0.841015636920929,
      "logps/rejected": -0.950976550579071,
      "loss": 1.1786,
      "nll_loss": 1.121484398841858,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.08415527641773224,
      "rewards/margins": 0.011012268252670765,
      "rewards/rejected": -0.09511718899011612,
      "step": 400
    },
    {
      "epoch": 0.01536414906971951,
      "grad_norm": 1.2817412115723037,
      "learning_rate": 3.950918386598359e-06,
      "log_odds_chosen": 0.2579589784145355,
      "log_odds_ratio": -0.65625,
      "logits/chosen": -0.9544922113418579,
      "logits/rejected": -0.8775390386581421,
      "logps/chosen": -0.807421863079071,
      "logps/rejected": -0.997851550579071,
      "loss": 1.1713,
      "nll_loss": 1.0574219226837158,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.08078613132238388,
      "rewards/margins": 0.019052887335419655,
      "rewards/rejected": -0.09982910007238388,
      "step": 410
    },
    {
      "epoch": 0.015738884412883403,
      "grad_norm": 1.3062131572564897,
      "learning_rate": 3.903600291794132e-06,
      "log_odds_chosen": 0.28492432832717896,
      "log_odds_ratio": -0.658496081829071,
      "logits/chosen": -0.8804687261581421,
      "logits/rejected": -0.8763672113418579,
      "logps/chosen": -0.797656238079071,
      "logps/rejected": -0.990234375,
      "loss": 1.1608,
      "nll_loss": 1.070898413658142,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07982178032398224,
      "rewards/margins": 0.01921386644244194,
      "rewards/rejected": -0.09903564304113388,
      "step": 420
    },
    {
      "epoch": 0.01611361975604729,
      "grad_norm": 1.285209359264166,
      "learning_rate": 3.857942577363297e-06,
      "log_odds_chosen": 0.2638183534145355,
      "log_odds_ratio": -0.6615234613418579,
      "logits/chosen": -0.943359375,
      "logits/rejected": -0.905468761920929,
      "logps/chosen": -0.8207031488418579,
      "logps/rejected": -1.010156273841858,
      "loss": 1.1817,
      "nll_loss": 1.1103515625,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08203125,
      "rewards/margins": 0.018891144543886185,
      "rewards/rejected": -0.10100097954273224,
      "step": 430
    },
    {
      "epoch": 0.01648835509921118,
      "grad_norm": 1.3828394803761586,
      "learning_rate": 3.813850356982369e-06,
      "log_odds_chosen": 0.37347412109375,
      "log_odds_ratio": -0.63330078125,
      "logits/chosen": -0.913867175579071,
      "logits/rejected": -0.854687511920929,
      "logps/chosen": -0.770312488079071,
      "logps/rejected": -1.0222656726837158,
      "loss": 1.1473,
      "nll_loss": 1.0896484851837158,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07712402194738388,
      "rewards/margins": 0.025167083367705345,
      "rewards/rejected": -0.1021728515625,
      "step": 440
    },
    {
      "epoch": 0.01686309044237507,
      "grad_norm": 1.2992193210869656,
      "learning_rate": 3.7712361663282537e-06,
      "log_odds_chosen": 0.2849365174770355,
      "log_odds_ratio": -0.653027355670929,
      "logits/chosen": -0.8681640625,
      "logits/rejected": -0.8301757574081421,
      "logps/chosen": -0.783984363079071,
      "logps/rejected": -0.9654296636581421,
      "loss": 1.1704,
      "nll_loss": 1.0255858898162842,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07839355617761612,
      "rewards/margins": 0.0181427001953125,
      "rewards/rejected": -0.09653320163488388,
      "step": 450
    },
    {
      "epoch": 0.017237825785538962,
      "grad_norm": 1.415768269212442,
      "learning_rate": 3.730019232961255e-06,
      "log_odds_chosen": 0.2916015684604645,
      "log_odds_ratio": -0.6419922113418579,
      "logits/chosen": -0.951953113079071,
      "logits/rejected": -0.916796863079071,
      "logps/chosen": -0.820507824420929,
      "logps/rejected": -1.015625,
      "loss": 1.1811,
      "nll_loss": 1.100976586341858,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.08203125,
      "rewards/margins": 0.01949768140912056,
      "rewards/rejected": -0.10161133110523224,
      "step": 460
    },
    {
      "epoch": 0.017612561128702853,
      "grad_norm": 1.299246974235602,
      "learning_rate": 3.6901248321155403e-06,
      "log_odds_chosen": 0.3186401426792145,
      "log_odds_ratio": -0.650585949420929,
      "logits/chosen": -0.938281238079071,
      "logits/rejected": -0.8999999761581421,
      "logps/chosen": -0.8374999761581421,
      "logps/rejected": -1.075781226158142,
      "loss": 1.1529,
      "nll_loss": 1.1349608898162842,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.08371581882238388,
      "rewards/margins": 0.02386627160012722,
      "rewards/rejected": -0.10759277641773224,
      "step": 470
    },
    {
      "epoch": 0.017987296471866743,
      "grad_norm": 1.3581185213611031,
      "learning_rate": 3.6514837167011073e-06,
      "log_odds_chosen": 0.3319335877895355,
      "log_odds_ratio": -0.64794921875,
      "logits/chosen": -0.946093738079071,
      "logits/rejected": -0.8919922113418579,
      "logps/chosen": -0.7818359136581421,
      "logps/rejected": -1.0185546875,
      "loss": 1.1737,
      "nll_loss": 1.078710913658142,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07814941555261612,
      "rewards/margins": 0.02377777174115181,
      "rewards/rejected": -0.10187987983226776,
      "step": 480
    },
    {
      "epoch": 0.018362031815030634,
      "grad_norm": 1.2534932282320193,
      "learning_rate": 3.6140316116210052e-06,
      "log_odds_chosen": 0.408935546875,
      "log_odds_ratio": -0.6097656488418579,
      "logits/chosen": -0.8726562261581421,
      "logits/rejected": -0.8529297113418579,
      "logps/chosen": -0.7994140386581421,
      "logps/rejected": -1.072265625,
      "loss": 1.149,
      "nll_loss": 1.0783202648162842,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07998047024011612,
      "rewards/margins": 0.02727203443646431,
      "rewards/rejected": -0.10720214992761612,
      "step": 490
    },
    {
      "epoch": 0.018736767158194525,
      "grad_norm": 1.282909684463891,
      "learning_rate": 3.5777087639996634e-06,
      "log_odds_chosen": 0.3089355528354645,
      "log_odds_ratio": -0.644824206829071,
      "logits/chosen": -0.8999999761581421,
      "logits/rejected": -0.8707031011581421,
      "logps/chosen": -0.78125,
      "logps/rejected": -0.985546886920929,
      "loss": 1.1746,
      "nll_loss": 1.1248047351837158,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07814941555261612,
      "rewards/margins": 0.02038116380572319,
      "rewards/rejected": -0.09857177734375,
      "step": 500
    },
    {
      "epoch": 0.019111502501358416,
      "grad_norm": 1.3839088430113948,
      "learning_rate": 3.5424595421603814e-06,
      "log_odds_chosen": 0.3773437440395355,
      "log_odds_ratio": -0.639355480670929,
      "logits/chosen": -0.894824206829071,
      "logits/rejected": -0.87158203125,
      "logps/chosen": -0.7720702886581421,
      "logps/rejected": -1.04296875,
      "loss": 1.15,
      "nll_loss": 1.1003906726837158,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07717285305261612,
      "rewards/margins": 0.02707672119140625,
      "rewards/rejected": -0.10429687798023224,
      "step": 510
    },
    {
      "epoch": 0.019486237844522306,
      "grad_norm": 1.3069981073285506,
      "learning_rate": 3.5082320772281165e-06,
      "log_odds_chosen": 0.2961181700229645,
      "log_odds_ratio": -0.6317383050918579,
      "logits/chosen": -0.891894519329071,
      "logits/rejected": -0.819628894329071,
      "logps/chosen": -0.7464843988418579,
      "logps/rejected": -0.9500976800918579,
      "loss": 1.1123,
      "nll_loss": 1.045312523841858,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07468261569738388,
      "rewards/margins": 0.02028808556497097,
      "rewards/rejected": -0.09494628757238388,
      "step": 520
    },
    {
      "epoch": 0.019860973187686197,
      "grad_norm": 1.4919514258676374,
      "learning_rate": 3.474977942104555e-06,
      "log_odds_chosen": 0.33204954862594604,
      "log_odds_ratio": -0.635546863079071,
      "logits/chosen": -0.9013671875,
      "logits/rejected": -0.8539062738418579,
      "logps/chosen": -0.817578136920929,
      "logps/rejected": -1.039648413658142,
      "loss": 1.1417,
      "nll_loss": 1.141210913658142,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.08175048977136612,
      "rewards/margins": 0.022249603644013405,
      "rewards/rejected": -0.10402832180261612,
      "step": 530
    },
    {
      "epoch": 0.020235708530850088,
      "grad_norm": 1.276981055003817,
      "learning_rate": 3.442651863295481e-06,
      "log_odds_chosen": 0.42182618379592896,
      "log_odds_ratio": -0.5997070074081421,
      "logits/chosen": -0.883593738079071,
      "logits/rejected": -0.8726562261581421,
      "logps/chosen": -0.788281261920929,
      "logps/rejected": -1.0740234851837158,
      "loss": 1.1475,
      "nll_loss": 1.0876953601837158,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07879638671875,
      "rewards/margins": 0.02856597863137722,
      "rewards/rejected": -0.10734863579273224,
      "step": 540
    },
    {
      "epoch": 0.02061044387401398,
      "grad_norm": 1.2526366627610184,
      "learning_rate": 3.4112114616897665e-06,
      "log_odds_chosen": 0.27076417207717896,
      "log_odds_ratio": -0.66748046875,
      "logits/chosen": -0.8685547113418579,
      "logits/rejected": -0.832812488079071,
      "logps/chosen": -0.826171875,
      "logps/rejected": -1.02734375,
      "loss": 1.1544,
      "nll_loss": 1.08984375,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0826416015625,
      "rewards/margins": 0.020046234130859375,
      "rewards/rejected": -0.10263671725988388,
      "step": 550
    },
    {
      "epoch": 0.02098517921717787,
      "grad_norm": 1.2790810930065761,
      "learning_rate": 3.3806170189140663e-06,
      "log_odds_chosen": 0.2944580018520355,
      "log_odds_ratio": -0.645214855670929,
      "logits/chosen": -0.8779296875,
      "logits/rejected": -0.830273449420929,
      "logps/chosen": -0.7818359136581421,
      "logps/rejected": -0.9859374761581421,
      "loss": 1.123,
      "nll_loss": 1.066796898841858,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07819823920726776,
      "rewards/margins": 0.02043914794921875,
      "rewards/rejected": -0.09860839694738388,
      "step": 560
    },
    {
      "epoch": 0.02135991456034176,
      "grad_norm": 1.2808999092225704,
      "learning_rate": 3.350831266333564e-06,
      "log_odds_chosen": 0.32042235136032104,
      "log_odds_ratio": -0.6615234613418579,
      "logits/chosen": -0.880078136920929,
      "logits/rejected": -0.8482421636581421,
      "logps/chosen": -0.7855468988418579,
      "logps/rejected": -0.9966796636581421,
      "loss": 1.1457,
      "nll_loss": 1.076562523841858,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07854004204273224,
      "rewards/margins": 0.02126617357134819,
      "rewards/rejected": -0.09978027641773224,
      "step": 570
    },
    {
      "epoch": 0.02173464990350565,
      "grad_norm": 1.2197299270497866,
      "learning_rate": 3.3218191941495984e-06,
      "log_odds_chosen": 0.21109619736671448,
      "log_odds_ratio": -0.6849609613418579,
      "logits/chosen": -0.919140636920929,
      "logits/rejected": -0.885546863079071,
      "logps/chosen": -0.783984363079071,
      "logps/rejected": -0.921679675579071,
      "loss": 1.1557,
      "nll_loss": 1.0330078601837158,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07840576022863388,
      "rewards/margins": 0.01381072960793972,
      "rewards/rejected": -0.09223632514476776,
      "step": 580
    },
    {
      "epoch": 0.02210938524666954,
      "grad_norm": 1.3836376020581092,
      "learning_rate": 3.293547878370473e-06,
      "log_odds_chosen": 0.20656737685203552,
      "log_odds_ratio": -0.69873046875,
      "logits/chosen": -0.904296875,
      "logits/rejected": -0.883007824420929,
      "logps/chosen": -0.812695324420929,
      "logps/rejected": -0.9468749761581421,
      "loss": 1.1451,
      "nll_loss": 1.086328148841858,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.08127441257238388,
      "rewards/margins": 0.01348876953125,
      "rewards/rejected": -0.09477539360523224,
      "step": 590
    },
    {
      "epoch": 0.02248412058983343,
      "grad_norm": 1.2867954402472304,
      "learning_rate": 3.2659863237109044e-06,
      "log_odds_chosen": 0.21937255561351776,
      "log_odds_ratio": -0.6923828125,
      "logits/chosen": -0.8648437261581421,
      "logits/rejected": -0.8207031488418579,
      "logps/chosen": -0.79296875,
      "logps/rejected": -0.959179699420929,
      "loss": 1.1327,
      "nll_loss": 1.1736328601837158,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.0792236328125,
      "rewards/margins": 0.01666259765625,
      "rewards/rejected": -0.09591064602136612,
      "step": 600
    },
    {
      "epoch": 0.02285885593299732,
      "grad_norm": 1.3345133988193374,
      "learning_rate": 3.239105320715664e-06,
      "log_odds_chosen": 0.29096680879592896,
      "log_odds_ratio": -0.6441406011581421,
      "logits/chosen": -0.818164050579071,
      "logits/rejected": -0.768750011920929,
      "logps/chosen": -0.7734375,
      "logps/rejected": -0.974609375,
      "loss": 1.1506,
      "nll_loss": 1.116601586341858,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07724609225988388,
      "rewards/margins": 0.02027587965130806,
      "rewards/rejected": -0.09758301079273224,
      "step": 610
    },
    {
      "epoch": 0.02323359127616121,
      "grad_norm": 1.287369235800961,
      "learning_rate": 3.2128773156099956e-06,
      "log_odds_chosen": 0.3392089903354645,
      "log_odds_ratio": -0.64453125,
      "logits/chosen": -0.899218738079071,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.751953125,
      "logps/rejected": -0.9765625,
      "loss": 1.1417,
      "nll_loss": 1.0322265625,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07513427734375,
      "rewards/margins": 0.02249755896627903,
      "rewards/rejected": -0.09768066555261612,
      "step": 620
    },
    {
      "epoch": 0.0236083266193251,
      "grad_norm": 1.2519290991525787,
      "learning_rate": 3.187276291558383e-06,
      "log_odds_chosen": 0.31859129667282104,
      "log_odds_ratio": -0.642578125,
      "logits/chosen": -0.966601550579071,
      "logits/rejected": -0.8892577886581421,
      "logps/chosen": -0.7740234136581421,
      "logps/rejected": -0.994335949420929,
      "loss": 1.1505,
      "nll_loss": 1.0515625476837158,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.077392578125,
      "rewards/margins": 0.02199401892721653,
      "rewards/rejected": -0.09943847358226776,
      "step": 630
    },
    {
      "epoch": 0.02398306196248899,
      "grad_norm": 1.406642444805499,
      "learning_rate": 3.1622776601683788e-06,
      "log_odds_chosen": 0.38538819551467896,
      "log_odds_ratio": -0.633105456829071,
      "logits/chosen": -0.8365234136581421,
      "logits/rejected": -0.778124988079071,
      "logps/chosen": -0.767382800579071,
      "logps/rejected": -1.0369141101837158,
      "loss": 1.153,
      "nll_loss": 1.1222655773162842,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07679443061351776,
      "rewards/margins": 0.026889801025390625,
      "rewards/rejected": -0.10367431491613388,
      "step": 640
    },
    {
      "epoch": 0.024357797305652882,
      "grad_norm": 1.2813272431493172,
      "learning_rate": 3.1378581622109444e-06,
      "log_odds_chosen": 0.34343260526657104,
      "log_odds_ratio": -0.649121105670929,
      "logits/chosen": -0.8675781488418579,
      "logits/rejected": -0.804882824420929,
      "logps/chosen": -0.820117175579071,
      "logps/rejected": -1.0412108898162842,
      "loss": 1.1518,
      "nll_loss": 1.122460961341858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08197021484375,
      "rewards/margins": 0.02206115797162056,
      "rewards/rejected": -0.10402832180261612,
      "step": 650
    },
    {
      "epoch": 0.024732532648816773,
      "grad_norm": 1.236352341849085,
      "learning_rate": 3.113995776646092e-06,
      "log_odds_chosen": 0.19350585341453552,
      "log_odds_ratio": -0.689160168170929,
      "logits/chosen": -0.8949218988418579,
      "logits/rejected": -0.86328125,
      "logps/chosen": -0.7923828363418579,
      "logps/rejected": -0.9222656488418579,
      "loss": 1.135,
      "nll_loss": 1.0275390148162842,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07927246391773224,
      "rewards/margins": 0.01296234130859375,
      "rewards/rejected": -0.09218750149011612,
      "step": 660
    },
    {
      "epoch": 0.025107267991980663,
      "grad_norm": 1.295522133567198,
      "learning_rate": 3.090669637145023e-06,
      "log_odds_chosen": 0.42724609375,
      "log_odds_ratio": -0.6048828363418579,
      "logits/chosen": -0.8828125,
      "logits/rejected": -0.830859363079071,
      "logps/chosen": -0.767382800579071,
      "logps/rejected": -1.0568358898162842,
      "loss": 1.1281,
      "nll_loss": 1.060937523841858,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07666015625,
      "rewards/margins": 0.02902984619140625,
      "rewards/rejected": -0.10566405951976776,
      "step": 670
    },
    {
      "epoch": 0.025482003335144554,
      "grad_norm": 1.2607881606158629,
      "learning_rate": 3.0678599553894814e-06,
      "log_odds_chosen": 0.31297606229782104,
      "log_odds_ratio": -0.6592773199081421,
      "logits/chosen": -0.8824218511581421,
      "logits/rejected": -0.845996081829071,
      "logps/chosen": -0.7689453363418579,
      "logps/rejected": -0.9906250238418579,
      "loss": 1.129,
      "nll_loss": 1.0871093273162842,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07695312798023224,
      "rewards/margins": 0.022211456671357155,
      "rewards/rejected": -0.09903564304113388,
      "step": 680
    },
    {
      "epoch": 0.025856738678308445,
      "grad_norm": 1.3582317423984298,
      "learning_rate": 3.0455479505075235e-06,
      "log_odds_chosen": 0.03597412258386612,
      "log_odds_ratio": -0.7650390863418579,
      "logits/chosen": -0.833789050579071,
      "logits/rejected": -0.7984374761581421,
      "logps/chosen": -0.8070312738418579,
      "logps/rejected": -0.846484363079071,
      "loss": 1.1556,
      "nll_loss": 1.1376953125,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.08066406100988388,
      "rewards/margins": 0.0040153502486646175,
      "rewards/rejected": -0.08469238132238388,
      "step": 690
    },
    {
      "epoch": 0.026231474021472335,
      "grad_norm": 1.3808198017247686,
      "learning_rate": 3.0237157840738173e-06,
      "log_odds_chosen": 0.31572264432907104,
      "log_odds_ratio": -0.6480468511581421,
      "logits/chosen": -0.8822265863418579,
      "logits/rejected": -0.8285156488418579,
      "logps/chosen": -0.7613281011581421,
      "logps/rejected": -0.975390613079071,
      "loss": 1.1488,
      "nll_loss": 1.040429711341858,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.076171875,
      "rewards/margins": 0.0214385986328125,
      "rewards/rejected": -0.09748534858226776,
      "step": 700
    },
    {
      "epoch": 0.026606209364636226,
      "grad_norm": 1.3667438900929596,
      "learning_rate": 3.002346500163206e-06,
      "log_odds_chosen": 0.34514158964157104,
      "log_odds_ratio": -0.6708008050918579,
      "logits/chosen": -0.825390636920929,
      "logits/rejected": -0.7916015386581421,
      "logps/chosen": -0.823437511920929,
      "logps/rejected": -1.050195336341858,
      "loss": 1.1233,
      "nll_loss": 1.064062476158142,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08231201022863388,
      "rewards/margins": 0.022786712273955345,
      "rewards/rejected": -0.10500488430261612,
      "step": 710
    },
    {
      "epoch": 0.026980944707800117,
      "grad_norm": 1.3264331952839115,
      "learning_rate": 2.9814239699997195e-06,
      "log_odds_chosen": 0.35114747285842896,
      "log_odds_ratio": -0.62060546875,
      "logits/chosen": -0.846484363079071,
      "logits/rejected": -0.8017578125,
      "logps/chosen": -0.73828125,
      "logps/rejected": -0.9595702886581421,
      "loss": 1.1379,
      "nll_loss": 1.119531273841858,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07384033501148224,
      "rewards/margins": 0.022193145006895065,
      "rewards/rejected": -0.09600830078125,
      "step": 720
    },
    {
      "epoch": 0.027355680050964008,
      "grad_norm": 1.2513321056336504,
      "learning_rate": 2.9609328407904207e-06,
      "log_odds_chosen": 0.30076295137405396,
      "log_odds_ratio": -0.685742199420929,
      "logits/chosen": -0.875781238079071,
      "logits/rejected": -0.8277343511581421,
      "logps/chosen": -0.7789062261581421,
      "logps/rejected": -1.0070312023162842,
      "loss": 1.1461,
      "nll_loss": 1.117773413658142,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07792969048023224,
      "rewards/margins": 0.02278442308306694,
      "rewards/rejected": -0.10073242336511612,
      "step": 730
    },
    {
      "epoch": 0.0277304153941279,
      "grad_norm": 1.2601948679903272,
      "learning_rate": 2.940858488375231e-06,
      "log_odds_chosen": 0.3624023497104645,
      "log_odds_ratio": -0.619921863079071,
      "logits/chosen": -0.8623046875,
      "logits/rejected": -0.8203125,
      "logps/chosen": -0.7972656488418579,
      "logps/rejected": -1.042578101158142,
      "loss": 1.1541,
      "nll_loss": 1.076757788658142,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0797119140625,
      "rewards/margins": 0.024505615234375,
      "rewards/rejected": -0.10421142727136612,
      "step": 740
    },
    {
      "epoch": 0.02810515073729179,
      "grad_norm": 1.3524312217623626,
      "learning_rate": 2.9211869733608857e-06,
      "log_odds_chosen": 0.2868896424770355,
      "log_odds_ratio": -0.6625000238418579,
      "logits/chosen": -0.905468761920929,
      "logits/rejected": -0.8580077886581421,
      "logps/chosen": -0.739453136920929,
      "logps/rejected": -0.9419921636581421,
      "loss": 1.1056,
      "nll_loss": 1.0390625,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07391357421875,
      "rewards/margins": 0.02025756798684597,
      "rewards/rejected": -0.09416504204273224,
      "step": 750
    },
    {
      "epoch": 0.02847988608045568,
      "grad_norm": 1.2419588510615616,
      "learning_rate": 2.901905000440047e-06,
      "log_odds_chosen": 0.502062976360321,
      "log_odds_ratio": -0.580761730670929,
      "logits/chosen": -0.881640613079071,
      "logits/rejected": -0.7865234613418579,
      "logps/chosen": -0.778124988079071,
      "logps/rejected": -1.126562476158142,
      "loss": 1.1276,
      "nll_loss": 1.042382836341858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07785644382238388,
      "rewards/margins": 0.034809112548828125,
      "rewards/rejected": -0.11264648288488388,
      "step": 760
    },
    {
      "epoch": 0.02885462142361957,
      "grad_norm": 1.239077106602393,
      "learning_rate": 2.8829998806257885e-06,
      "log_odds_chosen": 0.307037353515625,
      "log_odds_ratio": -0.649121105670929,
      "logits/chosen": -0.857617199420929,
      "logits/rejected": -0.8304687738418579,
      "logps/chosen": -0.7406250238418579,
      "logps/rejected": -0.9380859136581421,
      "loss": 1.1087,
      "nll_loss": 1.1042969226837158,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.07412109524011612,
      "rewards/margins": 0.019678497686982155,
      "rewards/rejected": -0.09375,
      "step": 770
    },
    {
      "epoch": 0.029229356766783458,
      "grad_norm": 1.2506448096806428,
      "learning_rate": 2.8644594961577314e-06,
      "log_odds_chosen": 0.23677977919578552,
      "log_odds_ratio": -0.662109375,
      "logits/chosen": -0.875,
      "logits/rejected": -0.829296886920929,
      "logps/chosen": -0.777539074420929,
      "logps/rejected": -0.938671886920929,
      "loss": 1.1347,
      "nll_loss": 1.088281273841858,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07780762016773224,
      "rewards/margins": 0.0161590576171875,
      "rewards/rejected": -0.09389648586511612,
      "step": 780
    },
    {
      "epoch": 0.02960409210994735,
      "grad_norm": 1.246377648061077,
      "learning_rate": 2.84627226785928e-06,
      "log_odds_chosen": 0.32026368379592896,
      "log_odds_ratio": -0.6419922113418579,
      "logits/chosen": -0.860156238079071,
      "logits/rejected": -0.830859363079071,
      "logps/chosen": -0.797656238079071,
      "logps/rejected": -1.003515601158142,
      "loss": 1.1397,
      "nll_loss": 0.969921886920929,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07982178032398224,
      "rewards/margins": 0.02054748497903347,
      "rewards/rejected": -0.100341796875,
      "step": 790
    },
    {
      "epoch": 0.02997882745311124,
      "grad_norm": 1.3632197835770192,
      "learning_rate": 2.82842712474619e-06,
      "log_odds_chosen": 0.28773194551467896,
      "log_odds_ratio": -0.637890636920929,
      "logits/chosen": -0.8006836175918579,
      "logits/rejected": -0.741406261920929,
      "logps/chosen": -0.766406238079071,
      "logps/rejected": -0.9521484375,
      "loss": 1.1336,
      "nll_loss": 1.065039038658142,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07657470554113388,
      "rewards/margins": 0.018706511706113815,
      "rewards/rejected": -0.09528808295726776,
      "step": 800
    },
    {
      "epoch": 0.03035356279627513,
      "grad_norm": 1.4351214275675634,
      "learning_rate": 2.810913475705226e-06,
      "log_odds_chosen": 0.39958494901657104,
      "log_odds_ratio": -0.6317383050918579,
      "logits/chosen": -0.8583984375,
      "logits/rejected": -0.796679675579071,
      "logps/chosen": -0.765429675579071,
      "logps/rejected": -1.042578101158142,
      "loss": 1.1422,
      "nll_loss": 1.0398437976837158,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07655028998851776,
      "rewards/margins": 0.027701567858457565,
      "rewards/rejected": -0.10428466647863388,
      "step": 810
    },
    {
      "epoch": 0.03072829813943902,
      "grad_norm": 1.375500477532617,
      "learning_rate": 2.7937211830783128e-06,
      "log_odds_chosen": 0.29896849393844604,
      "log_odds_ratio": -0.6810547113418579,
      "logits/chosen": -0.854687511920929,
      "logits/rejected": -0.793164074420929,
      "logps/chosen": -0.8316406011581421,
      "logps/rejected": -1.0546875,
      "loss": 1.1092,
      "nll_loss": 1.044921875,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.08320312201976776,
      "rewards/margins": 0.02226104773581028,
      "rewards/rejected": -0.10550536960363388,
      "step": 820
    },
    {
      "epoch": 0.03110303348260291,
      "grad_norm": 1.3646110469789245,
      "learning_rate": 2.776840538002493e-06,
      "log_odds_chosen": 0.4053955078125,
      "log_odds_ratio": -0.62939453125,
      "logits/chosen": -0.845703125,
      "logits/rejected": -0.7880859375,
      "logps/chosen": -0.786328136920929,
      "logps/rejected": -1.083593726158142,
      "loss": 1.1151,
      "nll_loss": 1.1173827648162842,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07864990085363388,
      "rewards/margins": 0.02971801720559597,
      "rewards/rejected": -0.1083984375,
      "step": 830
    },
    {
      "epoch": 0.031477768825766805,
      "grad_norm": 1.3667022476239978,
      "learning_rate": 2.7602622373694163e-06,
      "log_odds_chosen": 0.25456541776657104,
      "log_odds_ratio": -0.673535168170929,
      "logits/chosen": -0.877734363079071,
      "logits/rejected": -0.8392578363418579,
      "logps/chosen": -0.767773449420929,
      "logps/rejected": -0.9371093511581421,
      "loss": 1.1613,
      "nll_loss": 1.066015601158142,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07675781100988388,
      "rewards/margins": 0.01689910888671875,
      "rewards/rejected": -0.09372558444738388,
      "step": 840
    },
    {
      "epoch": 0.03185250416893069,
      "grad_norm": 1.3232465882652,
      "learning_rate": 2.743977362280141e-06,
      "log_odds_chosen": 0.3177123963832855,
      "log_odds_ratio": -0.635937511920929,
      "logits/chosen": -0.862500011920929,
      "logits/rejected": -0.8365234136581421,
      "logps/chosen": -0.785937488079071,
      "logps/rejected": -0.9990234375,
      "loss": 1.1378,
      "nll_loss": 1.068359375,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07857666164636612,
      "rewards/margins": 0.02130126953125,
      "rewards/rejected": -0.09992675483226776,
      "step": 850
    },
    {
      "epoch": 0.03222723951209458,
      "grad_norm": 1.2919944854878742,
      "learning_rate": 2.7279773578818937e-06,
      "log_odds_chosen": 0.29710692167282104,
      "log_odds_ratio": -0.669238269329071,
      "logits/chosen": -0.873046875,
      "logits/rejected": -0.8287109136581421,
      "logps/chosen": -0.7919921875,
      "logps/rejected": -1.01171875,
      "loss": 1.1392,
      "nll_loss": 1.049414038658142,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.07919921725988388,
      "rewards/margins": 0.021979521960020065,
      "rewards/rejected": -0.1011962890625,
      "step": 860
    },
    {
      "epoch": 0.032601974855258474,
      "grad_norm": 1.2592198497496643,
      "learning_rate": 2.7122540144832417e-06,
      "log_odds_chosen": 0.29670411348342896,
      "log_odds_ratio": -0.6356445550918579,
      "logits/chosen": -0.853320300579071,
      "logits/rejected": -0.8228515386581421,
      "logps/chosen": -0.77392578125,
      "logps/rejected": -0.9566406011581421,
      "loss": 1.1036,
      "nll_loss": 1.081445336341858,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07741699367761612,
      "rewards/margins": 0.01820983923971653,
      "rewards/rejected": -0.09562988579273224,
      "step": 870
    },
    {
      "epoch": 0.03297671019842236,
      "grad_norm": 1.3648435225887698,
      "learning_rate": 2.696799449852968e-06,
      "log_odds_chosen": 0.2766967713832855,
      "log_odds_ratio": -0.6903320550918579,
      "logits/chosen": -0.805859386920929,
      "logits/rejected": -0.759570300579071,
      "logps/chosen": -0.748828113079071,
      "logps/rejected": -0.975390613079071,
      "loss": 1.149,
      "nll_loss": 1.120703101158142,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07487793266773224,
      "rewards/margins": 0.02262573316693306,
      "rewards/rejected": -0.0975341796875,
      "step": 880
    },
    {
      "epoch": 0.033351445541586255,
      "grad_norm": 1.3275845898014818,
      "learning_rate": 2.6816060926159636e-06,
      "log_odds_chosen": 0.3766845762729645,
      "log_odds_ratio": -0.6285156011581421,
      "logits/chosen": -0.8501952886581421,
      "logits/rejected": -0.7890625,
      "logps/chosen": -0.791210949420929,
      "logps/rejected": -1.0671875476837158,
      "loss": 1.1138,
      "nll_loss": 1.0498046875,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.0791015625,
      "rewards/margins": 0.02754669263958931,
      "rewards/rejected": -0.106689453125,
      "step": 890
    },
    {
      "epoch": 0.03372618088475014,
      "grad_norm": 1.3912112507033918,
      "learning_rate": 2.6666666666666664e-06,
      "log_odds_chosen": 0.2993530333042145,
      "log_odds_ratio": -0.681640625,
      "logits/chosen": -0.8363281488418579,
      "logits/rejected": -0.7568359375,
      "logps/chosen": -0.812695324420929,
      "logps/rejected": -1.0263671875,
      "loss": 1.1193,
      "nll_loss": 1.1056640148162842,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08126220852136612,
      "rewards/margins": 0.02137298509478569,
      "rewards/rejected": -0.1026611328125,
      "step": 900
    },
    {
      "epoch": 0.03410091622791404,
      "grad_norm": 1.3111239083675468,
      "learning_rate": 2.6519741765271837e-06,
      "log_odds_chosen": 0.18571777641773224,
      "log_odds_ratio": -0.6722656488418579,
      "logits/chosen": -0.8423827886581421,
      "logits/rejected": -0.791015625,
      "logps/chosen": -0.774609386920929,
      "logps/rejected": -0.894335925579071,
      "loss": 1.1062,
      "nll_loss": 1.0490233898162842,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07742919772863388,
      "rewards/margins": 0.01194915734231472,
      "rewards/rejected": -0.08939208835363388,
      "step": 910
    },
    {
      "epoch": 0.034475651571077924,
      "grad_norm": 1.298764434035606,
      "learning_rate": 2.637521893583148e-06,
      "log_odds_chosen": 0.3630737364292145,
      "log_odds_ratio": -0.6224609613418579,
      "logits/chosen": -0.875781238079071,
      "logits/rejected": -0.83984375,
      "logps/chosen": -0.771484375,
      "logps/rejected": -1.01953125,
      "loss": 1.1035,
      "nll_loss": 1.028710961341858,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07709960639476776,
      "rewards/margins": 0.02477569505572319,
      "rewards/rejected": -0.10187987983226776,
      "step": 920
    },
    {
      "epoch": 0.03485038691424182,
      "grad_norm": 1.355189178299939,
      "learning_rate": 2.6233033431358115e-06,
      "log_odds_chosen": 0.42341309785842896,
      "log_odds_ratio": -0.611035168170929,
      "logits/chosen": -0.893750011920929,
      "logits/rejected": -0.853515625,
      "logps/chosen": -0.724609375,
      "logps/rejected": -0.998242199420929,
      "loss": 1.1059,
      "nll_loss": 1.043554663658142,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07243652641773224,
      "rewards/margins": 0.027368927374482155,
      "rewards/rejected": -0.09990234673023224,
      "step": 930
    },
    {
      "epoch": 0.035225122257405705,
      "grad_norm": 1.3967202550865445,
      "learning_rate": 2.6093122922137685e-06,
      "log_odds_chosen": 0.22802734375,
      "log_odds_ratio": -0.67724609375,
      "logits/chosen": -0.9375,
      "logits/rejected": -0.8695312738418579,
      "logps/chosen": -0.8236328363418579,
      "logps/rejected": -0.9878906011581421,
      "loss": 1.1248,
      "nll_loss": 1.069921851158142,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.082275390625,
      "rewards/margins": 0.016541291028261185,
      "rewards/rejected": -0.09880371391773224,
      "step": 940
    },
    {
      "epoch": 0.0355998576005696,
      "grad_norm": 1.347735703074152,
      "learning_rate": 2.5955427380922006e-06,
      "log_odds_chosen": 0.2572082579135895,
      "log_odds_ratio": -0.6787109375,
      "logits/chosen": -0.868359386920929,
      "logits/rejected": -0.8246093988418579,
      "logps/chosen": -0.812304675579071,
      "logps/rejected": -0.970507800579071,
      "loss": 1.1299,
      "nll_loss": 1.0978515148162842,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.08137206733226776,
      "rewards/margins": 0.015731047838926315,
      "rewards/rejected": -0.09702148288488388,
      "step": 950
    },
    {
      "epoch": 0.03597459294373349,
      "grad_norm": 1.335134277792992,
      "learning_rate": 2.5819888974716113e-06,
      "log_odds_chosen": 0.4056762754917145,
      "log_odds_ratio": -0.615039050579071,
      "logits/chosen": -0.900390625,
      "logits/rejected": -0.851269543170929,
      "logps/chosen": -0.7734375,
      "logps/rejected": -1.026757836341858,
      "loss": 1.0958,
      "nll_loss": 1.0369141101837158,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.077392578125,
      "rewards/margins": 0.02533416822552681,
      "rewards/rejected": -0.1026611328125,
      "step": 960
    },
    {
      "epoch": 0.03634932828689738,
      "grad_norm": 1.4351792827421745,
      "learning_rate": 2.5686451962717425e-06,
      "log_odds_chosen": 0.30354005098342896,
      "log_odds_ratio": -0.6629883050918579,
      "logits/chosen": -0.8853515386581421,
      "logits/rejected": -0.847460925579071,
      "logps/chosen": -0.7509765625,
      "logps/rejected": -0.9535156488418579,
      "loss": 1.1172,
      "nll_loss": 1.0478515625,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07520751655101776,
      "rewards/margins": 0.02022399939596653,
      "rewards/rejected": -0.09528808295726776,
      "step": 970
    },
    {
      "epoch": 0.03672406363006127,
      "grad_norm": 1.2982482182565214,
      "learning_rate": 2.5555062599997596e-06,
      "log_odds_chosen": 0.22535400092601776,
      "log_odds_ratio": -0.686328113079071,
      "logits/chosen": -0.931640625,
      "logits/rejected": -0.9048827886581421,
      "logps/chosen": -0.7925781011581421,
      "logps/rejected": -0.9609375,
      "loss": 1.1396,
      "nll_loss": 1.073632836341858,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07923583686351776,
      "rewards/margins": 0.016807174310088158,
      "rewards/rejected": -0.09609375149011612,
      "step": 980
    },
    {
      "epoch": 0.03709879897322516,
      "grad_norm": 1.3294098327512638,
      "learning_rate": 2.5425669046549126e-06,
      "log_odds_chosen": 0.38341063261032104,
      "log_odds_ratio": -0.647656261920929,
      "logits/chosen": -0.9224609136581421,
      "logits/rejected": -0.873242199420929,
      "logps/chosen": -0.7601562738418579,
      "logps/rejected": -1.0148437023162842,
      "loss": 1.1177,
      "nll_loss": 1.0537109375,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07603760063648224,
      "rewards/margins": 0.02543182298541069,
      "rewards/rejected": -0.10151366889476776,
      "step": 990
    },
    {
      "epoch": 0.03747353431638905,
      "grad_norm": 1.3958078859580991,
      "learning_rate": 2.5298221281347034e-06,
      "log_odds_chosen": 0.41058349609375,
      "log_odds_ratio": -0.6241210699081421,
      "logits/chosen": -0.8482421636581421,
      "logits/rejected": -0.8046875,
      "logps/chosen": -0.7728515863418579,
      "logps/rejected": -1.065039038658142,
      "loss": 1.123,
      "nll_loss": 1.0615234375,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07729492336511612,
      "rewards/margins": 0.029154205694794655,
      "rewards/rejected": -0.10650634765625,
      "step": 1000
    },
    {
      "epoch": 0.037848269659552944,
      "grad_norm": 1.3134952818793206,
      "learning_rate": 2.5172671021102103e-06,
      "log_odds_chosen": 0.3288330137729645,
      "log_odds_ratio": -0.6377929449081421,
      "logits/chosen": -0.899218738079071,
      "logits/rejected": -0.8392578363418579,
      "logps/chosen": -0.7544921636581421,
      "logps/rejected": -0.96484375,
      "loss": 1.0865,
      "nll_loss": 1.0291016101837158,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07545165717601776,
      "rewards/margins": 0.02109069749712944,
      "rewards/rejected": -0.09648437798023224,
      "step": 1010
    },
    {
      "epoch": 0.03822300500271683,
      "grad_norm": 1.3711094842571574,
      "learning_rate": 2.504897164340598e-06,
      "log_odds_chosen": 0.2563842833042145,
      "log_odds_ratio": -0.644824206829071,
      "logits/chosen": -0.82421875,
      "logits/rejected": -0.8119140863418579,
      "logps/chosen": -0.802539050579071,
      "logps/rejected": -0.977343738079071,
      "loss": 1.1057,
      "nll_loss": 1.080664038658142,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.08017577975988388,
      "rewards/margins": 0.017594147473573685,
      "rewards/rejected": -0.09770508110523224,
      "step": 1020
    },
    {
      "epoch": 0.03859774034588072,
      "grad_norm": 1.3065961172046305,
      "learning_rate": 2.492707811399023e-06,
      "log_odds_chosen": 0.26336669921875,
      "log_odds_ratio": -0.6683593988418579,
      "logits/chosen": -0.8392578363418579,
      "logits/rejected": -0.798632800579071,
      "logps/chosen": -0.8154296875,
      "logps/rejected": -1.012109398841858,
      "loss": 1.0977,
      "nll_loss": 1.101953148841858,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08146972954273224,
      "rewards/margins": 0.019649505615234375,
      "rewards/rejected": -0.10117187350988388,
      "step": 1030
    },
    {
      "epoch": 0.03897247568904461,
      "grad_norm": 1.2521923527884105,
      "learning_rate": 2.480694691784169e-06,
      "log_odds_chosen": 0.16287842392921448,
      "log_odds_ratio": -0.713085949420929,
      "logits/chosen": -0.891796886920929,
      "logits/rejected": -0.8626953363418579,
      "logps/chosen": -0.7738281488418579,
      "logps/rejected": -0.912890613079071,
      "loss": 1.1233,
      "nll_loss": 1.0509765148162842,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07740478217601776,
      "rewards/margins": 0.01389465294778347,
      "rewards/rejected": -0.09130859375,
      "step": 1040
    },
    {
      "epoch": 0.0393472110322085,
      "grad_norm": 1.3620328571772595,
      "learning_rate": 2.4688535993934706e-06,
      "log_odds_chosen": 0.3627563416957855,
      "log_odds_ratio": -0.6175781488418579,
      "logits/chosen": -0.8734375238418579,
      "logits/rejected": -0.806835949420929,
      "logps/chosen": -0.794726550579071,
      "logps/rejected": -1.0300781726837158,
      "loss": 1.1362,
      "nll_loss": 1.0205078125,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07949218899011612,
      "rewards/margins": 0.02347412146627903,
      "rewards/rejected": -0.10300292819738388,
      "step": 1050
    },
    {
      "epoch": 0.039721946375372394,
      "grad_norm": 1.231823676667514,
      "learning_rate": 2.457180467335805e-06,
      "log_odds_chosen": 0.2508911192417145,
      "log_odds_ratio": -0.6517578363418579,
      "logits/chosen": -0.898242175579071,
      "logits/rejected": -0.845703125,
      "logps/chosen": -0.767382800579071,
      "logps/rejected": -0.9287109375,
      "loss": 1.0905,
      "nll_loss": 1.0068359375,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0767822265625,
      "rewards/margins": 0.01607818529009819,
      "rewards/rejected": -0.09284667670726776,
      "step": 1060
    },
    {
      "epoch": 0.04009668171853628,
      "grad_norm": 1.3100730899024347,
      "learning_rate": 2.4456713620629725e-06,
      "log_odds_chosen": 0.23249511420726776,
      "log_odds_ratio": -0.68017578125,
      "logits/chosen": -0.856640636920929,
      "logits/rejected": -0.8218749761581421,
      "logps/chosen": -0.740429699420929,
      "logps/rejected": -0.9185546636581421,
      "loss": 1.1121,
      "nll_loss": 1.0939452648162842,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.07396240532398224,
      "rewards/margins": 0.01784973219037056,
      "rewards/rejected": -0.09187011420726776,
      "step": 1070
    },
    {
      "epoch": 0.040471417061700175,
      "grad_norm": 1.3116898311958183,
      "learning_rate": 2.4343224778007378e-06,
      "log_odds_chosen": 0.39564210176467896,
      "log_odds_ratio": -0.6200195550918579,
      "logits/chosen": -0.856640636920929,
      "logits/rejected": -0.8060547113418579,
      "logps/chosen": -0.796875,
      "logps/rejected": -1.0617187023162842,
      "loss": 1.1081,
      "nll_loss": 1.067773461341858,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07969971001148224,
      "rewards/margins": 0.026584625244140625,
      "rewards/rejected": -0.106201171875,
      "step": 1080
    },
    {
      "epoch": 0.04084615240486406,
      "grad_norm": 1.5636298747045254,
      "learning_rate": 2.4231301312615306e-06,
      "log_odds_chosen": 0.14897461235523224,
      "log_odds_ratio": -0.708789050579071,
      "logits/chosen": -0.923828125,
      "logits/rejected": -0.8910156488418579,
      "logps/chosen": -0.8150390386581421,
      "logps/rejected": -0.9423828125,
      "loss": 1.1302,
      "nll_loss": 1.071679711341858,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08151855319738388,
      "rewards/margins": 0.012713623233139515,
      "rewards/rejected": -0.09417724609375,
      "step": 1090
    },
    {
      "epoch": 0.04122088774802796,
      "grad_norm": 1.3479360797372282,
      "learning_rate": 2.412090756622109e-06,
      "log_odds_chosen": 0.2756713926792145,
      "log_odds_ratio": -0.673535168170929,
      "logits/chosen": -0.91015625,
      "logits/rejected": -0.87109375,
      "logps/chosen": -0.786914050579071,
      "logps/rejected": -0.9818359613418579,
      "loss": 1.0996,
      "nll_loss": 1.0390625,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07869873195886612,
      "rewards/margins": 0.019378662109375,
      "rewards/rejected": -0.09809570014476776,
      "step": 1100
    },
    {
      "epoch": 0.041595623091191844,
      "grad_norm": 1.379993828249997,
      "learning_rate": 2.401200900750657e-06,
      "log_odds_chosen": 0.3448242247104645,
      "log_odds_ratio": -0.639453113079071,
      "logits/chosen": -0.842578113079071,
      "logits/rejected": -0.7828124761581421,
      "logps/chosen": -0.683398425579071,
      "logps/rejected": -0.9046875238418579,
      "loss": 1.0999,
      "nll_loss": 0.993945300579071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06826172024011612,
      "rewards/margins": 0.022138213738799095,
      "rewards/rejected": -0.09039306640625,
      "step": 1110
    },
    {
      "epoch": 0.04197035843435574,
      "grad_norm": 2.1457745021796826,
      "learning_rate": 2.390457218668787e-06,
      "log_odds_chosen": 0.20140381157398224,
      "log_odds_ratio": -0.7017577886581421,
      "logits/chosen": -0.8238281011581421,
      "logits/rejected": -0.7939453125,
      "logps/chosen": -0.786328136920929,
      "logps/rejected": -0.9253906011581421,
      "loss": 1.0793,
      "nll_loss": 1.090234398841858,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07868652045726776,
      "rewards/margins": 0.01392440777271986,
      "rewards/rejected": -0.09260253608226776,
      "step": 1120
    },
    {
      "epoch": 0.042345093777519625,
      "grad_norm": 1.3833716316228606,
      "learning_rate": 2.379856469234918e-06,
      "log_odds_chosen": 0.2701416015625,
      "log_odds_ratio": -0.661425769329071,
      "logits/chosen": -0.8746093511581421,
      "logits/rejected": -0.807812511920929,
      "logps/chosen": -0.800000011920929,
      "logps/rejected": -0.9794921875,
      "loss": 1.1137,
      "nll_loss": 0.9820312261581421,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08005370944738388,
      "rewards/margins": 0.017926787957549095,
      "rewards/rejected": -0.097900390625,
      "step": 1130
    },
    {
      "epoch": 0.04271982912068352,
      "grad_norm": 1.2924557078188472,
      "learning_rate": 2.369395511036369e-06,
      "log_odds_chosen": 0.4886718690395355,
      "log_odds_ratio": -0.587695300579071,
      "logits/chosen": -0.869335949420929,
      "logits/rejected": -0.8134765625,
      "logps/chosen": -0.7197265625,
      "logps/rejected": -1.045507788658142,
      "loss": 1.1117,
      "nll_loss": 1.0294921398162842,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07197265326976776,
      "rewards/margins": 0.032623291015625,
      "rewards/rejected": -0.10456542670726776,
      "step": 1140
    },
    {
      "epoch": 0.04309456446384741,
      "grad_norm": 1.3358032665774562,
      "learning_rate": 2.359071298478354e-06,
      "log_odds_chosen": 0.32496339082717896,
      "log_odds_ratio": -0.6490234136581421,
      "logits/chosen": -0.8753906488418579,
      "logits/rejected": -0.8228515386581421,
      "logps/chosen": -0.7886718511581421,
      "logps/rejected": -1.0119140148162842,
      "loss": 1.0897,
      "nll_loss": 1.0751953125,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07888183742761612,
      "rewards/margins": 0.022319793701171875,
      "rewards/rejected": -0.10122070461511612,
      "step": 1150
    },
    {
      "epoch": 0.0434692998070113,
      "grad_norm": 1.4112213065103907,
      "learning_rate": 2.3488808780588137e-06,
      "log_odds_chosen": 0.45985108613967896,
      "log_odds_ratio": -0.603515625,
      "logits/chosen": -0.889843761920929,
      "logits/rejected": -0.831250011920929,
      "logps/chosen": -0.754687488079071,
      "logps/rejected": -1.05078125,
      "loss": 1.1396,
      "nll_loss": 1.0710937976837158,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07552490383386612,
      "rewards/margins": 0.02954559400677681,
      "rewards/rejected": -0.10500488430261612,
      "step": 1160
    },
    {
      "epoch": 0.04384403515017519,
      "grad_norm": 1.3299591887776359,
      "learning_rate": 2.3388213848187446e-06,
      "log_odds_chosen": 0.2849365174770355,
      "log_odds_ratio": -0.64794921875,
      "logits/chosen": -0.8208984136581421,
      "logits/rejected": -0.77734375,
      "logps/chosen": -0.7503906488418579,
      "logps/rejected": -0.923632800579071,
      "loss": 1.1113,
      "nll_loss": 1.0654296875,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07503662258386612,
      "rewards/margins": 0.01729125902056694,
      "rewards/rejected": -0.09239502251148224,
      "step": 1170
    },
    {
      "epoch": 0.04421877049333908,
      "grad_norm": 1.3457626366288316,
      "learning_rate": 2.328890038958328e-06,
      "log_odds_chosen": 0.24016113579273224,
      "log_odds_ratio": -0.6817382574081421,
      "logits/chosen": -0.9052734375,
      "logits/rejected": -0.8628906011581421,
      "logps/chosen": -0.779296875,
      "logps/rejected": -0.9580078125,
      "loss": 1.0857,
      "nll_loss": 1.0037109851837158,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07791747897863388,
      "rewards/margins": 0.01789550855755806,
      "rewards/rejected": -0.09577636420726776,
      "step": 1180
    },
    {
      "epoch": 0.04459350583650297,
      "grad_norm": 1.3268704796191422,
      "learning_rate": 2.3190841426097937e-06,
      "log_odds_chosen": 0.20723876357078552,
      "log_odds_ratio": -0.69921875,
      "logits/chosen": -0.8822265863418579,
      "logits/rejected": -0.8294922113418579,
      "logps/chosen": -0.771289050579071,
      "logps/rejected": -0.9292968511581421,
      "loss": 1.1067,
      "nll_loss": 1.0324218273162842,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.07711181789636612,
      "rewards/margins": 0.015781402587890625,
      "rewards/rejected": -0.0928955078125,
      "step": 1190
    },
    {
      "epoch": 0.04496824117966686,
      "grad_norm": 1.3239193994169152,
      "learning_rate": 2.309401076758503e-06,
      "log_odds_chosen": 0.32415771484375,
      "log_odds_ratio": -0.66455078125,
      "logits/chosen": -0.862109363079071,
      "logits/rejected": -0.8285156488418579,
      "logps/chosen": -0.759960949420929,
      "logps/rejected": -0.9916015863418579,
      "loss": 1.0891,
      "nll_loss": 0.9527343511581421,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07600097358226776,
      "rewards/margins": 0.02315216138958931,
      "rewards/rejected": -0.09907226264476776,
      "step": 1200
    },
    {
      "epoch": 0.04534297652283075,
      "grad_norm": 1.3791611039458274,
      "learning_rate": 2.299838298304276e-06,
      "log_odds_chosen": 0.4273925721645355,
      "log_odds_ratio": -0.579394519329071,
      "logits/chosen": -0.9115234613418579,
      "logits/rejected": -0.827343761920929,
      "logps/chosen": -0.730664074420929,
      "logps/rejected": -1.0107421875,
      "loss": 1.0802,
      "nll_loss": 0.9808593988418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07305908203125,
      "rewards/margins": 0.02798919752240181,
      "rewards/rejected": -0.10112304985523224,
      "step": 1210
    },
    {
      "epoch": 0.04571771186599464,
      "grad_norm": 1.2605751004698105,
      "learning_rate": 2.2903933372554728e-06,
      "log_odds_chosen": 0.44050294160842896,
      "log_odds_ratio": -0.594921886920929,
      "logits/chosen": -0.8330078125,
      "logits/rejected": -0.783984363079071,
      "logps/chosen": -0.76953125,
      "logps/rejected": -1.0615234375,
      "loss": 1.0958,
      "nll_loss": 1.0187499523162842,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07694091647863388,
      "rewards/margins": 0.02923278883099556,
      "rewards/rejected": -0.10612793266773224,
      "step": 1220
    },
    {
      "epoch": 0.04609244720915853,
      "grad_norm": 1.3096708642541972,
      "learning_rate": 2.281063794048804e-06,
      "log_odds_chosen": 0.512524425983429,
      "log_odds_ratio": -0.5888671875,
      "logits/chosen": -0.881054699420929,
      "logits/rejected": -0.8472656011581421,
      "logps/chosen": -0.802734375,
      "logps/rejected": -1.1474609375,
      "loss": 1.0981,
      "nll_loss": 1.058203101158142,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.08031006157398224,
      "rewards/margins": 0.03443298488855362,
      "rewards/rejected": -0.11464843899011612,
      "step": 1230
    },
    {
      "epoch": 0.04646718255232242,
      "grad_norm": 1.2867896326601196,
      "learning_rate": 2.271847336988259e-06,
      "log_odds_chosen": 0.2968994081020355,
      "log_odds_ratio": -0.6634765863418579,
      "logits/chosen": -0.8248046636581421,
      "logits/rejected": -0.7822265625,
      "logps/chosen": -0.802929699420929,
      "logps/rejected": -1.0144531726837158,
      "loss": 1.094,
      "nll_loss": 1.133398413658142,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.0802001953125,
      "rewards/margins": 0.02114105224609375,
      "rewards/rejected": -0.10141601413488388,
      "step": 1240
    },
    {
      "epoch": 0.046841917895486314,
      "grad_norm": 1.245995535924117,
      "learning_rate": 2.262741699796952e-06,
      "log_odds_chosen": 0.33122557401657104,
      "log_odds_ratio": -0.63525390625,
      "logits/chosen": -0.8482421636581421,
      "logits/rejected": -0.8082031011581421,
      "logps/chosen": -0.7621093988418579,
      "logps/rejected": -0.972851574420929,
      "loss": 1.1183,
      "nll_loss": 1.029882788658142,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07622070610523224,
      "rewards/margins": 0.02106933668255806,
      "rewards/rejected": -0.09731445461511612,
      "step": 1250
    },
    {
      "epoch": 0.0472166532386502,
      "grad_norm": 1.3343881650180343,
      "learning_rate": 2.253744679276044e-06,
      "log_odds_chosen": 0.20167236030101776,
      "log_odds_ratio": -0.6932617425918579,
      "logits/chosen": -0.8333984613418579,
      "logits/rejected": -0.7974609136581421,
      "logps/chosen": -0.792773425579071,
      "logps/rejected": -0.918749988079071,
      "loss": 1.0905,
      "nll_loss": 1.119140625,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07930908352136612,
      "rewards/margins": 0.01254425011575222,
      "rewards/rejected": -0.09185791015625,
      "step": 1260
    },
    {
      "epoch": 0.047591388581814095,
      "grad_norm": 1.3664868004506154,
      "learning_rate": 2.244854133065255e-06,
      "log_odds_chosen": 0.13221435248851776,
      "log_odds_ratio": -0.739062488079071,
      "logits/chosen": -0.8369140625,
      "logits/rejected": -0.821093738079071,
      "logps/chosen": -0.8187500238418579,
      "logps/rejected": -0.9130859375,
      "loss": 1.1007,
      "nll_loss": 1.1085937023162842,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.08187256008386612,
      "rewards/margins": 0.00943145714700222,
      "rewards/rejected": -0.09138183295726776,
      "step": 1270
    },
    {
      "epoch": 0.04796612392497798,
      "grad_norm": 1.3442799390899114,
      "learning_rate": 2.2360679774997895e-06,
      "log_odds_chosen": 0.26896971464157104,
      "log_odds_ratio": -0.6666015386581421,
      "logits/chosen": -0.8876953125,
      "logits/rejected": -0.835742175579071,
      "logps/chosen": -0.795117199420929,
      "logps/rejected": -0.9814453125,
      "loss": 1.0994,
      "nll_loss": 1.0791015625,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.0794677734375,
      "rewards/margins": 0.018688201904296875,
      "rewards/rejected": -0.09814453125,
      "step": 1280
    },
    {
      "epoch": 0.04834085926814188,
      "grad_norm": 1.2812291722322078,
      "learning_rate": 2.2273841855588183e-06,
      "log_odds_chosen": 0.3714843690395355,
      "log_odds_ratio": -0.619140625,
      "logits/chosen": -0.8343750238418579,
      "logits/rejected": -0.779492199420929,
      "logps/chosen": -0.736523449420929,
      "logps/rejected": -0.973828136920929,
      "loss": 1.0831,
      "nll_loss": 1.0021483898162842,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07366943359375,
      "rewards/margins": 0.02374267578125,
      "rewards/rejected": -0.09733887016773224,
      "step": 1290
    },
    {
      "epoch": 0.048715594611305764,
      "grad_norm": 1.3341767785381993,
      "learning_rate": 2.2188007849009167e-06,
      "log_odds_chosen": 0.3177734315395355,
      "log_odds_ratio": -0.6524413824081421,
      "logits/chosen": -0.8052734136581421,
      "logits/rejected": -0.783984363079071,
      "logps/chosen": -0.8017578125,
      "logps/rejected": -1.0222656726837158,
      "loss": 1.0925,
      "nll_loss": 1.087304711341858,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.08015136420726776,
      "rewards/margins": 0.022057723253965378,
      "rewards/rejected": -0.10214843600988388,
      "step": 1300
    },
    {
      "epoch": 0.04909032995446966,
      "grad_norm": 1.2915427681771623,
      "learning_rate": 2.2103158559821502e-06,
      "log_odds_chosen": 0.33477783203125,
      "log_odds_ratio": -0.675976574420929,
      "logits/chosen": -0.8394531011581421,
      "logits/rejected": -0.7728515863418579,
      "logps/chosen": -0.7339843511581421,
      "logps/rejected": -0.994140625,
      "loss": 1.0906,
      "nll_loss": 0.9765625,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07344970852136612,
      "rewards/margins": 0.02588653564453125,
      "rewards/rejected": -0.09937743842601776,
      "step": 1310
    },
    {
      "epoch": 0.049465065297633545,
      "grad_norm": 1.2514217398072038,
      "learning_rate": 2.2019275302527213e-06,
      "log_odds_chosen": 0.3370361328125,
      "log_odds_ratio": -0.6519531011581421,
      "logits/chosen": -0.8177734613418579,
      "logits/rejected": -0.748828113079071,
      "logps/chosen": -0.775195300579071,
      "logps/rejected": -0.9853515625,
      "loss": 1.1194,
      "nll_loss": 1.0378906726837158,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07756347954273224,
      "rewards/margins": 0.021070098504424095,
      "rewards/rejected": -0.09853515774011612,
      "step": 1320
    },
    {
      "epoch": 0.04983980064079744,
      "grad_norm": 1.4460927533505104,
      "learning_rate": 2.193633988428327e-06,
      "log_odds_chosen": 0.287841796875,
      "log_odds_ratio": -0.68603515625,
      "logits/chosen": -0.8291015625,
      "logits/rejected": -0.7811523675918579,
      "logps/chosen": -0.8033202886581421,
      "logps/rejected": -1.000585913658142,
      "loss": 1.0847,
      "nll_loss": 1.046875,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.08027343451976776,
      "rewards/margins": 0.019751738756895065,
      "rewards/rejected": -0.10003662109375,
      "step": 1330
    },
    {
      "epoch": 0.05021453598396133,
      "grad_norm": 1.307717934921501,
      "learning_rate": 2.185433458832612e-06,
      "log_odds_chosen": 0.31916505098342896,
      "log_odds_ratio": -0.63720703125,
      "logits/chosen": -0.861523449420929,
      "logits/rejected": -0.8023437261581421,
      "logps/chosen": -0.7173827886581421,
      "logps/rejected": -0.925585925579071,
      "loss": 1.0917,
      "nll_loss": 1.056640625,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07172851264476776,
      "rewards/margins": 0.02083282545208931,
      "rewards/rejected": -0.09248046576976776,
      "step": 1340
    },
    {
      "epoch": 0.05058927132712522,
      "grad_norm": 1.3048430559896342,
      "learning_rate": 2.177324215807269e-06,
      "log_odds_chosen": 0.41595458984375,
      "log_odds_ratio": -0.614550769329071,
      "logits/chosen": -0.86328125,
      "logits/rejected": -0.809374988079071,
      "logps/chosen": -0.741992175579071,
      "logps/rejected": -1.0046875476837158,
      "loss": 1.0876,
      "nll_loss": 1.013671875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07423095405101776,
      "rewards/margins": 0.02622375451028347,
      "rewards/rejected": -0.10043945163488388,
      "step": 1350
    },
    {
      "epoch": 0.05096400667028911,
      "grad_norm": 1.3020885426551922,
      "learning_rate": 2.1693045781865616e-06,
      "log_odds_chosen": 0.3321289122104645,
      "log_odds_ratio": -0.660839855670929,
      "logits/chosen": -0.86767578125,
      "logits/rejected": -0.838574230670929,
      "logps/chosen": -0.7486327886581421,
      "logps/rejected": -0.954296886920929,
      "loss": 1.0886,
      "nll_loss": 1.066796898841858,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07480468600988388,
      "rewards/margins": 0.02055206336081028,
      "rewards/rejected": -0.09548339992761612,
      "step": 1360
    },
    {
      "epoch": 0.051338742013452995,
      "grad_norm": 1.2263142332758468,
      "learning_rate": 2.1613729078331965e-06,
      "log_odds_chosen": 0.36424559354782104,
      "log_odds_ratio": -0.617480456829071,
      "logits/chosen": -0.875,
      "logits/rejected": -0.80859375,
      "logps/chosen": -0.734570324420929,
      "logps/rejected": -0.9789062738418579,
      "loss": 1.1063,
      "nll_loss": 1.0476562976837158,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07349853217601776,
      "rewards/margins": 0.024383544921875,
      "rewards/rejected": -0.09792480617761612,
      "step": 1370
    },
    {
      "epoch": 0.05171347735661689,
      "grad_norm": 1.3410266509888245,
      "learning_rate": 2.1535276082326617e-06,
      "log_odds_chosen": 0.2757934629917145,
      "log_odds_ratio": -0.679882824420929,
      "logits/chosen": -0.8570312261581421,
      "logits/rejected": -0.8199218511581421,
      "logps/chosen": -0.7796875238418579,
      "logps/rejected": -0.9501953125,
      "loss": 1.0891,
      "nll_loss": 1.024804711341858,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07797851413488388,
      "rewards/margins": 0.017049407586455345,
      "rewards/rejected": -0.09514160454273224,
      "step": 1380
    },
    {
      "epoch": 0.05208821269978078,
      "grad_norm": 1.2397623964347277,
      "learning_rate": 2.14576712314328e-06,
      "log_odds_chosen": 0.2834716737270355,
      "log_odds_ratio": -0.6488281488418579,
      "logits/chosen": -0.8837890625,
      "logits/rejected": -0.838671863079071,
      "logps/chosen": -0.7660156488418579,
      "logps/rejected": -0.956835925579071,
      "loss": 1.0764,
      "nll_loss": 1.0076172351837158,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.0765380859375,
      "rewards/margins": 0.01903533935546875,
      "rewards/rejected": -0.09561767429113388,
      "step": 1390
    },
    {
      "epoch": 0.05246294804294467,
      "grad_norm": 1.3569429701538913,
      "learning_rate": 2.138089935299395e-06,
      "log_odds_chosen": 0.35850828886032104,
      "log_odds_ratio": -0.628222644329071,
      "logits/chosen": -0.859570324420929,
      "logits/rejected": -0.7984374761581421,
      "logps/chosen": -0.76123046875,
      "logps/rejected": -1.0007812976837158,
      "loss": 1.0954,
      "nll_loss": 1.0615234375,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07623291015625,
      "rewards/margins": 0.02391510084271431,
      "rewards/rejected": -0.10008545219898224,
      "step": 1400
    },
    {
      "epoch": 0.05283768338610856,
      "grad_norm": 1.440399442061999,
      "learning_rate": 2.1304945651652297e-06,
      "log_odds_chosen": 0.296875,
      "log_odds_ratio": -0.6578124761581421,
      "logits/chosen": -0.8658202886581421,
      "logits/rejected": -0.816210925579071,
      "logps/chosen": -0.7705078125,
      "logps/rejected": -0.9652343988418579,
      "loss": 1.0654,
      "nll_loss": 1.0349609851837158,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07709960639476776,
      "rewards/margins": 0.01941986009478569,
      "rewards/rejected": -0.0965576171875,
      "step": 1410
    },
    {
      "epoch": 0.05321241872927245,
      "grad_norm": 1.30856545323612,
      "learning_rate": 2.122979569737101e-06,
      "log_odds_chosen": 0.46595460176467896,
      "log_odds_ratio": -0.6197265386581421,
      "logits/chosen": -0.919726550579071,
      "logits/rejected": -0.803515613079071,
      "logps/chosen": -0.7867187261581421,
      "logps/rejected": -1.1062500476837158,
      "loss": 1.1016,
      "nll_loss": 1.0859375,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07867431640625,
      "rewards/margins": 0.03198089450597763,
      "rewards/rejected": -0.11066894233226776,
      "step": 1420
    },
    {
      "epoch": 0.05358715407243634,
      "grad_norm": 1.3399557504248636,
      "learning_rate": 2.11554354139178e-06,
      "log_odds_chosen": 0.36802977323532104,
      "log_odds_ratio": -0.623046875,
      "logits/chosen": -0.8568359613418579,
      "logits/rejected": -0.805468738079071,
      "logps/chosen": -0.7359374761581421,
      "logps/rejected": -0.963085949420929,
      "loss": 1.1012,
      "nll_loss": 0.9740234613418579,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07358398288488388,
      "rewards/margins": 0.02268829382956028,
      "rewards/rejected": -0.0963134765625,
      "step": 1430
    },
    {
      "epoch": 0.053961889415600234,
      "grad_norm": 1.3325049959151167,
      "learning_rate": 2.1081851067789196e-06,
      "log_odds_chosen": 0.21673583984375,
      "log_odds_ratio": -0.69970703125,
      "logits/chosen": -0.8822265863418579,
      "logits/rejected": -0.837109386920929,
      "logps/chosen": -0.788867175579071,
      "logps/rejected": -0.956250011920929,
      "loss": 1.0888,
      "nll_loss": 1.0625,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.078857421875,
      "rewards/margins": 0.01674957200884819,
      "rewards/rejected": -0.09564208984375,
      "step": 1440
    },
    {
      "epoch": 0.05433662475876412,
      "grad_norm": 1.3562862852242885,
      "learning_rate": 2.1009029257555606e-06,
      "log_odds_chosen": 0.23139648139476776,
      "log_odds_ratio": -0.6792968511581421,
      "logits/chosen": -0.8990234136581421,
      "logits/rejected": -0.8427734375,
      "logps/chosen": -0.797070324420929,
      "logps/rejected": -0.9595702886581421,
      "loss": 1.0802,
      "nll_loss": 0.976367175579071,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.07972411811351776,
      "rewards/margins": 0.01623687706887722,
      "rewards/rejected": -0.09598388522863388,
      "step": 1450
    },
    {
      "epoch": 0.054711360101928015,
      "grad_norm": 1.3472423690766582,
      "learning_rate": 2.0936956903608545e-06,
      "log_odds_chosen": 0.33140867948532104,
      "log_odds_ratio": -0.6519531011581421,
      "logits/chosen": -0.888476550579071,
      "logits/rejected": -0.83984375,
      "logps/chosen": -0.7201172113418579,
      "logps/rejected": -0.938671886920929,
      "loss": 1.0794,
      "nll_loss": 1.002343773841858,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07210693508386612,
      "rewards/margins": 0.021842192858457565,
      "rewards/rejected": -0.09384765475988388,
      "step": 1460
    },
    {
      "epoch": 0.0550860954450919,
      "grad_norm": 1.3122990258485507,
      "learning_rate": 2.0865621238292046e-06,
      "log_odds_chosen": 0.46293944120407104,
      "log_odds_ratio": -0.575488269329071,
      "logits/chosen": -0.853515625,
      "logits/rejected": -0.7909179925918579,
      "logps/chosen": -0.7318359613418579,
      "logps/rejected": -1.0373046398162842,
      "loss": 1.1018,
      "nll_loss": 1.020898461341858,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07318115234375,
      "rewards/margins": 0.03056030347943306,
      "rewards/rejected": -0.103759765625,
      "step": 1470
    },
    {
      "epoch": 0.0554608307882558,
      "grad_norm": 1.3575800223621242,
      "learning_rate": 2.079500979640145e-06,
      "log_odds_chosen": 0.35725098848342896,
      "log_odds_ratio": -0.646289050579071,
      "logits/chosen": -0.8291015625,
      "logits/rejected": -0.794726550579071,
      "logps/chosen": -0.7347656488418579,
      "logps/rejected": -0.989062488079071,
      "loss": 1.068,
      "nll_loss": 0.991406261920929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07344970852136612,
      "rewards/margins": 0.02544097974896431,
      "rewards/rejected": -0.09896240383386612,
      "step": 1480
    },
    {
      "epoch": 0.055835566131419684,
      "grad_norm": 1.2964626588108923,
      "learning_rate": 2.072511040603359e-06,
      "log_odds_chosen": 0.23166504502296448,
      "log_odds_ratio": -0.673144519329071,
      "logits/chosen": -0.8915039300918579,
      "logits/rejected": -0.819531261920929,
      "logps/chosen": -0.7369140386581421,
      "logps/rejected": -0.9013671875,
      "loss": 1.0876,
      "nll_loss": 1.0263671875,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07375488430261612,
      "rewards/margins": 0.016468048095703125,
      "rewards/rejected": -0.09018554538488388,
      "step": 1490
    },
    {
      "epoch": 0.05621030147458358,
      "grad_norm": 1.3288281584887136,
      "learning_rate": 2.065591117977289e-06,
      "log_odds_chosen": 0.35069578886032104,
      "log_odds_ratio": -0.643750011920929,
      "logits/chosen": -0.9263671636581421,
      "logits/rejected": -0.862109363079071,
      "logps/chosen": -0.712109386920929,
      "logps/rejected": -0.9571288824081421,
      "loss": 1.0888,
      "nll_loss": 1.0207030773162842,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07124023139476776,
      "rewards/margins": 0.02452239952981472,
      "rewards/rejected": -0.09571532905101776,
      "step": 1500
    },
    {
      "epoch": 0.056585036817747465,
      "grad_norm": 1.2822286445890592,
      "learning_rate": 2.058740050619915e-06,
      "log_odds_chosen": 0.20001220703125,
      "log_odds_ratio": -0.714648425579071,
      "logits/chosen": -0.9224609136581421,
      "logits/rejected": -0.850781261920929,
      "logps/chosen": -0.783984363079071,
      "logps/rejected": -0.929492175579071,
      "loss": 1.0839,
      "nll_loss": 1.034570336341858,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07838134467601776,
      "rewards/margins": 0.014554595574736595,
      "rewards/rejected": -0.09291992336511612,
      "step": 1510
    },
    {
      "epoch": 0.05695977216091136,
      "grad_norm": 1.3782279524587173,
      "learning_rate": 2.0519567041703083e-06,
      "log_odds_chosen": 0.43110352754592896,
      "log_odds_ratio": -0.6044921875,
      "logits/chosen": -0.857421875,
      "logits/rejected": -0.8199218511581421,
      "logps/chosen": -0.6978515386581421,
      "logps/rejected": -0.969531238079071,
      "loss": 1.1167,
      "nll_loss": 1.103124976158142,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06982421875,
      "rewards/margins": 0.02707672119140625,
      "rewards/rejected": -0.09691162407398224,
      "step": 1520
    },
    {
      "epoch": 0.05733450750407525,
      "grad_norm": 1.485179491687544,
      "learning_rate": 2.0452399702596544e-06,
      "log_odds_chosen": 0.3860839903354645,
      "log_odds_ratio": -0.621777355670929,
      "logits/chosen": -0.924023449420929,
      "logits/rejected": -0.8580077886581421,
      "logps/chosen": -0.750195324420929,
      "logps/rejected": -1.0029296875,
      "loss": 1.0829,
      "nll_loss": 1.0613281726837158,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07500000298023224,
      "rewards/margins": 0.02520599402487278,
      "rewards/rejected": -0.10024414211511612,
      "step": 1530
    },
    {
      "epoch": 0.05770924284723914,
      "grad_norm": 1.287404446537744,
      "learning_rate": 2.0385887657505017e-06,
      "log_odds_chosen": 0.489501953125,
      "log_odds_ratio": -0.609375,
      "logits/chosen": -0.8753906488418579,
      "logits/rejected": -0.810742199420929,
      "logps/chosen": -0.766406238079071,
      "logps/rejected": -1.0822265148162842,
      "loss": 1.0997,
      "nll_loss": 0.9964843988418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07658691704273224,
      "rewards/margins": 0.0316314697265625,
      "rewards/rejected": -0.1082763671875,
      "step": 1540
    },
    {
      "epoch": 0.05808397819040303,
      "grad_norm": 1.2170970324116483,
      "learning_rate": 2.032002032003048e-06,
      "log_odds_chosen": 0.31280517578125,
      "log_odds_ratio": -0.6558593511581421,
      "logits/chosen": -0.8970702886581421,
      "logits/rejected": -0.8466796875,
      "logps/chosen": -0.755859375,
      "logps/rejected": -0.9828125238418579,
      "loss": 1.0907,
      "nll_loss": 1.022070288658142,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07565917819738388,
      "rewards/margins": 0.022647857666015625,
      "rewards/rejected": -0.09819336235523224,
      "step": 1550
    },
    {
      "epoch": 0.058458713533566915,
      "grad_norm": 1.298505036622975,
      "learning_rate": 2.025478734167333e-06,
      "log_odds_chosen": 0.3814941346645355,
      "log_odds_ratio": -0.6484375,
      "logits/chosen": -0.995898425579071,
      "logits/rejected": -0.9164062738418579,
      "logps/chosen": -0.812695324420929,
      "logps/rejected": -1.089257836341858,
      "loss": 1.087,
      "nll_loss": 1.0607421398162842,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.08132324367761612,
      "rewards/margins": 0.02763671800494194,
      "rewards/rejected": -0.10895995795726776,
      "step": 1560
    },
    {
      "epoch": 0.05883344887673081,
      "grad_norm": 1.407482670288847,
      "learning_rate": 2.0190178605002747e-06,
      "log_odds_chosen": 0.4596313536167145,
      "log_odds_ratio": -0.600292980670929,
      "logits/chosen": -0.9097656011581421,
      "logits/rejected": -0.8843749761581421,
      "logps/chosen": -0.718945324420929,
      "logps/rejected": -1.0222656726837158,
      "loss": 1.1009,
      "nll_loss": 1.0234375,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07193603366613388,
      "rewards/margins": 0.03037109412252903,
      "rewards/rejected": -0.102294921875,
      "step": 1570
    },
    {
      "epoch": 0.0592081842198947,
      "grad_norm": 1.3228984652027285,
      "learning_rate": 2.0126184217065104e-06,
      "log_odds_chosen": 0.327392578125,
      "log_odds_ratio": -0.6318359375,
      "logits/chosen": -0.9150390625,
      "logits/rejected": -0.85546875,
      "logps/chosen": -0.725390613079071,
      "logps/rejected": -0.941210925579071,
      "loss": 1.0756,
      "nll_loss": 0.9994140863418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07253418117761612,
      "rewards/margins": 0.021572113037109375,
      "rewards/rejected": -0.09406737983226776,
      "step": 1580
    },
    {
      "epoch": 0.05958291956305859,
      "grad_norm": 1.2941617136308547,
      "learning_rate": 2.0062794503020765e-06,
      "log_odds_chosen": 0.3719238340854645,
      "log_odds_ratio": -0.62841796875,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.765625,
      "logps/rejected": -1.018164038658142,
      "loss": 1.0862,
      "nll_loss": 1.0300781726837158,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07655028998851776,
      "rewards/margins": 0.02530059777200222,
      "rewards/rejected": -0.10185547173023224,
      "step": 1590
    },
    {
      "epoch": 0.05995765490622248,
      "grad_norm": 1.4060176779322457,
      "learning_rate": 2e-06,
      "log_odds_chosen": 0.3711181581020355,
      "log_odds_ratio": -0.6400390863418579,
      "logits/chosen": -0.9351562261581421,
      "logits/rejected": -0.900585949420929,
      "logps/chosen": -0.805468738079071,
      "logps/rejected": -1.064062476158142,
      "loss": 1.0837,
      "nll_loss": 1.0398437976837158,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.08054199069738388,
      "rewards/margins": 0.0259857177734375,
      "rewards/rejected": -0.10651855170726776,
      "step": 1600
    },
    {
      "epoch": 0.06033239024938637,
      "grad_norm": 1.3259226293285085,
      "learning_rate": 1.993779145116907e-06,
      "log_odds_chosen": 0.4258178770542145,
      "log_odds_ratio": -0.604687511920929,
      "logits/chosen": -0.930468738079071,
      "logits/rejected": -0.8705078363418579,
      "logps/chosen": -0.716601550579071,
      "logps/rejected": -0.987109363079071,
      "loss": 1.063,
      "nll_loss": 1.0224609375,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07164306938648224,
      "rewards/margins": 0.02704925462603569,
      "rewards/rejected": -0.09869384765625,
      "step": 1610
    },
    {
      "epoch": 0.06070712559255026,
      "grad_norm": 1.3013736677377288,
      "learning_rate": 1.987615979999813e-06,
      "log_odds_chosen": 0.2946533262729645,
      "log_odds_ratio": -0.6617187261581421,
      "logits/chosen": -0.9419921636581421,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -0.784375011920929,
      "logps/rejected": -0.992968738079071,
      "loss": 1.0712,
      "nll_loss": 1.014257788658142,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07843017578125,
      "rewards/margins": 0.02094421349465847,
      "rewards/rejected": -0.09938964992761612,
      "step": 1620
    },
    {
      "epoch": 0.061081860935714154,
      "grad_norm": 1.2342082268089318,
      "learning_rate": 1.9815096184722797e-06,
      "log_odds_chosen": 0.4427734315395355,
      "log_odds_ratio": -0.5888671875,
      "logits/chosen": -0.8804687261581421,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.7445312738418579,
      "logps/rejected": -1.033593773841858,
      "loss": 1.0693,
      "nll_loss": 1.0421874523162842,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07448730617761612,
      "rewards/margins": 0.02888030931353569,
      "rewards/rejected": -0.10332031548023224,
      "step": 1630
    },
    {
      "epoch": 0.06145659627887804,
      "grad_norm": 1.408555307064288,
      "learning_rate": 1.9754591932991793e-06,
      "log_odds_chosen": 0.384033203125,
      "log_odds_ratio": -0.6099609136581421,
      "logits/chosen": -0.8785156011581421,
      "logits/rejected": -0.7972656488418579,
      "logps/chosen": -0.710156261920929,
      "logps/rejected": -0.956250011920929,
      "loss": 1.1033,
      "nll_loss": 1.023046851158142,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07097168266773224,
      "rewards/margins": 0.02469024620950222,
      "rewards/rejected": -0.09562988579273224,
      "step": 1640
    },
    {
      "epoch": 0.061831331622041935,
      "grad_norm": 1.3330696703737306,
      "learning_rate": 1.9694638556693235e-06,
      "log_odds_chosen": 0.25103759765625,
      "log_odds_ratio": -0.67333984375,
      "logits/chosen": -0.8335937261581421,
      "logits/rejected": -0.7652343511581421,
      "logps/chosen": -0.7310546636581421,
      "logps/rejected": -0.868359386920929,
      "loss": 1.0759,
      "nll_loss": 1.007421851158142,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.07314453274011612,
      "rewards/margins": 0.013775634579360485,
      "rewards/rejected": -0.08688964694738388,
      "step": 1650
    },
    {
      "epoch": 0.06220606696520582,
      "grad_norm": 1.3237069549273253,
      "learning_rate": 1.963522774695264e-06,
      "log_odds_chosen": 0.33503419160842896,
      "log_odds_ratio": -0.643847644329071,
      "logits/chosen": -0.87890625,
      "logits/rejected": -0.836718738079071,
      "logps/chosen": -0.8111327886581421,
      "logps/rejected": -1.018164038658142,
      "loss": 1.0721,
      "nll_loss": 1.0275390148162842,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.08112792670726776,
      "rewards/margins": 0.02072448655962944,
      "rewards/rejected": -0.1019287109375,
      "step": 1660
    },
    {
      "epoch": 0.06258080230836971,
      "grad_norm": 1.2756835830261364,
      "learning_rate": 1.9576351369295853e-06,
      "log_odds_chosen": 0.38520509004592896,
      "log_odds_ratio": -0.6468750238418579,
      "logits/chosen": -0.8238281011581421,
      "logits/rejected": -0.7999023199081421,
      "logps/chosen": -0.7494140863418579,
      "logps/rejected": -1.0089843273162842,
      "loss": 1.0862,
      "nll_loss": 1.020898461341858,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07493896782398224,
      "rewards/margins": 0.025970458984375,
      "rewards/rejected": -0.10096435248851776,
      "step": 1670
    },
    {
      "epoch": 0.06295553765153361,
      "grad_norm": 1.297319409289796,
      "learning_rate": 1.951800145897066e-06,
      "log_odds_chosen": 0.40363770723342896,
      "log_odds_ratio": -0.621289074420929,
      "logits/chosen": -0.8646484613418579,
      "logits/rejected": -0.786425769329071,
      "logps/chosen": -0.740429699420929,
      "logps/rejected": -1.017675757408142,
      "loss": 1.0735,
      "nll_loss": 1.040624976158142,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07398681342601776,
      "rewards/margins": 0.02772064134478569,
      "rewards/rejected": -0.1016845703125,
      "step": 1680
    },
    {
      "epoch": 0.0633302729946975,
      "grad_norm": 1.3121380887776866,
      "learning_rate": 1.9460170216420797e-06,
      "log_odds_chosen": 0.33452147245407104,
      "log_odds_ratio": -0.657031238079071,
      "logits/chosen": -0.876757800579071,
      "logits/rejected": -0.8173828125,
      "logps/chosen": -0.773242175579071,
      "logps/rejected": -1.009374976158142,
      "loss": 1.0792,
      "nll_loss": 1.0359375476837158,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07740478217601776,
      "rewards/margins": 0.02359466627240181,
      "rewards/rejected": -0.10100097954273224,
      "step": 1690
    },
    {
      "epoch": 0.06370500833786139,
      "grad_norm": 1.3251627317223647,
      "learning_rate": 1.9402850002906637e-06,
      "log_odds_chosen": 0.38493651151657104,
      "log_odds_ratio": -0.6273437738418579,
      "logits/chosen": -0.833984375,
      "logits/rejected": -0.756640613079071,
      "logps/chosen": -0.7906249761581421,
      "logps/rejected": -1.060546875,
      "loss": 1.0884,
      "nll_loss": 1.098046898841858,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07912597805261612,
      "rewards/margins": 0.026927947998046875,
      "rewards/rejected": -0.10605468600988388,
      "step": 1700
    },
    {
      "epoch": 0.06407974368102527,
      "grad_norm": 1.3717927477696763,
      "learning_rate": 1.9346033336266974e-06,
      "log_odds_chosen": 0.30615234375,
      "log_odds_ratio": -0.6625000238418579,
      "logits/chosen": -0.8466796875,
      "logits/rejected": -0.800585925579071,
      "logps/chosen": -0.7787109613418579,
      "logps/rejected": -0.996289074420929,
      "loss": 1.0679,
      "nll_loss": 1.065820336341858,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07794189453125,
      "rewards/margins": 0.02177734300494194,
      "rewards/rejected": -0.099609375,
      "step": 1710
    },
    {
      "epoch": 0.06445447902418916,
      "grad_norm": 1.3119919385298935,
      "learning_rate": 1.9289712886816486e-06,
      "log_odds_chosen": 0.44781494140625,
      "log_odds_ratio": -0.6063476800918579,
      "logits/chosen": -0.834765613079071,
      "logits/rejected": -0.7837890386581421,
      "logps/chosen": -0.775195300579071,
      "logps/rejected": -1.067968726158142,
      "loss": 1.0907,
      "nll_loss": 1.0177733898162842,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07747802883386612,
      "rewards/margins": 0.029384613037109375,
      "rewards/rejected": -0.10683593899011612,
      "step": 1720
    },
    {
      "epoch": 0.06482921436735306,
      "grad_norm": 1.402650119550694,
      "learning_rate": 1.92338814733738e-06,
      "log_odds_chosen": 0.4808349609375,
      "log_odds_ratio": -0.601367175579071,
      "logits/chosen": -0.8525390625,
      "logits/rejected": -0.791015625,
      "logps/chosen": -0.736328125,
      "logps/rejected": -1.0988280773162842,
      "loss": 1.0918,
      "nll_loss": 1.01171875,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.0736083984375,
      "rewards/margins": 0.036346435546875,
      "rewards/rejected": -0.1099853515625,
      "step": 1730
    },
    {
      "epoch": 0.06520394971051695,
      "grad_norm": 1.3363750674358204,
      "learning_rate": 1.9178532059415367e-06,
      "log_odds_chosen": 0.4194580018520355,
      "log_odds_ratio": -0.6246093511581421,
      "logits/chosen": -0.8642578125,
      "logits/rejected": -0.8255859613418579,
      "logps/chosen": -0.724804699420929,
      "logps/rejected": -1.0138671398162842,
      "loss": 1.073,
      "nll_loss": 1.0673828125,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.072509765625,
      "rewards/margins": 0.02894439734518528,
      "rewards/rejected": -0.1014404296875,
      "step": 1740
    },
    {
      "epoch": 0.06557868505368084,
      "grad_norm": 1.2533428181469561,
      "learning_rate": 1.9123657749350298e-06,
      "log_odds_chosen": 0.22043457627296448,
      "log_odds_ratio": -0.690625011920929,
      "logits/chosen": -0.861328125,
      "logits/rejected": -0.794726550579071,
      "logps/chosen": -0.7728515863418579,
      "logps/rejected": -0.9345703125,
      "loss": 1.0821,
      "nll_loss": 1.0216796398162842,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07729492336511612,
      "rewards/margins": 0.016178131103515625,
      "rewards/rejected": -0.09344482421875,
      "step": 1750
    },
    {
      "epoch": 0.06595342039684472,
      "grad_norm": 1.3494710425567795,
      "learning_rate": 1.9069251784911844e-06,
      "log_odds_chosen": 0.401123046875,
      "log_odds_ratio": -0.612500011920929,
      "logits/chosen": -0.8583984375,
      "logits/rejected": -0.8047851324081421,
      "logps/chosen": -0.728515625,
      "logps/rejected": -0.9898437261581421,
      "loss": 1.0691,
      "nll_loss": 1.0197265148162842,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07290039211511612,
      "rewards/margins": 0.02613678015768528,
      "rewards/rejected": -0.09904785454273224,
      "step": 1760
    },
    {
      "epoch": 0.06632815574000862,
      "grad_norm": 1.357494906035872,
      "learning_rate": 1.9015307541661132e-06,
      "log_odds_chosen": 0.311676025390625,
      "log_odds_ratio": -0.646777331829071,
      "logits/chosen": -0.872265636920929,
      "logits/rejected": -0.8197265863418579,
      "logps/chosen": -0.748242199420929,
      "logps/rejected": -0.9654296636581421,
      "loss": 1.0622,
      "nll_loss": 1.013281226158142,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07485351711511612,
      "rewards/margins": 0.02165374718606472,
      "rewards/rejected": -0.09656982123851776,
      "step": 1770
    },
    {
      "epoch": 0.06670289108317251,
      "grad_norm": 1.3131386250950805,
      "learning_rate": 1.8961818525599089e-06,
      "log_odds_chosen": 0.3556884825229645,
      "log_odds_ratio": -0.637499988079071,
      "logits/chosen": -0.8359375,
      "logits/rejected": -0.766796886920929,
      "logps/chosen": -0.726855456829071,
      "logps/rejected": -0.9677734375,
      "loss": 1.0864,
      "nll_loss": 1.024804711341858,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07266845554113388,
      "rewards/margins": 0.02412262000143528,
      "rewards/rejected": -0.09676513820886612,
      "step": 1780
    },
    {
      "epoch": 0.0670776264263364,
      "grad_norm": 1.2926226285730016,
      "learning_rate": 1.890877836988262e-06,
      "log_odds_chosen": 0.2973388731479645,
      "log_odds_ratio": -0.666210949420929,
      "logits/chosen": -0.847460925579071,
      "logits/rejected": -0.818359375,
      "logps/chosen": -0.786328136920929,
      "logps/rejected": -0.984570324420929,
      "loss": 1.0894,
      "nll_loss": 1.052343726158142,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07864990085363388,
      "rewards/margins": 0.01981353759765625,
      "rewards/rejected": -0.09841308742761612,
      "step": 1790
    },
    {
      "epoch": 0.06745236176950029,
      "grad_norm": 1.3746672584560344,
      "learning_rate": 1.8856180831641269e-06,
      "log_odds_chosen": 0.22022704780101776,
      "log_odds_ratio": -0.694140613079071,
      "logits/chosen": -0.8666015863418579,
      "logits/rejected": -0.8216797113418579,
      "logps/chosen": -0.744140625,
      "logps/rejected": -0.917187511920929,
      "loss": 1.069,
      "nll_loss": 0.9447265863418579,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.07436523586511612,
      "rewards/margins": 0.01735839806497097,
      "rewards/rejected": -0.09177245944738388,
      "step": 1800
    },
    {
      "epoch": 0.06782709711266419,
      "grad_norm": 1.3676178666805452,
      "learning_rate": 1.8804019788890737e-06,
      "log_odds_chosen": 0.19434814155101776,
      "log_odds_ratio": -0.6904296875,
      "logits/chosen": -0.8121093511581421,
      "logits/rejected": -0.7587890625,
      "logps/chosen": -0.777539074420929,
      "logps/rejected": -0.8999999761581421,
      "loss": 1.0774,
      "nll_loss": 1.050195336341858,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07778320461511612,
      "rewards/margins": 0.012263488955795765,
      "rewards/rejected": -0.09003905951976776,
      "step": 1810
    },
    {
      "epoch": 0.06820183245582807,
      "grad_norm": 1.267367401603532,
      "learning_rate": 1.8752289237539816e-06,
      "log_odds_chosen": 0.28565675020217896,
      "log_odds_ratio": -0.655957043170929,
      "logits/chosen": -0.9091796875,
      "logits/rejected": -0.8714843988418579,
      "logps/chosen": -0.763476550579071,
      "logps/rejected": -0.9498046636581421,
      "loss": 1.0852,
      "nll_loss": 1.0070312023162842,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07633056491613388,
      "rewards/margins": 0.01862945593893528,
      "rewards/rejected": -0.09495849907398224,
      "step": 1820
    },
    {
      "epoch": 0.06857656779899196,
      "grad_norm": 1.3304101630010832,
      "learning_rate": 1.8700983288487376e-06,
      "log_odds_chosen": 0.44224852323532104,
      "log_odds_ratio": -0.611621081829071,
      "logits/chosen": -0.8677734136581421,
      "logits/rejected": -0.7923828363418579,
      "logps/chosen": -0.6915038824081421,
      "logps/rejected": -0.9878906011581421,
      "loss": 1.0695,
      "nll_loss": 0.9378906488418579,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06912841647863388,
      "rewards/margins": 0.02964324876666069,
      "rewards/rejected": -0.09880371391773224,
      "step": 1830
    },
    {
      "epoch": 0.06895130314215585,
      "grad_norm": 1.4114627393137849,
      "learning_rate": 1.8650096164806275e-06,
      "log_odds_chosen": 0.4996581971645355,
      "log_odds_ratio": -0.592089831829071,
      "logits/chosen": -0.890625,
      "logits/rejected": -0.8072265386581421,
      "logps/chosen": -0.7261718511581421,
      "logps/rejected": -1.054101586341858,
      "loss": 1.074,
      "nll_loss": 1.0535156726837158,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07260742038488388,
      "rewards/margins": 0.032791901379823685,
      "rewards/rejected": -0.10532226413488388,
      "step": 1840
    },
    {
      "epoch": 0.06932603848531975,
      "grad_norm": 1.4644025057280838,
      "learning_rate": 1.8599622199011084e-06,
      "log_odds_chosen": 0.36848145723342896,
      "log_odds_ratio": -0.635058581829071,
      "logits/chosen": -0.939453125,
      "logits/rejected": -0.875781238079071,
      "logps/chosen": -0.736523449420929,
      "logps/rejected": -1.010156273841858,
      "loss": 1.0672,
      "nll_loss": 1.0107421875,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07365722954273224,
      "rewards/margins": 0.02728118933737278,
      "rewards/rejected": -0.10097656399011612,
      "step": 1850
    },
    {
      "epoch": 0.06970077382848364,
      "grad_norm": 1.379349827089515,
      "learning_rate": 1.854955583040673e-06,
      "log_odds_chosen": 0.447845458984375,
      "log_odds_ratio": -0.619335949420929,
      "logits/chosen": -0.8880859613418579,
      "logits/rejected": -0.8291015625,
      "logps/chosen": -0.71484375,
      "logps/rejected": -1.005859375,
      "loss": 1.0721,
      "nll_loss": 1.03515625,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07155761867761612,
      "rewards/margins": 0.029033660888671875,
      "rewards/rejected": -0.10053710639476776,
      "step": 1860
    },
    {
      "epoch": 0.07007550917164752,
      "grad_norm": 1.4298592744087875,
      "learning_rate": 1.849989160251521e-06,
      "log_odds_chosen": 0.47272950410842896,
      "log_odds_ratio": -0.607128918170929,
      "logits/chosen": -0.9462890625,
      "logits/rejected": -0.845703125,
      "logps/chosen": -0.7115234136581421,
      "logps/rejected": -1.046289086341858,
      "loss": 1.0675,
      "nll_loss": 0.90234375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07114257663488388,
      "rewards/margins": 0.033504486083984375,
      "rewards/rejected": -0.1046142578125,
      "step": 1870
    },
    {
      "epoch": 0.07045024451481141,
      "grad_norm": 1.305246089227333,
      "learning_rate": 1.8450624160577701e-06,
      "log_odds_chosen": 0.452392578125,
      "log_odds_ratio": -0.6162109375,
      "logits/chosen": -0.8853515386581421,
      "logits/rejected": -0.8558593988418579,
      "logps/chosen": -0.7457031011581421,
      "logps/rejected": -1.051367163658142,
      "loss": 1.0728,
      "nll_loss": 1.052734375,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07456054538488388,
      "rewards/margins": 0.030605316162109375,
      "rewards/rejected": -0.10520019382238388,
      "step": 1880
    },
    {
      "epoch": 0.0708249798579753,
      "grad_norm": 1.3353780265259632,
      "learning_rate": 1.8401748249129445e-06,
      "log_odds_chosen": 0.3108581602573395,
      "log_odds_ratio": -0.6416015625,
      "logits/chosen": -0.907031238079071,
      "logits/rejected": -0.8427734375,
      "logps/chosen": -0.758593738079071,
      "logps/rejected": -0.9857422113418579,
      "loss": 1.0875,
      "nll_loss": 0.974804699420929,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07584228366613388,
      "rewards/margins": 0.02272186242043972,
      "rewards/rejected": -0.09855957329273224,
      "step": 1890
    },
    {
      "epoch": 0.0711997152011392,
      "grad_norm": 1.3697330213538628,
      "learning_rate": 1.8353258709644938e-06,
      "log_odds_chosen": 0.29603272676467896,
      "log_odds_ratio": -0.67236328125,
      "logits/chosen": -0.8677734136581421,
      "logits/rejected": -0.8213866949081421,
      "logps/chosen": -0.7572265863418579,
      "logps/rejected": -0.9755859375,
      "loss": 1.0721,
      "nll_loss": 1.009374976158142,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07572021335363388,
      "rewards/margins": 0.02181396447122097,
      "rewards/rejected": -0.09746094048023224,
      "step": 1900
    },
    {
      "epoch": 0.07157445054430309,
      "grad_norm": 1.292421369067475,
      "learning_rate": 1.830515047825102e-06,
      "log_odds_chosen": 0.28948974609375,
      "log_odds_ratio": -0.68408203125,
      "logits/chosen": -0.8472656011581421,
      "logits/rejected": -0.810351550579071,
      "logps/chosen": -0.792773425579071,
      "logps/rejected": -1.0066406726837158,
      "loss": 1.0632,
      "nll_loss": 1.066015601158142,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07919921725988388,
      "rewards/margins": 0.021395111456513405,
      "rewards/rejected": -0.10067138820886612,
      "step": 1910
    },
    {
      "epoch": 0.07194918588746697,
      "grad_norm": 1.2855512947269945,
      "learning_rate": 1.8257418583505536e-06,
      "log_odds_chosen": 0.4115966856479645,
      "log_odds_ratio": -0.637011706829071,
      "logits/chosen": -0.7964843511581421,
      "logits/rejected": -0.769335925579071,
      "logps/chosen": -0.74658203125,
      "logps/rejected": -1.013671875,
      "loss": 1.0749,
      "nll_loss": 1.0753905773162842,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07473144680261612,
      "rewards/margins": 0.02677154541015625,
      "rewards/rejected": -0.1014404296875,
      "step": 1920
    },
    {
      "epoch": 0.07232392123063086,
      "grad_norm": 1.3662588197580339,
      "learning_rate": 1.8210058144239416e-06,
      "log_odds_chosen": 0.39765626192092896,
      "log_odds_ratio": -0.634765625,
      "logits/chosen": -0.837890625,
      "logits/rejected": -0.780468761920929,
      "logps/chosen": -0.752734363079071,
      "logps/rejected": -1.028906226158142,
      "loss": 1.0844,
      "nll_loss": 1.019921898841858,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07528076320886612,
      "rewards/margins": 0.02755584754049778,
      "rewards/rejected": -0.10280761867761612,
      "step": 1930
    },
    {
      "epoch": 0.07269865657379476,
      "grad_norm": 1.4842393598627575,
      "learning_rate": 1.816306436745999e-06,
      "log_odds_chosen": 0.38499754667282104,
      "log_odds_ratio": -0.612500011920929,
      "logits/chosen": -0.9095703363418579,
      "logits/rejected": -0.807812511920929,
      "logps/chosen": -0.750195324420929,
      "logps/rejected": -0.995312511920929,
      "loss": 1.0702,
      "nll_loss": 0.978710949420929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.074951171875,
      "rewards/margins": 0.02449646033346653,
      "rewards/rejected": -0.09951172024011612,
      "step": 1940
    },
    {
      "epoch": 0.07307339191695865,
      "grad_norm": 1.3099971593198165,
      "learning_rate": 1.8116432546313529e-06,
      "log_odds_chosen": 0.2970214784145355,
      "log_odds_ratio": -0.6566406488418579,
      "logits/chosen": -0.87158203125,
      "logits/rejected": -0.8236328363418579,
      "logps/chosen": -0.7699218988418579,
      "logps/rejected": -0.973828136920929,
      "loss": 1.097,
      "nll_loss": 1.022851586341858,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07706298679113388,
      "rewards/margins": 0.02027587965130806,
      "rewards/rejected": -0.09731445461511612,
      "step": 1950
    },
    {
      "epoch": 0.07344812726012254,
      "grad_norm": 1.3420775481914895,
      "learning_rate": 1.8070158058105026e-06,
      "log_odds_chosen": 0.3552612364292145,
      "log_odds_ratio": -0.634570300579071,
      "logits/chosen": -0.8687499761581421,
      "logits/rejected": -0.8099609613418579,
      "logps/chosen": -0.740234375,
      "logps/rejected": -0.972460925579071,
      "loss": 1.1012,
      "nll_loss": 1.0007812976837158,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07407226413488388,
      "rewards/margins": 0.02322540245950222,
      "rewards/rejected": -0.09738769382238388,
      "step": 1960
    },
    {
      "epoch": 0.07382286260328642,
      "grad_norm": 1.3484054898771898,
      "learning_rate": 1.8024236362373315e-06,
      "log_odds_chosen": 0.540417492389679,
      "log_odds_ratio": -0.5780273675918579,
      "logits/chosen": -0.8427734375,
      "logits/rejected": -0.786328136920929,
      "logps/chosen": -0.6634765863418579,
      "logps/rejected": -1.0286133289337158,
      "loss": 1.0595,
      "nll_loss": 0.978320300579071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06633301079273224,
      "rewards/margins": 0.03653259202837944,
      "rewards/rejected": -0.10288085788488388,
      "step": 1970
    },
    {
      "epoch": 0.07419759794645032,
      "grad_norm": 1.4609317567149596,
      "learning_rate": 1.7978662999019787e-06,
      "log_odds_chosen": 0.3894104063510895,
      "log_odds_ratio": -0.6319335699081421,
      "logits/chosen": -0.9183593988418579,
      "logits/rejected": -0.8433593511581421,
      "logps/chosen": -0.728710949420929,
      "logps/rejected": -1.0011718273162842,
      "loss": 1.0903,
      "nll_loss": 1.0283203125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07294921576976776,
      "rewards/margins": 0.02725067175924778,
      "rewards/rejected": -0.10014648735523224,
      "step": 1980
    },
    {
      "epoch": 0.07457233328961421,
      "grad_norm": 1.3164473649466708,
      "learning_rate": 1.793343358648881e-06,
      "log_odds_chosen": 0.33714598417282104,
      "log_odds_ratio": -0.6407226324081421,
      "logits/chosen": -0.902539074420929,
      "logits/rejected": -0.8568359613418579,
      "logps/chosen": -0.7046874761581421,
      "logps/rejected": -0.9117187261581421,
      "loss": 1.0439,
      "nll_loss": 0.994335949420929,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07052002102136612,
      "rewards/margins": 0.02064361609518528,
      "rewards/rejected": -0.0911865234375,
      "step": 1990
    },
    {
      "epoch": 0.0749470686327781,
      "grad_norm": 1.2440599998070594,
      "learning_rate": 1.7888543819998317e-06,
      "log_odds_chosen": 0.38646239042282104,
      "log_odds_ratio": -0.6473633050918579,
      "logits/chosen": -0.7935546636581421,
      "logits/rejected": -0.7777343988418579,
      "logps/chosen": -0.729296863079071,
      "logps/rejected": -0.9853515625,
      "loss": 1.0746,
      "nll_loss": 1.0294921398162842,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07292480766773224,
      "rewards/margins": 0.02563018724322319,
      "rewards/rejected": -0.0985107421875,
      "step": 2000
    },
    {
      "epoch": 0.07532180397594199,
      "grad_norm": 1.4175034833227922,
      "learning_rate": 1.7843989469818819e-06,
      "log_odds_chosen": 0.38856202363967896,
      "log_odds_ratio": -0.6153320074081421,
      "logits/chosen": -0.889453113079071,
      "logits/rejected": -0.805468738079071,
      "logps/chosen": -0.7255859375,
      "logps/rejected": -0.9886718988418579,
      "loss": 1.0779,
      "nll_loss": 1.017968773841858,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07257080078125,
      "rewards/margins": 0.026336669921875,
      "rewards/rejected": -0.09888915717601776,
      "step": 2010
    },
    {
      "epoch": 0.07569653931910589,
      "grad_norm": 1.3898954480902352,
      "learning_rate": 1.779976637959939e-06,
      "log_odds_chosen": 0.4620361328125,
      "log_odds_ratio": -0.594531238079071,
      "logits/chosen": -0.906445324420929,
      "logits/rejected": -0.825878918170929,
      "logps/chosen": -0.7376953363418579,
      "logps/rejected": -1.0634765625,
      "loss": 1.0817,
      "nll_loss": 1.029296875,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07382812350988388,
      "rewards/margins": 0.03250274807214737,
      "rewards/rejected": -0.1063232421875,
      "step": 2020
    },
    {
      "epoch": 0.07607127466226977,
      "grad_norm": 1.3356977181932346,
      "learning_rate": 1.7755870464739012e-06,
      "log_odds_chosen": 0.38068848848342896,
      "log_odds_ratio": -0.6366211175918579,
      "logits/chosen": -0.8296874761581421,
      "logits/rejected": -0.7861328125,
      "logps/chosen": -0.742382824420929,
      "logps/rejected": -0.9916015863418579,
      "loss": 1.0645,
      "nll_loss": 0.9644531011581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07423095405101776,
      "rewards/margins": 0.024895478039979935,
      "rewards/rejected": -0.09916992485523224,
      "step": 2030
    },
    {
      "epoch": 0.07644601000543366,
      "grad_norm": 1.4155526013193411,
      "learning_rate": 1.7712297710801907e-06,
      "log_odds_chosen": 0.3899169862270355,
      "log_odds_ratio": -0.6255859136581421,
      "logits/chosen": -0.856249988079071,
      "logits/rejected": -0.801562488079071,
      "logps/chosen": -0.7300781011581421,
      "logps/rejected": -0.9697265625,
      "loss": 1.0541,
      "nll_loss": 1.000585913658142,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07297363132238388,
      "rewards/margins": 0.02406463585793972,
      "rewards/rejected": -0.09698486328125,
      "step": 2040
    },
    {
      "epoch": 0.07682074534859755,
      "grad_norm": 1.460032469588326,
      "learning_rate": 1.7669044171975444e-06,
      "log_odds_chosen": 0.3195434510707855,
      "log_odds_ratio": -0.637499988079071,
      "logits/chosen": -0.8521484136581421,
      "logits/rejected": -0.7969726324081421,
      "logps/chosen": -0.727734386920929,
      "logps/rejected": -0.94140625,
      "loss": 1.086,
      "nll_loss": 1.0505859851837158,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07281494140625,
      "rewards/margins": 0.02138977125287056,
      "rewards/rejected": -0.09415283054113388,
      "step": 2050
    },
    {
      "epoch": 0.07719548069176144,
      "grad_norm": 1.4440770794422741,
      "learning_rate": 1.7626105969569268e-06,
      "log_odds_chosen": 0.3939575254917145,
      "log_odds_ratio": -0.612011730670929,
      "logits/chosen": -0.8802734613418579,
      "logits/rejected": -0.827929675579071,
      "logps/chosen": -0.710742175579071,
      "logps/rejected": -0.9820312261581421,
      "loss": 1.0806,
      "nll_loss": 1.0087890625,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07110595703125,
      "rewards/margins": 0.027159880846738815,
      "rewards/rejected": -0.09822998195886612,
      "step": 2060
    },
    {
      "epoch": 0.07757021603492534,
      "grad_norm": 1.272919851362208,
      "learning_rate": 1.758347929055432e-06,
      "log_odds_chosen": 0.32081300020217896,
      "log_odds_ratio": -0.6561523675918579,
      "logits/chosen": -0.8011718988418579,
      "logits/rejected": -0.7417968511581421,
      "logps/chosen": -0.721875011920929,
      "logps/rejected": -0.9330078363418579,
      "loss": 1.0645,
      "nll_loss": 1.0242187976837158,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07215575873851776,
      "rewards/margins": 0.02108459547162056,
      "rewards/rejected": -0.09328613430261612,
      "step": 2070
    },
    {
      "epoch": 0.07794495137808923,
      "grad_norm": 1.2912601358926015,
      "learning_rate": 1.7541160386140582e-06,
      "log_odds_chosen": 0.3819946348667145,
      "log_odds_ratio": -0.6255859136581421,
      "logits/chosen": -0.9169921875,
      "logits/rejected": -0.8333984613418579,
      "logps/chosen": -0.7255859375,
      "logps/rejected": -0.981249988079071,
      "loss": 1.0577,
      "nll_loss": 1.0232422351837158,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07252196967601776,
      "rewards/margins": 0.02557525597512722,
      "rewards/rejected": -0.09809570014476776,
      "step": 2080
    },
    {
      "epoch": 0.07831968672125311,
      "grad_norm": 1.3398020221806541,
      "learning_rate": 1.7499145570392284e-06,
      "log_odds_chosen": 0.1431884765625,
      "log_odds_ratio": -0.711230456829071,
      "logits/chosen": -0.8779296875,
      "logits/rejected": -0.841601550579071,
      "logps/chosen": -0.7919921875,
      "logps/rejected": -0.8910156488418579,
      "loss": 1.0759,
      "nll_loss": 1.025390625,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07917480170726776,
      "rewards/margins": 0.009982299990952015,
      "rewards/rejected": -0.08916015923023224,
      "step": 2090
    },
    {
      "epoch": 0.078694422064417,
      "grad_norm": 1.3626027606308637,
      "learning_rate": 1.745743121887939e-06,
      "log_odds_chosen": 0.33195799589157104,
      "log_odds_ratio": -0.6585937738418579,
      "logits/chosen": -0.921875,
      "logits/rejected": -0.9037109613418579,
      "logps/chosen": -0.7748047113418579,
      "logps/rejected": -1.001953125,
      "loss": 1.0604,
      "nll_loss": 0.9976562261581421,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.07744140923023224,
      "rewards/margins": 0.022718047723174095,
      "rewards/rejected": -0.1002197265625,
      "step": 2100
    },
    {
      "epoch": 0.0790691574075809,
      "grad_norm": 1.4896003360708892,
      "learning_rate": 1.7416013767364324e-06,
      "log_odds_chosen": 0.36536866426467896,
      "log_odds_ratio": -0.614453136920929,
      "logits/chosen": -0.859179675579071,
      "logits/rejected": -0.8001953363418579,
      "logps/chosen": -0.7232421636581421,
      "logps/rejected": -0.959765613079071,
      "loss": 1.0468,
      "nll_loss": 1.0666015148162842,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07232666015625,
      "rewards/margins": 0.023737335577607155,
      "rewards/rejected": -0.0960693359375,
      "step": 2110
    },
    {
      "epoch": 0.07944389275074479,
      "grad_norm": 1.4251068514947394,
      "learning_rate": 1.7374889710522776e-06,
      "log_odds_chosen": 0.4336791932582855,
      "log_odds_ratio": -0.630566418170929,
      "logits/chosen": -0.9007812738418579,
      "logits/rejected": -0.830859363079071,
      "logps/chosen": -0.7611328363418579,
      "logps/rejected": -1.0744140148162842,
      "loss": 1.0597,
      "nll_loss": 0.9658203125,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.076171875,
      "rewards/margins": 0.03127593919634819,
      "rewards/rejected": -0.10744629055261612,
      "step": 2120
    },
    {
      "epoch": 0.07981862809390868,
      "grad_norm": 1.3885679799846453,
      "learning_rate": 1.7334055600697579e-06,
      "log_odds_chosen": 0.2954467833042145,
      "log_odds_ratio": -0.6666015386581421,
      "logits/chosen": -0.884570300579071,
      "logits/rejected": -0.8501952886581421,
      "logps/chosen": -0.7275390625,
      "logps/rejected": -0.925976574420929,
      "loss": 1.0621,
      "nll_loss": 1.0078125,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07282714545726776,
      "rewards/margins": 0.01975708082318306,
      "rewards/rejected": -0.09261474758386612,
      "step": 2130
    },
    {
      "epoch": 0.08019336343707256,
      "grad_norm": 1.3086645828024155,
      "learning_rate": 1.7293508046684678e-06,
      "log_odds_chosen": 0.3739990293979645,
      "log_odds_ratio": -0.62158203125,
      "logits/chosen": -0.8892577886581421,
      "logits/rejected": -0.8345702886581421,
      "logps/chosen": -0.685839831829071,
      "logps/rejected": -0.928906261920929,
      "loss": 1.0585,
      "nll_loss": 0.9906250238418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06854248046875,
      "rewards/margins": 0.024321746081113815,
      "rewards/rejected": -0.09287109225988388,
      "step": 2140
    },
    {
      "epoch": 0.08056809878023646,
      "grad_norm": 1.373269288551161,
      "learning_rate": 1.7253243712550145e-06,
      "log_odds_chosen": 0.2845825254917145,
      "log_odds_ratio": -0.6781250238418579,
      "logits/chosen": -0.9267578125,
      "logits/rejected": -0.8587890863418579,
      "logps/chosen": -0.784960925579071,
      "logps/rejected": -0.990039050579071,
      "loss": 1.0675,
      "nll_loss": 0.9710937738418579,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07851562649011612,
      "rewards/margins": 0.02050476148724556,
      "rewards/rejected": -0.09897460788488388,
      "step": 2150
    },
    {
      "epoch": 0.08094283412340035,
      "grad_norm": 1.30412566131051,
      "learning_rate": 1.7213259316477406e-06,
      "log_odds_chosen": 0.4154052734375,
      "log_odds_ratio": -0.6180664300918579,
      "logits/chosen": -0.8785156011581421,
      "logits/rejected": -0.796093761920929,
      "logps/chosen": -0.748046875,
      "logps/rejected": -1.025781273841858,
      "loss": 1.0669,
      "nll_loss": 1.033789038658142,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07480468600988388,
      "rewards/margins": 0.02768859826028347,
      "rewards/rejected": -0.1024169921875,
      "step": 2160
    },
    {
      "epoch": 0.08131756946656424,
      "grad_norm": 1.2858863002690737,
      "learning_rate": 1.7173551629643674e-06,
      "log_odds_chosen": 0.39222413301467896,
      "log_odds_ratio": -0.62548828125,
      "logits/chosen": -0.8720703125,
      "logits/rejected": -0.774707019329071,
      "logps/chosen": -0.759765625,
      "logps/rejected": -1.0330078601837158,
      "loss": 1.0815,
      "nll_loss": 0.9833984375,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07602538913488388,
      "rewards/margins": 0.02726135216653347,
      "rewards/rejected": -0.10324706882238388,
      "step": 2170
    },
    {
      "epoch": 0.08169230480972813,
      "grad_norm": 1.4415055813726187,
      "learning_rate": 1.713411747512477e-06,
      "log_odds_chosen": 0.41252440214157104,
      "log_odds_ratio": -0.609570324420929,
      "logits/chosen": -0.9185546636581421,
      "logits/rejected": -0.853710949420929,
      "logps/chosen": -0.7298828363418579,
      "logps/rejected": -1.0037109851837158,
      "loss": 1.042,
      "nll_loss": 0.9740234613418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07298584282398224,
      "rewards/margins": 0.02740020677447319,
      "rewards/rejected": -0.10039062798023224,
      "step": 2180
    },
    {
      "epoch": 0.08206704015289203,
      "grad_norm": 1.2388919200338284,
      "learning_rate": 1.709495372682753e-06,
      "log_odds_chosen": 0.45097655057907104,
      "log_odds_ratio": -0.6187499761581421,
      "logits/chosen": -0.8705078363418579,
      "logits/rejected": -0.8167968988418579,
      "logps/chosen": -0.770703136920929,
      "logps/rejected": -1.083593726158142,
      "loss": 1.0639,
      "nll_loss": 0.9720703363418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07708740234375,
      "rewards/margins": 0.03138275071978569,
      "rewards/rejected": -0.1083984375,
      "step": 2190
    },
    {
      "epoch": 0.08244177549605591,
      "grad_norm": 1.508194544489001,
      "learning_rate": 1.7056057308448832e-06,
      "log_odds_chosen": 0.44237059354782104,
      "log_odds_ratio": -0.626660168170929,
      "logits/chosen": -0.834765613079071,
      "logits/rejected": -0.775195300579071,
      "logps/chosen": -0.776562511920929,
      "logps/rejected": -1.0876953601837158,
      "loss": 1.1025,
      "nll_loss": 1.0068359375,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07763671875,
      "rewards/margins": 0.03108367882668972,
      "rewards/rejected": -0.1087646484375,
      "step": 2200
    },
    {
      "epoch": 0.0828165108392198,
      "grad_norm": 1.3279862929207142,
      "learning_rate": 1.701742519246068e-06,
      "log_odds_chosen": 0.32635498046875,
      "log_odds_ratio": -0.6451171636581421,
      "logits/chosen": -0.8785156011581421,
      "logits/rejected": -0.822070300579071,
      "logps/chosen": -0.740429699420929,
      "logps/rejected": -0.9671875238418579,
      "loss": 1.071,
      "nll_loss": 1.003320336341858,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07403564453125,
      "rewards/margins": 0.02262573316693306,
      "rewards/rejected": -0.09665527194738388,
      "step": 2210
    },
    {
      "epoch": 0.08319124618238369,
      "grad_norm": 1.293965997643241,
      "learning_rate": 1.6979054399120355e-06,
      "log_odds_chosen": 0.41185301542282104,
      "log_odds_ratio": -0.615527331829071,
      "logits/chosen": -0.801953136920929,
      "logits/rejected": -0.741015613079071,
      "logps/chosen": -0.705078125,
      "logps/rejected": -0.9671875238418579,
      "loss": 1.0677,
      "nll_loss": 0.9664062261581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07054443657398224,
      "rewards/margins": 0.026242827996611595,
      "rewards/rejected": -0.0968017578125,
      "step": 2220
    },
    {
      "epoch": 0.08356598152554758,
      "grad_norm": 1.332199534901523,
      "learning_rate": 1.6940941995505069e-06,
      "log_odds_chosen": 0.4027343690395355,
      "log_odds_ratio": -0.6504882574081421,
      "logits/chosen": -0.83984375,
      "logits/rejected": -0.791015625,
      "logps/chosen": -0.738085925579071,
      "logps/rejected": -1.021484375,
      "loss": 1.0836,
      "nll_loss": 1.046484351158142,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07380370795726776,
      "rewards/margins": 0.02831115759909153,
      "rewards/rejected": -0.10208740085363388,
      "step": 2230
    },
    {
      "epoch": 0.08394071686871148,
      "grad_norm": 1.3800622995038034,
      "learning_rate": 1.6903085094570331e-06,
      "log_odds_chosen": 0.4754882752895355,
      "log_odds_ratio": -0.596484363079071,
      "logits/chosen": -0.8109375238418579,
      "logits/rejected": -0.7601562738418579,
      "logps/chosen": -0.666308581829071,
      "logps/rejected": -0.973437488079071,
      "loss": 1.0451,
      "nll_loss": 0.9701172113418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06662597507238388,
      "rewards/margins": 0.03067169152200222,
      "rewards/rejected": -0.09738769382238388,
      "step": 2240
    },
    {
      "epoch": 0.08431545221187536,
      "grad_norm": 1.533052478112303,
      "learning_rate": 1.6865480854231356e-06,
      "log_odds_chosen": 0.4351806640625,
      "log_odds_ratio": -0.629101574420929,
      "logits/chosen": -0.8609374761581421,
      "logits/rejected": -0.8177734613418579,
      "logps/chosen": -0.7066406011581421,
      "logps/rejected": -1.0060546398162842,
      "loss": 1.0506,
      "nll_loss": 0.9664062261581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07066650688648224,
      "rewards/margins": 0.02992401085793972,
      "rewards/rejected": -0.1005859375,
      "step": 2250
    },
    {
      "epoch": 0.08469018755503925,
      "grad_norm": 1.2779206714400688,
      "learning_rate": 1.682812647646685e-06,
      "log_odds_chosen": 0.3876953125,
      "log_odds_ratio": -0.623730480670929,
      "logits/chosen": -0.876757800579071,
      "logits/rejected": -0.779492199420929,
      "logps/chosen": -0.7474609613418579,
      "logps/rejected": -1.010351538658142,
      "loss": 1.0594,
      "nll_loss": 0.9701172113418579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07470703125,
      "rewards/margins": 0.026291657239198685,
      "rewards/rejected": -0.10102538764476776,
      "step": 2260
    },
    {
      "epoch": 0.08506492289820314,
      "grad_norm": 1.3935306985699092,
      "learning_rate": 1.6791019206444541e-06,
      "log_odds_chosen": 0.43885499238967896,
      "log_odds_ratio": -0.613574206829071,
      "logits/chosen": -0.8519531488418579,
      "logits/rejected": -0.830273449420929,
      "logps/chosen": -0.7398437261581421,
      "logps/rejected": -1.0539062023162842,
      "loss": 1.0434,
      "nll_loss": 0.9833984375,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07406006008386612,
      "rewards/margins": 0.03143615648150444,
      "rewards/rejected": -0.10544433444738388,
      "step": 2270
    },
    {
      "epoch": 0.08543965824136704,
      "grad_norm": 1.283799893414406,
      "learning_rate": 1.675415633166782e-06,
      "log_odds_chosen": 0.3905273377895355,
      "log_odds_ratio": -0.624707043170929,
      "logits/chosen": -0.8648437261581421,
      "logits/rejected": -0.806835949420929,
      "logps/chosen": -0.775585949420929,
      "logps/rejected": -1.0330078601837158,
      "loss": 1.064,
      "nll_loss": 1.0343749523162842,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07755126804113388,
      "rewards/margins": 0.02574005164206028,
      "rewards/rejected": -0.10329589992761612,
      "step": 2280
    },
    {
      "epoch": 0.08581439358453093,
      "grad_norm": 1.305923020966053,
      "learning_rate": 1.6717535181142914e-06,
      "log_odds_chosen": 0.43977051973342896,
      "log_odds_ratio": -0.6084960699081421,
      "logits/chosen": -0.874804675579071,
      "logits/rejected": -0.787109375,
      "logps/chosen": -0.735156238079071,
      "logps/rejected": -1.0373046398162842,
      "loss": 1.0587,
      "nll_loss": 0.9869140386581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07353515923023224,
      "rewards/margins": 0.03020172193646431,
      "rewards/rejected": -0.10374756157398224,
      "step": 2290
    },
    {
      "epoch": 0.08618912892769481,
      "grad_norm": 1.4458973916887714,
      "learning_rate": 1.668115312456598e-06,
      "log_odds_chosen": 0.28533935546875,
      "log_odds_ratio": -0.6800781488418579,
      "logits/chosen": -0.8374999761581421,
      "logits/rejected": -0.779296875,
      "logps/chosen": -0.7650390863418579,
      "logps/rejected": -0.9701172113418579,
      "loss": 1.0709,
      "nll_loss": 1.076562523841858,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07650146633386612,
      "rewards/margins": 0.02050323411822319,
      "rewards/rejected": -0.096923828125,
      "step": 2300
    },
    {
      "epoch": 0.0865638642708587,
      "grad_norm": 1.4019934064380153,
      "learning_rate": 1.6645007571529578e-06,
      "log_odds_chosen": 0.4383789002895355,
      "log_odds_ratio": -0.60205078125,
      "logits/chosen": -0.8363281488418579,
      "logits/rejected": -0.7880859375,
      "logps/chosen": -0.7347656488418579,
      "logps/rejected": -1.0333983898162842,
      "loss": 1.0445,
      "nll_loss": 1.048437476158142,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07344970852136612,
      "rewards/margins": 0.02985839918255806,
      "rewards/rejected": -0.10344238579273224,
      "step": 2310
    },
    {
      "epoch": 0.0869385996140226,
      "grad_norm": 1.3265632757733745,
      "learning_rate": 1.6609095970747992e-06,
      "log_odds_chosen": 0.42762452363967896,
      "log_odds_ratio": -0.5931640863418579,
      "logits/chosen": -0.926953136920929,
      "logits/rejected": -0.8570312261581421,
      "logps/chosen": -0.723828136920929,
      "logps/rejected": -0.995312511920929,
      "loss": 1.074,
      "nll_loss": 0.9839843511581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07243652641773224,
      "rewards/margins": 0.02714843675494194,
      "rewards/rejected": -0.099609375,
      "step": 2320
    },
    {
      "epoch": 0.08731333495718649,
      "grad_norm": 1.3837607461446944,
      "learning_rate": 1.6573415809300833e-06,
      "log_odds_chosen": 0.34315186738967896,
      "log_odds_ratio": -0.659472644329071,
      "logits/chosen": -0.8667968511581421,
      "logits/rejected": -0.7992187738418579,
      "logps/chosen": -0.732421875,
      "logps/rejected": -0.981640636920929,
      "loss": 1.0781,
      "nll_loss": 0.987500011920929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07326660305261612,
      "rewards/margins": 0.024887848645448685,
      "rewards/rejected": -0.09807129204273224,
      "step": 2330
    },
    {
      "epoch": 0.08768807030035038,
      "grad_norm": 1.3944970006448916,
      "learning_rate": 1.6537964611894462e-06,
      "log_odds_chosen": 0.3636108338832855,
      "log_odds_ratio": -0.631640613079071,
      "logits/chosen": -0.8822265863418579,
      "logits/rejected": -0.824999988079071,
      "logps/chosen": -0.733203113079071,
      "logps/rejected": -0.9599609375,
      "loss": 1.0552,
      "nll_loss": 0.9507812261581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07329101860523224,
      "rewards/margins": 0.022798920050263405,
      "rewards/rejected": -0.09600830078125,
      "step": 2340
    },
    {
      "epoch": 0.08806280564351426,
      "grad_norm": 1.3706151832143487,
      "learning_rate": 1.6502739940140692e-06,
      "log_odds_chosen": 0.3412231504917145,
      "log_odds_ratio": -0.6392577886581421,
      "logits/chosen": -0.860156238079071,
      "logits/rejected": -0.8072265386581421,
      "logps/chosen": -0.717480480670929,
      "logps/rejected": -0.943164050579071,
      "loss": 1.084,
      "nll_loss": 1.0050780773162842,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07178954780101776,
      "rewards/margins": 0.02258605882525444,
      "rewards/rejected": -0.09428711235523224,
      "step": 2350
    },
    {
      "epoch": 0.08843754098667816,
      "grad_norm": 1.3302668506940387,
      "learning_rate": 1.6467739391852364e-06,
      "log_odds_chosen": 0.3390136659145355,
      "log_odds_ratio": -0.655468761920929,
      "logits/chosen": -0.8609374761581421,
      "logits/rejected": -0.785351574420929,
      "logps/chosen": -0.752148449420929,
      "logps/rejected": -0.9765625,
      "loss": 1.0572,
      "nll_loss": 1.062890648841858,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07523193210363388,
      "rewards/margins": 0.02238159254193306,
      "rewards/rejected": -0.09760741889476776,
      "step": 2360
    },
    {
      "epoch": 0.08881227632984205,
      "grad_norm": 1.2647752185480046,
      "learning_rate": 1.6432960600355221e-06,
      "log_odds_chosen": 0.49799805879592896,
      "log_odds_ratio": -0.5736328363418579,
      "logits/chosen": -0.843945324420929,
      "logits/rejected": -0.8121093511581421,
      "logps/chosen": -0.7333984375,
      "logps/rejected": -1.0517578125,
      "loss": 1.0795,
      "nll_loss": 1.041406273841858,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07337646186351776,
      "rewards/margins": 0.03177947923541069,
      "rewards/rejected": -0.10520019382238388,
      "step": 2370
    },
    {
      "epoch": 0.08918701167300594,
      "grad_norm": 1.2941495855715557,
      "learning_rate": 1.6398401233815756e-06,
      "log_odds_chosen": 0.31939697265625,
      "log_odds_ratio": -0.6502929925918579,
      "logits/chosen": -0.890820324420929,
      "logits/rejected": -0.832226574420929,
      "logps/chosen": -0.788281261920929,
      "logps/rejected": -0.9927734136581421,
      "loss": 1.0712,
      "nll_loss": 0.9814453125,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07878418266773224,
      "rewards/margins": 0.02040405198931694,
      "rewards/rejected": -0.09915771335363388,
      "step": 2380
    },
    {
      "epoch": 0.08956174701616983,
      "grad_norm": 1.2786034136008522,
      "learning_rate": 1.6364058994584524e-06,
      "log_odds_chosen": 0.3528808653354645,
      "log_odds_ratio": -0.630078136920929,
      "logits/chosen": -0.8597656488418579,
      "logits/rejected": -0.7904297113418579,
      "logps/chosen": -0.705078125,
      "logps/rejected": -0.938281238079071,
      "loss": 1.0558,
      "nll_loss": 0.9765625,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07047118991613388,
      "rewards/margins": 0.02334289625287056,
      "rewards/rejected": -0.09383545070886612,
      "step": 2390
    },
    {
      "epoch": 0.08993648235933371,
      "grad_norm": 1.32345334530713,
      "learning_rate": 1.6329931618554522e-06,
      "log_odds_chosen": 0.515380859375,
      "log_odds_ratio": -0.5972656011581421,
      "logits/chosen": -0.862109363079071,
      "logits/rejected": -0.8154296875,
      "logps/chosen": -0.731249988079071,
      "logps/rejected": -1.063085913658142,
      "loss": 1.0404,
      "nll_loss": 0.971875011920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0731201171875,
      "rewards/margins": 0.03308563306927681,
      "rewards/rejected": -0.10625000298023224,
      "step": 2400
    },
    {
      "epoch": 0.09031121770249761,
      "grad_norm": 1.2900559558738087,
      "learning_rate": 1.6296016874534209e-06,
      "log_odds_chosen": 0.4020141661167145,
      "log_odds_ratio": -0.6034179925918579,
      "logits/chosen": -0.879101574420929,
      "logits/rejected": -0.8412109613418579,
      "logps/chosen": -0.7197265625,
      "logps/rejected": -0.985156238079071,
      "loss": 1.0546,
      "nll_loss": 0.9515625238418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07193603366613388,
      "rewards/margins": 0.02651824988424778,
      "rewards/rejected": -0.09846191108226776,
      "step": 2410
    },
    {
      "epoch": 0.0906859530456615,
      "grad_norm": 1.345015845244426,
      "learning_rate": 1.6262312563634835e-06,
      "log_odds_chosen": 0.2947998046875,
      "log_odds_ratio": -0.6485351324081421,
      "logits/chosen": -0.861523449420929,
      "logits/rejected": -0.8008788824081421,
      "logps/chosen": -0.744335949420929,
      "logps/rejected": -0.9320312738418579,
      "loss": 1.0451,
      "nll_loss": 1.014062523841858,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07447509467601776,
      "rewards/margins": 0.01882782019674778,
      "rewards/rejected": -0.09323730319738388,
      "step": 2420
    },
    {
      "epoch": 0.09106068838882539,
      "grad_norm": 1.3657736122691302,
      "learning_rate": 1.6228816518671587e-06,
      "log_odds_chosen": 0.4228149354457855,
      "log_odds_ratio": -0.6107422113418579,
      "logits/chosen": -0.833203136920929,
      "logits/rejected": -0.755859375,
      "logps/chosen": -0.71533203125,
      "logps/rejected": -0.966992199420929,
      "loss": 1.0512,
      "nll_loss": 0.9515625238418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07149658352136612,
      "rewards/margins": 0.02521057054400444,
      "rewards/rejected": -0.09675292670726776,
      "step": 2430
    },
    {
      "epoch": 0.09143542373198928,
      "grad_norm": 1.306494857657684,
      "learning_rate": 1.619552660357832e-06,
      "log_odds_chosen": 0.40924072265625,
      "log_odds_ratio": -0.6001952886581421,
      "logits/chosen": -0.879101574420929,
      "logits/rejected": -0.800585925579071,
      "logps/chosen": -0.709765613079071,
      "logps/rejected": -0.9515625238418579,
      "loss": 1.0435,
      "nll_loss": 0.986132800579071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0709228515625,
      "rewards/margins": 0.02412109449505806,
      "rewards/rejected": -0.09514160454273224,
      "step": 2440
    },
    {
      "epoch": 0.09181015907515318,
      "grad_norm": 8.978228211021301,
      "learning_rate": 1.616244071283537e-06,
      "log_odds_chosen": 0.23222656548023224,
      "log_odds_ratio": -0.696972668170929,
      "logits/chosen": -0.8384765386581421,
      "logits/rejected": -0.8218749761581421,
      "logps/chosen": -0.802539050579071,
      "logps/rejected": -0.955273449420929,
      "loss": 1.0819,
      "nll_loss": 0.993945300579071,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08018799126148224,
      "rewards/margins": 0.01533355750143528,
      "rewards/rejected": -0.09553222358226776,
      "step": 2450
    },
    {
      "epoch": 0.09218489441831706,
      "grad_norm": 1.4500318941070554,
      "learning_rate": 1.6129556770910235e-06,
      "log_odds_chosen": 0.22219237685203552,
      "log_odds_ratio": -0.6871093511581421,
      "logits/chosen": -0.8199218511581421,
      "logits/rejected": -0.784375011920929,
      "logps/chosen": -0.7412109375,
      "logps/rejected": -0.888867199420929,
      "loss": 1.0583,
      "nll_loss": 0.9566406011581421,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07406006008386612,
      "rewards/margins": 0.014818573370575905,
      "rewards/rejected": -0.08887939155101776,
      "step": 2460
    },
    {
      "epoch": 0.09255962976148095,
      "grad_norm": 1.4247492008660172,
      "learning_rate": 1.6096872731710673e-06,
      "log_odds_chosen": 0.22451171278953552,
      "log_odds_ratio": -0.681445300579071,
      "logits/chosen": -0.932812511920929,
      "logits/rejected": -0.851367175579071,
      "logps/chosen": -0.771484375,
      "logps/rejected": -0.925585925579071,
      "loss": 1.0868,
      "nll_loss": 1.001562476158142,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07712402194738388,
      "rewards/margins": 0.01537246722728014,
      "rewards/rejected": -0.09251709282398224,
      "step": 2470
    },
    {
      "epoch": 0.09293436510464484,
      "grad_norm": 1.2533221311179619,
      "learning_rate": 1.6064386578049978e-06,
      "log_odds_chosen": 0.30583494901657104,
      "log_odds_ratio": -0.661328136920929,
      "logits/chosen": -0.833789050579071,
      "logits/rejected": -0.804882824420929,
      "logps/chosen": -0.7470703125,
      "logps/rejected": -0.945507824420929,
      "loss": 1.0727,
      "nll_loss": 1.0011718273162842,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07474365085363388,
      "rewards/margins": 0.019878387451171875,
      "rewards/rejected": -0.09451904147863388,
      "step": 2480
    },
    {
      "epoch": 0.09330910044780874,
      "grad_norm": 1.2425768143306049,
      "learning_rate": 1.6032096321124046e-06,
      "log_odds_chosen": 0.3602050840854645,
      "log_odds_ratio": -0.651171863079071,
      "logits/chosen": -0.8873046636581421,
      "logits/rejected": -0.8114258050918579,
      "logps/chosen": -0.758007824420929,
      "logps/rejected": -0.9955078363418579,
      "loss": 1.0645,
      "nll_loss": 0.993359386920929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07581786811351776,
      "rewards/margins": 0.023723602294921875,
      "rewards/rejected": -0.09953613579273224,
      "step": 2490
    },
    {
      "epoch": 0.09368383579097263,
      "grad_norm": 1.300802365722513,
      "learning_rate": 1.6e-06,
      "log_odds_chosen": 0.4066406190395355,
      "log_odds_ratio": -0.6283203363418579,
      "logits/chosen": -0.9369140863418579,
      "logits/rejected": -0.872265636920929,
      "logps/chosen": -0.74267578125,
      "logps/rejected": -1.0164062976837158,
      "loss": 1.0574,
      "nll_loss": 0.985156238079071,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07430420070886612,
      "rewards/margins": 0.027341460809111595,
      "rewards/rejected": -0.10158691555261612,
      "step": 2500
    },
    {
      "epoch": 0.09405857113413651,
      "grad_norm": 1.2752469877739374,
      "learning_rate": 1.5968095681115984e-06,
      "log_odds_chosen": 0.41987913846969604,
      "log_odds_ratio": -0.615917980670929,
      "logits/chosen": -0.812695324420929,
      "logits/rejected": -0.770703136920929,
      "logps/chosen": -0.725292980670929,
      "logps/rejected": -0.9957031011581421,
      "loss": 1.0424,
      "nll_loss": 1.0255858898162842,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07252196967601776,
      "rewards/margins": 0.027008820325136185,
      "rewards/rejected": -0.09953613579273224,
      "step": 2510
    },
    {
      "epoch": 0.0944333064773004,
      "grad_norm": 1.4577334215136066,
      "learning_rate": 1.5936381457791914e-06,
      "log_odds_chosen": 0.3767944276332855,
      "log_odds_ratio": -0.6395508050918579,
      "logits/chosen": -0.8617187738418579,
      "logits/rejected": -0.773632824420929,
      "logps/chosen": -0.75,
      "logps/rejected": -1.0078125,
      "loss": 1.064,
      "nll_loss": 0.987109363079071,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07500000298023224,
      "rewards/margins": 0.0257568359375,
      "rewards/rejected": -0.10085449367761612,
      "step": 2520
    },
    {
      "epoch": 0.0948080418204643,
      "grad_norm": 1.4818607448342616,
      "learning_rate": 1.590485544975088e-06,
      "log_odds_chosen": 0.3328491151332855,
      "log_odds_ratio": -0.638671875,
      "logits/chosen": -0.833984375,
      "logits/rejected": -0.810742199420929,
      "logps/chosen": -0.7281249761581421,
      "logps/rejected": -0.9400390386581421,
      "loss": 1.056,
      "nll_loss": 0.9576171636581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07282714545726776,
      "rewards/margins": 0.021150970831513405,
      "rewards/rejected": -0.093994140625,
      "step": 2530
    },
    {
      "epoch": 0.09518277716362819,
      "grad_norm": 1.4773989512231651,
      "learning_rate": 1.5873515802650901e-06,
      "log_odds_chosen": 0.3712402284145355,
      "log_odds_ratio": -0.628222644329071,
      "logits/chosen": -0.8658202886581421,
      "logits/rejected": -0.8109375238418579,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -0.94140625,
      "loss": 1.0592,
      "nll_loss": 0.990234375,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07066650688648224,
      "rewards/margins": 0.02354278601706028,
      "rewards/rejected": -0.09414062649011612,
      "step": 2540
    },
    {
      "epoch": 0.09555751250679208,
      "grad_norm": 1.3641131299993432,
      "learning_rate": 1.584236068762679e-06,
      "log_odds_chosen": 0.4190307557582855,
      "log_odds_ratio": -0.610644519329071,
      "logits/chosen": -0.9007812738418579,
      "logits/rejected": -0.808789074420929,
      "logps/chosen": -0.6958984136581421,
      "logps/rejected": -0.9935547113418579,
      "loss": 1.0381,
      "nll_loss": 0.9664062261581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06955566257238388,
      "rewards/margins": 0.02980194054543972,
      "rewards/rejected": -0.099365234375,
      "step": 2550
    },
    {
      "epoch": 0.09593224784995596,
      "grad_norm": 1.4861846923423327,
      "learning_rate": 1.5811388300841894e-06,
      "log_odds_chosen": 0.504150390625,
      "log_odds_ratio": -0.59228515625,
      "logits/chosen": -0.8681640625,
      "logits/rejected": -0.8374999761581421,
      "logps/chosen": -0.721484363079071,
      "logps/rejected": -1.0548827648162842,
      "loss": 1.0712,
      "nll_loss": 1.0517578125,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07215575873851776,
      "rewards/margins": 0.03345947340130806,
      "rewards/rejected": -0.10563965141773224,
      "step": 2560
    },
    {
      "epoch": 0.09630698319311985,
      "grad_norm": 1.5204771121607,
      "learning_rate": 1.5780596863049431e-06,
      "log_odds_chosen": 0.3312011659145355,
      "log_odds_ratio": -0.6278320550918579,
      "logits/chosen": -0.863085925579071,
      "logits/rejected": -0.8052734136581421,
      "logps/chosen": -0.72021484375,
      "logps/rejected": -0.92578125,
      "loss": 1.0505,
      "nll_loss": 0.984375,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07204589992761612,
      "rewards/margins": 0.020477294921875,
      "rewards/rejected": -0.092529296875,
      "step": 2570
    },
    {
      "epoch": 0.09668171853628375,
      "grad_norm": 1.3695902319264017,
      "learning_rate": 1.5749984619163156e-06,
      "log_odds_chosen": 0.47236329317092896,
      "log_odds_ratio": -0.5907226800918579,
      "logits/chosen": -0.9115234613418579,
      "logits/rejected": -0.842578113079071,
      "logps/chosen": -0.749804675579071,
      "logps/rejected": -1.052148461341858,
      "loss": 1.0685,
      "nll_loss": 0.988476574420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.074951171875,
      "rewards/margins": 0.03030700609087944,
      "rewards/rejected": -0.10533447563648224,
      "step": 2580
    },
    {
      "epoch": 0.09705645387944764,
      "grad_norm": 1.2677717867564278,
      "learning_rate": 1.5719549837837187e-06,
      "log_odds_chosen": 0.44776612520217896,
      "log_odds_ratio": -0.5986328125,
      "logits/chosen": -0.8548828363418579,
      "logits/rejected": -0.8158203363418579,
      "logps/chosen": -0.760937511920929,
      "logps/rejected": -1.0548827648162842,
      "loss": 1.0584,
      "nll_loss": 0.970507800579071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07606200873851776,
      "rewards/margins": 0.029360199347138405,
      "rewards/rejected": -0.10546875,
      "step": 2590
    },
    {
      "epoch": 0.09743118922261153,
      "grad_norm": 1.339423166267085,
      "learning_rate": 1.5689290811054722e-06,
      "log_odds_chosen": 0.30394285917282104,
      "log_odds_ratio": -0.6548827886581421,
      "logits/chosen": -0.8203125,
      "logits/rejected": -0.800000011920929,
      "logps/chosen": -0.73388671875,
      "logps/rejected": -0.9404296875,
      "loss": 1.035,
      "nll_loss": 0.9791015386581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07337646186351776,
      "rewards/margins": 0.02067108079791069,
      "rewards/rejected": -0.09404297173023224,
      "step": 2600
    },
    {
      "epoch": 0.09780592456577542,
      "grad_norm": 1.3183521395492996,
      "learning_rate": 1.5659205853725426e-06,
      "log_odds_chosen": 0.3476806581020355,
      "log_odds_ratio": -0.638476550579071,
      "logits/chosen": -0.8731445074081421,
      "logits/rejected": -0.7939453125,
      "logps/chosen": -0.73046875,
      "logps/rejected": -0.952929675579071,
      "loss": 1.0619,
      "nll_loss": 1.0363280773162842,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07308349758386612,
      "rewards/margins": 0.02221984788775444,
      "rewards/rejected": -0.09531249850988388,
      "step": 2610
    },
    {
      "epoch": 0.09818065990893932,
      "grad_norm": 1.383215811474061,
      "learning_rate": 1.562929330329127e-06,
      "log_odds_chosen": 0.301513671875,
      "log_odds_ratio": -0.690722644329071,
      "logits/chosen": -0.796093761920929,
      "logits/rejected": -0.7734375,
      "logps/chosen": -0.766894519329071,
      "logps/rejected": -0.950390636920929,
      "loss": 1.0779,
      "nll_loss": 0.9681640863418579,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07668457180261612,
      "rewards/margins": 0.01835021935403347,
      "rewards/rejected": -0.09503173828125,
      "step": 2620
    },
    {
      "epoch": 0.0985553952521032,
      "grad_norm": 1.310814017434704,
      "learning_rate": 1.5599551519340636e-06,
      "log_odds_chosen": 0.4796142578125,
      "log_odds_ratio": -0.598437488079071,
      "logits/chosen": -0.8246093988418579,
      "logits/rejected": -0.791210949420929,
      "logps/chosen": -0.7308593988418579,
      "logps/rejected": -1.045507788658142,
      "loss": 1.0484,
      "nll_loss": 1.007226586341858,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07294921576976776,
      "rewards/margins": 0.03158264234662056,
      "rewards/rejected": -0.10451660305261612,
      "step": 2630
    },
    {
      "epoch": 0.09893013059526709,
      "grad_norm": 1.246615689682294,
      "learning_rate": 1.556997888323046e-06,
      "log_odds_chosen": 0.404296875,
      "log_odds_ratio": -0.6195312738418579,
      "logits/chosen": -0.867968738079071,
      "logits/rejected": -0.8072265386581421,
      "logps/chosen": -0.7203124761581421,
      "logps/rejected": -1.0050780773162842,
      "loss": 1.0575,
      "nll_loss": 0.961132824420929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07203368842601776,
      "rewards/margins": 0.02843475341796875,
      "rewards/rejected": -0.10050048679113388,
      "step": 2640
    },
    {
      "epoch": 0.09930486593843098,
      "grad_norm": 1.2479317132273169,
      "learning_rate": 1.5540573797716226e-06,
      "log_odds_chosen": 0.46776121854782104,
      "log_odds_ratio": -0.5865234136581421,
      "logits/chosen": -0.8705078363418579,
      "logits/rejected": -0.7847656011581421,
      "logps/chosen": -0.705273449420929,
      "logps/rejected": -1.005859375,
      "loss": 1.0539,
      "nll_loss": 0.958203136920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07050780951976776,
      "rewards/margins": 0.03001098707318306,
      "rewards/rejected": -0.10061035305261612,
      "step": 2650
    },
    {
      "epoch": 0.09967960128159488,
      "grad_norm": 1.401825557295271,
      "learning_rate": 1.5511334686589623e-06,
      "log_odds_chosen": 0.3066162168979645,
      "log_odds_ratio": -0.6626952886581421,
      "logits/chosen": -0.8431640863418579,
      "logits/rejected": -0.7822265625,
      "logps/chosen": -0.7884765863418579,
      "logps/rejected": -1.0138671398162842,
      "loss": 1.0507,
      "nll_loss": 1.063085913658142,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07889404147863388,
      "rewards/margins": 0.02247924730181694,
      "rewards/rejected": -0.10142822563648224,
      "step": 2660
    },
    {
      "epoch": 0.10005433662475877,
      "grad_norm": 1.3155175287884469,
      "learning_rate": 1.548225999432367e-06,
      "log_odds_chosen": 0.43437498807907104,
      "log_odds_ratio": -0.6034179925918579,
      "logits/chosen": -0.898632824420929,
      "logits/rejected": -0.8412109613418579,
      "logps/chosen": -0.720410168170929,
      "logps/rejected": -1.025390625,
      "loss": 1.0363,
      "nll_loss": 0.983593761920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07199706882238388,
      "rewards/margins": 0.03048858605325222,
      "rewards/rejected": -0.10251464694738388,
      "step": 2670
    },
    {
      "epoch": 0.10042907196792265,
      "grad_norm": 1.3911859776611055,
      "learning_rate": 1.5453348185725114e-06,
      "log_odds_chosen": 0.3401122987270355,
      "log_odds_ratio": -0.6517578363418579,
      "logits/chosen": -0.8792968988418579,
      "logits/rejected": -0.8082031011581421,
      "logps/chosen": -0.7330077886581421,
      "logps/rejected": -0.9652343988418579,
      "loss": 1.0573,
      "nll_loss": 0.976367175579071,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07327880710363388,
      "rewards/margins": 0.02324523963034153,
      "rewards/rejected": -0.09647216647863388,
      "step": 2680
    },
    {
      "epoch": 0.10080380731108654,
      "grad_norm": 1.3711159956669248,
      "learning_rate": 1.542459774559398e-06,
      "log_odds_chosen": 0.45512694120407104,
      "log_odds_ratio": -0.603710949420929,
      "logits/chosen": -0.861523449420929,
      "logits/rejected": -0.797070324420929,
      "logps/chosen": -0.7679687738418579,
      "logps/rejected": -1.052343726158142,
      "loss": 1.0653,
      "nll_loss": 1.060937523841858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07670898735523224,
      "rewards/margins": 0.02850036695599556,
      "rewards/rejected": -0.10512695461511612,
      "step": 2690
    },
    {
      "epoch": 0.10117854265425044,
      "grad_norm": 6.228333732581943,
      "learning_rate": 1.539600717839002e-06,
      "log_odds_chosen": 0.29570311307907104,
      "log_odds_ratio": -0.652050793170929,
      "logits/chosen": -0.9115234613418579,
      "logits/rejected": -0.8548828363418579,
      "logps/chosen": -0.732421875,
      "logps/rejected": -0.925000011920929,
      "loss": 1.0349,
      "nll_loss": 0.944531261920929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07321777194738388,
      "rewards/margins": 0.01933593675494194,
      "rewards/rejected": -0.09259033203125,
      "step": 2700
    },
    {
      "epoch": 0.10155327799741433,
      "grad_norm": 1.5563940617705572,
      "learning_rate": 1.536757500790597e-06,
      "log_odds_chosen": 0.3065185546875,
      "log_odds_ratio": -0.6405273675918579,
      "logits/chosen": -0.8853515386581421,
      "logits/rejected": -0.8599609136581421,
      "logps/chosen": -0.712109386920929,
      "logps/rejected": -0.9166015386581421,
      "loss": 1.056,
      "nll_loss": 1.005859375,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0712890625,
      "rewards/margins": 0.02043914794921875,
      "rewards/rejected": -0.09165038913488388,
      "step": 2710
    },
    {
      "epoch": 0.10192801334057822,
      "grad_norm": 1.5334401460276372,
      "learning_rate": 1.5339299776947407e-06,
      "log_odds_chosen": 0.39320069551467896,
      "log_odds_ratio": -0.6429687738418579,
      "logits/chosen": -0.863476574420929,
      "logits/rejected": -0.7925781011581421,
      "logps/chosen": -0.7279297113418579,
      "logps/rejected": -1.012109398841858,
      "loss": 1.0554,
      "nll_loss": 0.993945300579071,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07277832180261612,
      "rewards/margins": 0.028401946648955345,
      "rewards/rejected": -0.10117187350988388,
      "step": 2720
    },
    {
      "epoch": 0.1023027486837421,
      "grad_norm": 1.3911621200811586,
      "learning_rate": 1.5311180047019054e-06,
      "log_odds_chosen": 0.479736328125,
      "log_odds_ratio": -0.6000000238418579,
      "logits/chosen": -0.9261718988418579,
      "logits/rejected": -0.8441406488418579,
      "logps/chosen": -0.7691406011581421,
      "logps/rejected": -1.1046874523162842,
      "loss": 1.0593,
      "nll_loss": 0.9892578125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.076904296875,
      "rewards/margins": 0.0335540771484375,
      "rewards/rejected": -0.11052246391773224,
      "step": 2730
    },
    {
      "epoch": 0.10267748402690599,
      "grad_norm": 1.4155252382917276,
      "learning_rate": 1.5283214398017402e-06,
      "log_odds_chosen": 0.615039050579071,
      "log_odds_ratio": -0.574414074420929,
      "logits/chosen": -0.9125000238418579,
      "logits/rejected": -0.8447265625,
      "logps/chosen": -0.775390625,
      "logps/rejected": -1.174414038658142,
      "loss": 1.045,
      "nll_loss": 0.969531238079071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07750244438648224,
      "rewards/margins": 0.03988952562212944,
      "rewards/rejected": -0.11748047173023224,
      "step": 2740
    },
    {
      "epoch": 0.10305221937006989,
      "grad_norm": 1.4265403755854626,
      "learning_rate": 1.5255401427929477e-06,
      "log_odds_chosen": 0.33305662870407104,
      "log_odds_ratio": -0.648144543170929,
      "logits/chosen": -0.851757824420929,
      "logits/rejected": -0.8212890625,
      "logps/chosen": -0.772656261920929,
      "logps/rejected": -0.996874988079071,
      "loss": 1.0491,
      "nll_loss": 1.0154297351837158,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07724609225988388,
      "rewards/margins": 0.02235870435833931,
      "rewards/rejected": -0.09958495944738388,
      "step": 2750
    },
    {
      "epoch": 0.10342695471323378,
      "grad_norm": 1.4928944182657975,
      "learning_rate": 1.5227739752537617e-06,
      "log_odds_chosen": 0.33869630098342896,
      "log_odds_ratio": -0.6509765386581421,
      "logits/chosen": -0.8470703363418579,
      "logits/rejected": -0.798046886920929,
      "logps/chosen": -0.787890613079071,
      "logps/rejected": -0.999804675579071,
      "loss": 1.0525,
      "nll_loss": 1.018164038658142,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07874755561351776,
      "rewards/margins": 0.02129821851849556,
      "rewards/rejected": -0.10000000149011612,
      "step": 2760
    },
    {
      "epoch": 0.10380169005639767,
      "grad_norm": 1.3996854541593136,
      "learning_rate": 1.5200228005130127e-06,
      "log_odds_chosen": 0.4115234315395355,
      "log_odds_ratio": -0.619140625,
      "logits/chosen": -0.892773449420929,
      "logits/rejected": -0.8330078125,
      "logps/chosen": -0.7391601800918579,
      "logps/rejected": -1.0078125,
      "loss": 1.0634,
      "nll_loss": 1.006250023841858,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07392577826976776,
      "rewards/margins": 0.02692565880715847,
      "rewards/rejected": -0.10090331733226776,
      "step": 2770
    },
    {
      "epoch": 0.10417642539956155,
      "grad_norm": 1.3103033505112327,
      "learning_rate": 1.5172864836217631e-06,
      "log_odds_chosen": 0.4068847596645355,
      "log_odds_ratio": -0.61767578125,
      "logits/chosen": -0.852343738079071,
      "logits/rejected": -0.787890613079071,
      "logps/chosen": -0.682910144329071,
      "logps/rejected": -0.949999988079071,
      "loss": 1.0574,
      "nll_loss": 0.970507800579071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06822510063648224,
      "rewards/margins": 0.026782989501953125,
      "rewards/rejected": -0.09500732272863388,
      "step": 2780
    },
    {
      "epoch": 0.10455116074272545,
      "grad_norm": 1.3262710124555905,
      "learning_rate": 1.514564891325506e-06,
      "log_odds_chosen": 0.3238769471645355,
      "log_odds_ratio": -0.645800769329071,
      "logits/chosen": -0.9195312261581421,
      "logits/rejected": -0.835742175579071,
      "logps/chosen": -0.7300781011581421,
      "logps/rejected": -0.9537109136581421,
      "loss": 1.0344,
      "nll_loss": 0.9408203363418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07304687798023224,
      "rewards/margins": 0.02229003980755806,
      "rewards/rejected": -0.09536132961511612,
      "step": 2790
    },
    {
      "epoch": 0.10492589608588934,
      "grad_norm": 1.4282205070306724,
      "learning_rate": 1.5118578920369086e-06,
      "log_odds_chosen": 0.6168457269668579,
      "log_odds_ratio": -0.554882824420929,
      "logits/chosen": -0.899609386920929,
      "logits/rejected": -0.817187488079071,
      "logps/chosen": -0.7113281488418579,
      "logps/rejected": -1.113867163658142,
      "loss": 1.0743,
      "nll_loss": 0.995898425579071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07111816108226776,
      "rewards/margins": 0.04029083251953125,
      "rewards/rejected": -0.11142577975988388,
      "step": 2800
    },
    {
      "epoch": 0.10530063142905323,
      "grad_norm": 1.402718365091222,
      "learning_rate": 1.5091653558090898e-06,
      "log_odds_chosen": 0.3576904237270355,
      "log_odds_ratio": -0.674511730670929,
      "logits/chosen": -0.874804675579071,
      "logits/rejected": -0.8189452886581421,
      "logps/chosen": -0.735156238079071,
      "logps/rejected": -0.984179675579071,
      "loss": 1.0763,
      "nll_loss": 1.0607421398162842,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07351074367761612,
      "rewards/margins": 0.02490386925637722,
      "rewards/rejected": -0.09843750298023224,
      "step": 2810
    },
    {
      "epoch": 0.10567536677221712,
      "grad_norm": 1.355687348618025,
      "learning_rate": 1.506487154309419e-06,
      "log_odds_chosen": 0.4351440370082855,
      "log_odds_ratio": -0.60009765625,
      "logits/chosen": -0.8343750238418579,
      "logits/rejected": -0.8052734136581421,
      "logps/chosen": -0.7754882574081421,
      "logps/rejected": -1.050195336341858,
      "loss": 1.0359,
      "nll_loss": 1.005859375,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07757568359375,
      "rewards/margins": 0.02747039869427681,
      "rewards/rejected": -0.10502929985523224,
      "step": 2820
    },
    {
      "epoch": 0.10605010211538102,
      "grad_norm": 1.3104937431701513,
      "learning_rate": 1.5038231607938247e-06,
      "log_odds_chosen": 0.32050782442092896,
      "log_odds_ratio": -0.6502929925918579,
      "logits/chosen": -0.8890625238418579,
      "logits/rejected": -0.792187511920929,
      "logps/chosen": -0.7259765863418579,
      "logps/rejected": -0.9496093988418579,
      "loss": 1.0509,
      "nll_loss": 0.978515625,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07264403998851776,
      "rewards/margins": 0.02229003980755806,
      "rewards/rejected": -0.09493408352136612,
      "step": 2830
    },
    {
      "epoch": 0.1064248374585449,
      "grad_norm": 1.338946737638142,
      "learning_rate": 1.501173250081603e-06,
      "log_odds_chosen": 0.3890624940395355,
      "log_odds_ratio": -0.649218738079071,
      "logits/chosen": -0.9048827886581421,
      "logits/rejected": -0.877148449420929,
      "logps/chosen": -0.7369140386581421,
      "logps/rejected": -0.9742187261581421,
      "loss": 1.0427,
      "nll_loss": 0.931640625,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07364501804113388,
      "rewards/margins": 0.02383270300924778,
      "rewards/rejected": -0.09748534858226776,
      "step": 2840
    },
    {
      "epoch": 0.10679957280170879,
      "grad_norm": 10.82395886459993,
      "learning_rate": 1.4985372985307103e-06,
      "log_odds_chosen": 0.3562988340854645,
      "log_odds_ratio": -0.658398449420929,
      "logits/chosen": -0.877148449420929,
      "logits/rejected": -0.848828136920929,
      "logps/chosen": -0.765820324420929,
      "logps/rejected": -1.0437500476837158,
      "loss": 1.0441,
      "nll_loss": 1.0244140625,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07664795219898224,
      "rewards/margins": 0.027779389172792435,
      "rewards/rejected": -0.1043701171875,
      "step": 2850
    },
    {
      "epoch": 0.10717430814487268,
      "grad_norm": 1.418888142798182,
      "learning_rate": 1.4959151840135313e-06,
      "log_odds_chosen": 0.37657469511032104,
      "log_odds_ratio": -0.620312511920929,
      "logits/chosen": -0.8603515625,
      "logits/rejected": -0.796093761920929,
      "logps/chosen": -0.7159179449081421,
      "logps/rejected": -0.952929675579071,
      "loss": 1.0746,
      "nll_loss": 1.013085961341858,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07156982272863388,
      "rewards/margins": 0.02373199537396431,
      "rewards/rejected": -0.09536132961511612,
      "step": 2860
    },
    {
      "epoch": 0.10754904348803658,
      "grad_norm": 1.304331940243903,
      "learning_rate": 1.4933067858931148e-06,
      "log_odds_chosen": 0.3372558653354645,
      "log_odds_ratio": -0.6405273675918579,
      "logits/chosen": -0.8558593988418579,
      "logits/rejected": -0.814257800579071,
      "logps/chosen": -0.755859375,
      "logps/rejected": -0.998046875,
      "loss": 1.0528,
      "nll_loss": 1.0203125476837158,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07563476264476776,
      "rewards/margins": 0.0241851806640625,
      "rewards/rejected": -0.09974364936351776,
      "step": 2870
    },
    {
      "epoch": 0.10792377883120047,
      "grad_norm": 1.3167753032780405,
      "learning_rate": 1.4907119849998597e-06,
      "log_odds_chosen": 0.31871336698532104,
      "log_odds_ratio": -0.6541992425918579,
      "logits/chosen": -0.8931640386581421,
      "logits/rejected": -0.8167968988418579,
      "logps/chosen": -0.7699218988418579,
      "logps/rejected": -0.9986327886581421,
      "loss": 1.0686,
      "nll_loss": 0.9664062261581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07700195163488388,
      "rewards/margins": 0.02287902869284153,
      "rewards/rejected": -0.09990234673023224,
      "step": 2880
    },
    {
      "epoch": 0.10829851417436435,
      "grad_norm": 1.4345691140569696,
      "learning_rate": 1.488130663608649e-06,
      "log_odds_chosen": 0.43115234375,
      "log_odds_ratio": -0.6201171875,
      "logits/chosen": -0.873046875,
      "logits/rejected": -0.7847656011581421,
      "logps/chosen": -0.7535156011581421,
      "logps/rejected": -1.040429711341858,
      "loss": 1.0535,
      "nll_loss": 0.960156261920929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07539062201976776,
      "rewards/margins": 0.02859191969037056,
      "rewards/rejected": -0.10393066704273224,
      "step": 2890
    },
    {
      "epoch": 0.10867324951752824,
      "grad_norm": 1.3432686736592119,
      "learning_rate": 1.4855627054164149e-06,
      "log_odds_chosen": 0.44331055879592896,
      "log_odds_ratio": -0.60986328125,
      "logits/chosen": -0.919140636920929,
      "logits/rejected": -0.830859363079071,
      "logps/chosen": -0.751171886920929,
      "logps/rejected": -1.063867211341858,
      "loss": 1.0376,
      "nll_loss": 0.9810546636581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07510986179113388,
      "rewards/margins": 0.031269073486328125,
      "rewards/rejected": -0.10635986179113388,
      "step": 2900
    },
    {
      "epoch": 0.10904798486069213,
      "grad_norm": 1.4077899272667154,
      "learning_rate": 1.4830079955201294e-06,
      "log_odds_chosen": 0.3739868104457855,
      "log_odds_ratio": -0.6141601800918579,
      "logits/chosen": -0.8353515863418579,
      "logits/rejected": -0.803515613079071,
      "logps/chosen": -0.7203124761581421,
      "logps/rejected": -0.948046863079071,
      "loss": 1.0479,
      "nll_loss": 1.0271484851837158,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07209472358226776,
      "rewards/margins": 0.022735595703125,
      "rewards/rejected": -0.09487304836511612,
      "step": 2910
    },
    {
      "epoch": 0.10942272020385603,
      "grad_norm": 1.3920894743612655,
      "learning_rate": 1.4804664203952103e-06,
      "log_odds_chosen": 0.4704223573207855,
      "log_odds_ratio": -0.580371081829071,
      "logits/chosen": -0.892382800579071,
      "logits/rejected": -0.7935546636581421,
      "logps/chosen": -0.745312511920929,
      "logps/rejected": -1.066992163658142,
      "loss": 1.0506,
      "nll_loss": 0.96875,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.074462890625,
      "rewards/margins": 0.032234955579042435,
      "rewards/rejected": -0.1068115234375,
      "step": 2920
    },
    {
      "epoch": 0.10979745554701992,
      "grad_norm": 1.306195481815449,
      "learning_rate": 1.4779378678743327e-06,
      "log_odds_chosen": 0.527539074420929,
      "log_odds_ratio": -0.59375,
      "logits/chosen": -0.8623046875,
      "logits/rejected": -0.8208984136581421,
      "logps/chosen": -0.716992199420929,
      "logps/rejected": -1.073828101158142,
      "loss": 1.0205,
      "nll_loss": 1.0232422351837158,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07172851264476776,
      "rewards/margins": 0.03559570387005806,
      "rewards/rejected": -0.10737304389476776,
      "step": 2930
    },
    {
      "epoch": 0.1101721908901838,
      "grad_norm": 1.3245260499564022,
      "learning_rate": 1.4754222271266348e-06,
      "log_odds_chosen": 0.2508178651332855,
      "log_odds_ratio": -0.6880859136581421,
      "logits/chosen": -0.8623046875,
      "logits/rejected": -0.7669922113418579,
      "logps/chosen": -0.7835937738418579,
      "logps/rejected": -0.983203113079071,
      "loss": 1.0677,
      "nll_loss": 1.003320336341858,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.07839355617761612,
      "rewards/margins": 0.01993865892291069,
      "rewards/rejected": -0.09831543266773224,
      "step": 2940
    },
    {
      "epoch": 0.11054692623334769,
      "grad_norm": 1.3637189278385484,
      "learning_rate": 1.4729193886373175e-06,
      "log_odds_chosen": 0.46021729707717896,
      "log_odds_ratio": -0.612500011920929,
      "logits/chosen": -0.8515625,
      "logits/rejected": -0.7958984375,
      "logps/chosen": -0.7201172113418579,
      "logps/rejected": -1.0283203125,
      "loss": 1.045,
      "nll_loss": 0.996289074420929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07200928032398224,
      "rewards/margins": 0.030829619616270065,
      "rewards/rejected": -0.10284423828125,
      "step": 2950
    },
    {
      "epoch": 0.1109216615765116,
      "grad_norm": 1.385492102544818,
      "learning_rate": 1.4704292441876156e-06,
      "log_odds_chosen": 0.26982420682907104,
      "log_odds_ratio": -0.6551758050918579,
      "logits/chosen": -0.859375,
      "logits/rejected": -0.8109375238418579,
      "logps/chosen": -0.7281249761581421,
      "logps/rejected": -0.8941406011581421,
      "loss": 1.0577,
      "nll_loss": 1.006445288658142,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07281494140625,
      "rewards/margins": 0.01665344275534153,
      "rewards/rejected": -0.08939208835363388,
      "step": 2960
    },
    {
      "epoch": 0.11129639691967548,
      "grad_norm": 1.4128608361572472,
      "learning_rate": 1.4679516868351474e-06,
      "log_odds_chosen": 0.5520995855331421,
      "log_odds_ratio": -0.574414074420929,
      "logits/chosen": -0.8369140625,
      "logits/rejected": -0.7470703125,
      "logps/chosen": -0.7115234136581421,
      "logps/rejected": -1.0900390148162842,
      "loss": 1.0674,
      "nll_loss": 1.005468726158142,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07115478813648224,
      "rewards/margins": 0.03804473951458931,
      "rewards/rejected": -0.10917969048023224,
      "step": 2970
    },
    {
      "epoch": 0.11167113226283937,
      "grad_norm": 1.4286241063294378,
      "learning_rate": 1.4654866108946234e-06,
      "log_odds_chosen": 0.38372802734375,
      "log_odds_ratio": -0.650585949420929,
      "logits/chosen": -0.8271484375,
      "logits/rejected": -0.7544921636581421,
      "logps/chosen": -0.7748047113418579,
      "logps/rejected": -1.046875,
      "loss": 1.0403,
      "nll_loss": 1.0349609851837158,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.07744140923023224,
      "rewards/margins": 0.02729492262005806,
      "rewards/rejected": -0.10478515923023224,
      "step": 2980
    },
    {
      "epoch": 0.11204586760600325,
      "grad_norm": 1.3578702093035853,
      "learning_rate": 1.4630339119189101e-06,
      "log_odds_chosen": 0.3600830137729645,
      "log_odds_ratio": -0.645214855670929,
      "logits/chosen": -0.8343750238418579,
      "logits/rejected": -0.77734375,
      "logps/chosen": -0.7533203363418579,
      "logps/rejected": -1.009179711341858,
      "loss": 1.0553,
      "nll_loss": 1.038476586341858,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07541503757238388,
      "rewards/margins": 0.025475312024354935,
      "rewards/rejected": -0.10078124701976776,
      "step": 2990
    },
    {
      "epoch": 0.11242060294916716,
      "grad_norm": 1.4973476719854797,
      "learning_rate": 1.4605934866804429e-06,
      "log_odds_chosen": 0.28306883573532104,
      "log_odds_ratio": -0.6590820550918579,
      "logits/chosen": -0.8814452886581421,
      "logits/rejected": -0.8326171636581421,
      "logps/chosen": -0.7222656011581421,
      "logps/rejected": -0.91015625,
      "loss": 1.0378,
      "nll_loss": 0.9898437261581421,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07220458984375,
      "rewards/margins": 0.01876220665872097,
      "rewards/rejected": -0.09097900241613388,
      "step": 3000
    },
    {
      "epoch": 0.11279533829233104,
      "grad_norm": 1.351935436491923,
      "learning_rate": 1.4581652331529784e-06,
      "log_odds_chosen": 0.45228272676467896,
      "log_odds_ratio": -0.6122070550918579,
      "logits/chosen": -0.88671875,
      "logits/rejected": -0.787109375,
      "logps/chosen": -0.722460925579071,
      "logps/rejected": -1.0275390148162842,
      "loss": 1.0319,
      "nll_loss": 0.960742175579071,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07225342094898224,
      "rewards/margins": 0.03048553504049778,
      "rewards/rejected": -0.10275878757238388,
      "step": 3010
    },
    {
      "epoch": 0.11317007363549493,
      "grad_norm": 1.3548448838930804,
      "learning_rate": 1.4557490504936778e-06,
      "log_odds_chosen": 0.32916259765625,
      "log_odds_ratio": -0.6507812738418579,
      "logits/chosen": -0.836132824420929,
      "logits/rejected": -0.777148425579071,
      "logps/chosen": -0.73828125,
      "logps/rejected": -0.96875,
      "loss": 1.0352,
      "nll_loss": 0.9808593988418579,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.0738525390625,
      "rewards/margins": 0.02313232421875,
      "rewards/rejected": -0.09697265923023224,
      "step": 3020
    },
    {
      "epoch": 0.11354480897865882,
      "grad_norm": 1.3897826454034978,
      "learning_rate": 1.453344839025519e-06,
      "log_odds_chosen": 0.3767333924770355,
      "log_odds_ratio": -0.64013671875,
      "logits/chosen": -0.892773449420929,
      "logits/rejected": -0.830859363079071,
      "logps/chosen": -0.7457031011581421,
      "logps/rejected": -1.0109374523162842,
      "loss": 1.0441,
      "nll_loss": 0.962109386920929,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07451172173023224,
      "rewards/margins": 0.02654113806784153,
      "rewards/rejected": -0.10104980319738388,
      "step": 3030
    },
    {
      "epoch": 0.11391954432182272,
      "grad_norm": 1.3537484755226268,
      "learning_rate": 1.4509525002200234e-06,
      "log_odds_chosen": 0.29631346464157104,
      "log_odds_ratio": -0.663378894329071,
      "logits/chosen": -0.823046863079071,
      "logits/rejected": -0.7749999761581421,
      "logps/chosen": -0.7626953125,
      "logps/rejected": -0.96484375,
      "loss": 1.0502,
      "nll_loss": 1.071679711341858,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07628174126148224,
      "rewards/margins": 0.0201568603515625,
      "rewards/rejected": -0.09645996242761612,
      "step": 3040
    },
    {
      "epoch": 0.1142942796649866,
      "grad_norm": 1.4529715760412636,
      "learning_rate": 1.4485719366802965e-06,
      "log_odds_chosen": 0.34100341796875,
      "log_odds_ratio": -0.6158202886581421,
      "logits/chosen": -0.8990234136581421,
      "logits/rejected": -0.8486328125,
      "logps/chosen": -0.7132812738418579,
      "logps/rejected": -0.9330078363418579,
      "loss": 1.0383,
      "nll_loss": 0.925000011920929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07132568210363388,
      "rewards/margins": 0.02197723463177681,
      "rewards/rejected": -0.09331054985523224,
      "step": 3050
    },
    {
      "epoch": 0.1146690150081505,
      "grad_norm": 1.355040101527696,
      "learning_rate": 1.4462030521243742e-06,
      "log_odds_chosen": 0.43339842557907104,
      "log_odds_ratio": -0.6005859375,
      "logits/chosen": -0.8675781488418579,
      "logits/rejected": -0.770703136920929,
      "logps/chosen": -0.755859375,
      "logps/rejected": -1.017187476158142,
      "loss": 1.0573,
      "nll_loss": 1.050195336341858,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07562255859375,
      "rewards/margins": 0.02608337439596653,
      "rewards/rejected": -0.10175780951976776,
      "step": 3060
    },
    {
      "epoch": 0.11504375035131438,
      "grad_norm": 1.410623572165947,
      "learning_rate": 1.443845751368867e-06,
      "log_odds_chosen": 0.44951170682907104,
      "log_odds_ratio": -0.618847668170929,
      "logits/chosen": -0.8890625238418579,
      "logits/rejected": -0.8257812261581421,
      "logps/chosen": -0.7318359613418579,
      "logps/rejected": -1.0613281726837158,
      "loss": 1.0539,
      "nll_loss": 0.914257824420929,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07318115234375,
      "rewards/margins": 0.03299713134765625,
      "rewards/rejected": -0.10615234076976776,
      "step": 3070
    },
    {
      "epoch": 0.11541848569447828,
      "grad_norm": 1.5298201983969217,
      "learning_rate": 1.4414999403128943e-06,
      "log_odds_chosen": 0.5574706792831421,
      "log_odds_ratio": -0.568066418170929,
      "logits/chosen": -0.8794921636581421,
      "logits/rejected": -0.8089843988418579,
      "logps/chosen": -0.733593761920929,
      "logps/rejected": -1.1076171398162842,
      "loss": 1.0391,
      "nll_loss": 0.988085925579071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07331542670726776,
      "rewards/margins": 0.03743591159582138,
      "rewards/rejected": -0.11076660454273224,
      "step": 3080
    },
    {
      "epoch": 0.11579322103764217,
      "grad_norm": 1.382810363524273,
      "learning_rate": 1.439165525922309e-06,
      "log_odds_chosen": 0.273193359375,
      "log_odds_ratio": -0.7012695074081421,
      "logits/chosen": -0.867968738079071,
      "logits/rejected": -0.794726550579071,
      "logps/chosen": -0.791015625,
      "logps/rejected": -0.9535156488418579,
      "loss": 1.0252,
      "nll_loss": 1.02734375,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07913818210363388,
      "rewards/margins": 0.01619873009622097,
      "rewards/rejected": -0.09528808295726776,
      "step": 3090
    },
    {
      "epoch": 0.11616795638080606,
      "grad_norm": 1.3402787117028703,
      "learning_rate": 1.4368424162141992e-06,
      "log_odds_chosen": 0.3442138731479645,
      "log_odds_ratio": -0.65234375,
      "logits/chosen": -0.879687488079071,
      "logits/rejected": -0.821484386920929,
      "logps/chosen": -0.76171875,
      "logps/rejected": -0.991992175579071,
      "loss": 1.0462,
      "nll_loss": 1.017187476158142,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07624511420726776,
      "rewards/margins": 0.023032378405332565,
      "rewards/rejected": -0.09920654445886612,
      "step": 3100
    },
    {
      "epoch": 0.11654269172396994,
      "grad_norm": 1.414084665918844,
      "learning_rate": 1.434530520241665e-06,
      "log_odds_chosen": 0.5091186761856079,
      "log_odds_ratio": -0.556640625,
      "logits/chosen": -0.881640613079071,
      "logits/rejected": -0.8057616949081421,
      "logps/chosen": -0.707226574420929,
      "logps/rejected": -1.033789038658142,
      "loss": 1.05,
      "nll_loss": 0.97265625,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07069091498851776,
      "rewards/margins": 0.03265991061925888,
      "rewards/rejected": -0.10334472358226776,
      "step": 3110
    },
    {
      "epoch": 0.11691742706713383,
      "grad_norm": 1.459365124348827,
      "learning_rate": 1.4322297480788657e-06,
      "log_odds_chosen": 0.37774962186813354,
      "log_odds_ratio": -0.606640636920929,
      "logits/chosen": -0.8695312738418579,
      "logits/rejected": -0.820117175579071,
      "logps/chosen": -0.7171875238418579,
      "logps/rejected": -0.959765613079071,
      "loss": 1.0294,
      "nll_loss": 0.991406261920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07175292819738388,
      "rewards/margins": 0.02429809607565403,
      "rewards/rejected": -0.09605713188648224,
      "step": 3120
    },
    {
      "epoch": 0.11729216241029773,
      "grad_norm": 1.4303194782284898,
      "learning_rate": 1.4299400108063247e-06,
      "log_odds_chosen": 0.46087646484375,
      "log_odds_ratio": -0.6112304925918579,
      "logits/chosen": -0.8623046875,
      "logits/rejected": -0.7826172113418579,
      "logps/chosen": -0.7510741949081421,
      "logps/rejected": -1.0515625476837158,
      "loss": 1.0467,
      "nll_loss": 0.9126952886581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07498779147863388,
      "rewards/margins": 0.0301055908203125,
      "rewards/rejected": -0.10520019382238388,
      "step": 3130
    },
    {
      "epoch": 0.11766689775346162,
      "grad_norm": 1.4087191188240238,
      "learning_rate": 1.4276612204964992e-06,
      "log_odds_chosen": 0.42930907011032104,
      "log_odds_ratio": -0.6119629144668579,
      "logits/chosen": -0.89453125,
      "logits/rejected": -0.8314453363418579,
      "logps/chosen": -0.689257800579071,
      "logps/rejected": -0.973828136920929,
      "loss": 1.035,
      "nll_loss": 0.9248046875,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06893310695886612,
      "rewards/margins": 0.02838134765625,
      "rewards/rejected": -0.09730224311351776,
      "step": 3140
    },
    {
      "epoch": 0.1180416330966255,
      "grad_norm": 1.4360149391464714,
      "learning_rate": 1.4253932901995967e-06,
      "log_odds_chosen": 0.43780517578125,
      "log_odds_ratio": -0.599414050579071,
      "logits/chosen": -0.904492199420929,
      "logits/rejected": -0.848828136920929,
      "logps/chosen": -0.7173827886581421,
      "logps/rejected": -0.9957031011581421,
      "loss": 1.0345,
      "nll_loss": 0.951367199420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07167968899011612,
      "rewards/margins": 0.02783966064453125,
      "rewards/rejected": -0.099578857421875,
      "step": 3150
    },
    {
      "epoch": 0.1184163684397894,
      "grad_norm": 1.3786843080791558,
      "learning_rate": 1.42313613392964e-06,
      "log_odds_chosen": 0.33204954862594604,
      "log_odds_ratio": -0.639941394329071,
      "logits/chosen": -0.856640636920929,
      "logits/rejected": -0.8255859613418579,
      "logps/chosen": -0.718554675579071,
      "logps/rejected": -0.951367199420929,
      "loss": 1.0514,
      "nll_loss": 1.029296875,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07186279445886612,
      "rewards/margins": 0.02335662767291069,
      "rewards/rejected": -0.09511718899011612,
      "step": 3160
    },
    {
      "epoch": 0.1187911037829533,
      "grad_norm": 1.443707117486184,
      "learning_rate": 1.4208896666507756e-06,
      "log_odds_chosen": 0.4242309629917145,
      "log_odds_ratio": -0.616015613079071,
      "logits/chosen": -0.900585949420929,
      "logits/rejected": -0.843554675579071,
      "logps/chosen": -0.713671863079071,
      "logps/rejected": -0.9808593988418579,
      "loss": 1.0487,
      "nll_loss": 0.991992175579071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07142333686351776,
      "rewards/margins": 0.02670135535299778,
      "rewards/rejected": -0.09812011569738388,
      "step": 3170
    },
    {
      "epoch": 0.11916583912611718,
      "grad_norm": 1.3871849605017201,
      "learning_rate": 1.4186538042638173e-06,
      "log_odds_chosen": 0.3927368223667145,
      "log_odds_ratio": -0.642285168170929,
      "logits/chosen": -0.827929675579071,
      "logits/rejected": -0.7798827886581421,
      "logps/chosen": -0.7515624761581421,
      "logps/rejected": -1.0149414539337158,
      "loss": 1.0622,
      "nll_loss": 1.0236327648162842,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07514648139476776,
      "rewards/margins": 0.02628173865377903,
      "rewards/rejected": -0.10137939453125,
      "step": 3180
    },
    {
      "epoch": 0.11954057446928107,
      "grad_norm": 1.319691391190568,
      "learning_rate": 1.416428463593022e-06,
      "log_odds_chosen": 0.2774291932582855,
      "log_odds_ratio": -0.6651366949081421,
      "logits/chosen": -0.901171863079071,
      "logits/rejected": -0.8363281488418579,
      "logps/chosen": -0.757031261920929,
      "logps/rejected": -0.9515625238418579,
      "loss": 1.0296,
      "nll_loss": 1.000390648841858,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07570800930261612,
      "rewards/margins": 0.0194854736328125,
      "rewards/rejected": -0.09511718899011612,
      "step": 3190
    },
    {
      "epoch": 0.11991530981244496,
      "grad_norm": 1.4873151124605148,
      "learning_rate": 1.414213562373095e-06,
      "log_odds_chosen": 0.3711181581020355,
      "log_odds_ratio": -0.664843738079071,
      "logits/chosen": -0.824023425579071,
      "logits/rejected": -0.7855468988418579,
      "logps/chosen": -0.7586914300918579,
      "logps/rejected": -1.0148437023162842,
      "loss": 1.0344,
      "nll_loss": 1.0009765625,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07591553032398224,
      "rewards/margins": 0.02562713623046875,
      "rewards/rejected": -0.10136719048023224,
      "step": 3200
    },
    {
      "epoch": 0.12029004515560886,
      "grad_norm": 1.4291912255371704,
      "learning_rate": 1.4120090192364154e-06,
      "log_odds_chosen": 0.47503662109375,
      "log_odds_ratio": -0.5986328125,
      "logits/chosen": -0.8119140863418579,
      "logits/rejected": -0.7724609375,
      "logps/chosen": -0.7127929925918579,
      "logps/rejected": -1.005859375,
      "loss": 1.0298,
      "nll_loss": 0.951367199420929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07125244289636612,
      "rewards/margins": 0.02936859056353569,
      "rewards/rejected": -0.10063476860523224,
      "step": 3210
    },
    {
      "epoch": 0.12066478049877274,
      "grad_norm": 1.3770430903188104,
      "learning_rate": 1.4098147537004828e-06,
      "log_odds_chosen": 0.4725585877895355,
      "log_odds_ratio": -0.612500011920929,
      "logits/chosen": -0.798535168170929,
      "logits/rejected": -0.7416015863418579,
      "logps/chosen": -0.7197265625,
      "logps/rejected": -1.0525391101837158,
      "loss": 1.0624,
      "nll_loss": 1.0343749523162842,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07198486477136612,
      "rewards/margins": 0.03326873853802681,
      "rewards/rejected": -0.10533447563648224,
      "step": 3220
    },
    {
      "epoch": 0.12103951584193663,
      "grad_norm": 1.3476537562015602,
      "learning_rate": 1.4076306861555735e-06,
      "log_odds_chosen": 0.32861328125,
      "log_odds_ratio": -0.6463867425918579,
      "logits/chosen": -0.83984375,
      "logits/rejected": -0.792187511920929,
      "logps/chosen": -0.777148425579071,
      "logps/rejected": -0.9908202886581421,
      "loss": 1.0616,
      "nll_loss": 0.9916015863418579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07767333835363388,
      "rewards/margins": 0.021366119384765625,
      "rewards/rejected": -0.09907226264476776,
      "step": 3230
    },
    {
      "epoch": 0.12141425118510052,
      "grad_norm": 1.3844653506462807,
      "learning_rate": 1.405456737852613e-06,
      "log_odds_chosen": 0.3564086854457855,
      "log_odds_ratio": -0.6587890386581421,
      "logits/chosen": -0.862500011920929,
      "logits/rejected": -0.8050781488418579,
      "logps/chosen": -0.7914062738418579,
      "logps/rejected": -1.029296875,
      "loss": 1.0619,
      "nll_loss": 1.0216796398162842,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07911376655101776,
      "rewards/margins": 0.02381134033203125,
      "rewards/rejected": -0.10289306938648224,
      "step": 3240
    },
    {
      "epoch": 0.12178898652826442,
      "grad_norm": 1.353635566483262,
      "learning_rate": 1.4032928308912468e-06,
      "log_odds_chosen": 0.4760375916957855,
      "log_odds_ratio": -0.586718738079071,
      "logits/chosen": -0.821484386920929,
      "logits/rejected": -0.736328125,
      "logps/chosen": -0.6826171875,
      "logps/rejected": -0.9898437261581421,
      "loss": 1.0458,
      "nll_loss": 0.975781261920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06828613579273224,
      "rewards/margins": 0.03066406212747097,
      "rewards/rejected": -0.09895019233226776,
      "step": 3250
    },
    {
      "epoch": 0.12216372187142831,
      "grad_norm": 1.3721574217411596,
      "learning_rate": 1.4011388882081175e-06,
      "log_odds_chosen": 0.236053466796875,
      "log_odds_ratio": -0.6756836175918579,
      "logits/chosen": -0.858593761920929,
      "logits/rejected": -0.79248046875,
      "logps/chosen": -0.7416015863418579,
      "logps/rejected": -0.914257824420929,
      "loss": 1.0522,
      "nll_loss": 0.9603515863418579,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07412109524011612,
      "rewards/margins": 0.017298126593232155,
      "rewards/rejected": -0.09140624850988388,
      "step": 3260
    },
    {
      "epoch": 0.1225384572145922,
      "grad_norm": 1.396525346195492,
      "learning_rate": 1.3989948335653378e-06,
      "log_odds_chosen": 0.4423828125,
      "log_odds_ratio": -0.60595703125,
      "logits/chosen": -0.8345702886581421,
      "logits/rejected": -0.7632812261581421,
      "logps/chosen": -0.6947265863418579,
      "logps/rejected": -0.9808593988418579,
      "loss": 1.0643,
      "nll_loss": 0.957226574420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06943359225988388,
      "rewards/margins": 0.02863464318215847,
      "rewards/rejected": -0.09810791164636612,
      "step": 3270
    },
    {
      "epoch": 0.12291319255775608,
      "grad_norm": 1.4458279225959236,
      "learning_rate": 1.3968605915391564e-06,
      "log_odds_chosen": 0.44831544160842896,
      "log_odds_ratio": -0.6128906011581421,
      "logits/chosen": -0.839648425579071,
      "logits/rejected": -0.7759765386581421,
      "logps/chosen": -0.704296886920929,
      "logps/rejected": -1.0148437023162842,
      "loss": 1.0261,
      "nll_loss": 0.974804699420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07039795070886612,
      "rewards/margins": 0.031015777960419655,
      "rewards/rejected": -0.10152588039636612,
      "step": 3280
    },
    {
      "epoch": 0.12328792790091997,
      "grad_norm": 1.325385003158149,
      "learning_rate": 1.3947360875088132e-06,
      "log_odds_chosen": 0.42840576171875,
      "log_odds_ratio": -0.612011730670929,
      "logits/chosen": -0.8130859136581421,
      "logits/rejected": -0.7891601324081421,
      "logps/chosen": -0.7083984613418579,
      "logps/rejected": -0.9857422113418579,
      "loss": 1.033,
      "nll_loss": 1.034765601158142,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07082519680261612,
      "rewards/margins": 0.027794647961854935,
      "rewards/rejected": -0.09857177734375,
      "step": 3290
    },
    {
      "epoch": 0.12366266324408387,
      "grad_norm": 1.3590426568847132,
      "learning_rate": 1.3926212476455828e-06,
      "log_odds_chosen": 0.3639892637729645,
      "log_odds_ratio": -0.6075195074081421,
      "logits/chosen": -0.8447265625,
      "logits/rejected": -0.8021484613418579,
      "logps/chosen": -0.726757824420929,
      "logps/rejected": -0.953125,
      "loss": 1.0328,
      "nll_loss": 0.958984375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07271728664636612,
      "rewards/margins": 0.022624969482421875,
      "rewards/rejected": -0.09530029445886612,
      "step": 3300
    },
    {
      "epoch": 0.12403739858724776,
      "grad_norm": 1.4071983633699567,
      "learning_rate": 1.3905159989019964e-06,
      "log_odds_chosen": 0.36762696504592896,
      "log_odds_ratio": -0.635058581829071,
      "logits/chosen": -0.857617199420929,
      "logits/rejected": -0.7896484136581421,
      "logps/chosen": -0.7110351324081421,
      "logps/rejected": -0.9476562738418579,
      "loss": 1.0491,
      "nll_loss": 0.9535156488418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.071044921875,
      "rewards/margins": 0.02376861497759819,
      "rewards/rejected": -0.09479980170726776,
      "step": 3310
    },
    {
      "epoch": 0.12441213393041164,
      "grad_norm": 1.4820643471773292,
      "learning_rate": 1.3884202690012465e-06,
      "log_odds_chosen": 0.41984862089157104,
      "log_odds_ratio": -0.621777355670929,
      "logits/chosen": -0.9048827886581421,
      "logits/rejected": -0.817578136920929,
      "logps/chosen": -0.757617175579071,
      "logps/rejected": -1.0478515625,
      "loss": 1.0382,
      "nll_loss": 0.943164050579071,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07573242485523224,
      "rewards/margins": 0.029056549072265625,
      "rewards/rejected": -0.10479736328125,
      "step": 3320
    },
    {
      "epoch": 0.12478686927357553,
      "grad_norm": 1.495950432269586,
      "learning_rate": 1.3863339864267636e-06,
      "log_odds_chosen": 0.46563720703125,
      "log_odds_ratio": -0.6078125238418579,
      "logits/chosen": -0.7525390386581421,
      "logits/rejected": -0.704394519329071,
      "logps/chosen": -0.716796875,
      "logps/rejected": -1.0109374523162842,
      "loss": 1.0475,
      "nll_loss": 0.994335949420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0716552734375,
      "rewards/margins": 0.029465485364198685,
      "rewards/rejected": -0.10109863430261612,
      "step": 3330
    },
    {
      "epoch": 0.12516160461673942,
      "grad_norm": 1.4863140357349514,
      "learning_rate": 1.3842570804119655e-06,
      "log_odds_chosen": 0.3900512754917145,
      "log_odds_ratio": -0.609082043170929,
      "logits/chosen": -0.7890625,
      "logits/rejected": -0.7186523675918579,
      "logps/chosen": -0.6919921636581421,
      "logps/rejected": -0.9349609613418579,
      "loss": 1.0472,
      "nll_loss": 0.9330078363418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06922607123851776,
      "rewards/margins": 0.02424926683306694,
      "rewards/rejected": -0.09348144382238388,
      "step": 3340
    },
    {
      "epoch": 0.12553633995990332,
      "grad_norm": 1.3249405967927652,
      "learning_rate": 1.3821894809301763e-06,
      "log_odds_chosen": 0.531005859375,
      "log_odds_ratio": -0.567675769329071,
      "logits/chosen": -0.7613281011581421,
      "logits/rejected": -0.6839843988418579,
      "logps/chosen": -0.6958984136581421,
      "logps/rejected": -1.0212891101837158,
      "loss": 1.038,
      "nll_loss": 0.9546874761581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.069580078125,
      "rewards/margins": 0.03241271898150444,
      "rewards/rejected": -0.10205078125,
      "step": 3350
    },
    {
      "epoch": 0.12591107530306722,
      "grad_norm": 1.384409442735042,
      "learning_rate": 1.3801311186847081e-06,
      "log_odds_chosen": 0.49871826171875,
      "log_odds_ratio": -0.5743163824081421,
      "logits/chosen": -0.788867175579071,
      "logits/rejected": -0.7109375,
      "logps/chosen": -0.7005859613418579,
      "logps/rejected": -1.0158202648162842,
      "loss": 1.0579,
      "nll_loss": 1.005468726158142,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07005615532398224,
      "rewards/margins": 0.03151092678308487,
      "rewards/rejected": -0.10161133110523224,
      "step": 3360
    },
    {
      "epoch": 0.1262858106462311,
      "grad_norm": 1.312068841764017,
      "learning_rate": 1.378081925099109e-06,
      "log_odds_chosen": 0.41779786348342896,
      "log_odds_ratio": -0.6202148199081421,
      "logits/chosen": -0.7855468988418579,
      "logits/rejected": -0.70458984375,
      "logps/chosen": -0.70703125,
      "logps/rejected": -0.9691406488418579,
      "loss": 1.0357,
      "nll_loss": 0.9966796636581421,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07070312649011612,
      "rewards/margins": 0.02627105638384819,
      "rewards/rejected": -0.09694824367761612,
      "step": 3370
    },
    {
      "epoch": 0.126660545989395,
      "grad_norm": 1.3732567496341053,
      "learning_rate": 1.376041832307563e-06,
      "log_odds_chosen": 0.24857178330421448,
      "log_odds_ratio": -0.6629883050918579,
      "logits/chosen": -0.8179687261581421,
      "logits/rejected": -0.759960949420929,
      "logps/chosen": -0.777539074420929,
      "logps/rejected": -0.9595702886581421,
      "loss": 1.0447,
      "nll_loss": 0.9984375238418579,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07770995795726776,
      "rewards/margins": 0.01820068433880806,
      "rewards/rejected": -0.09593506157398224,
      "step": 3380
    },
    {
      "epoch": 0.12703528133255887,
      "grad_norm": 1.356797273376919,
      "learning_rate": 1.3740107731454524e-06,
      "log_odds_chosen": 0.30512696504592896,
      "log_odds_ratio": -0.6717773675918579,
      "logits/chosen": -0.827929675579071,
      "logits/rejected": -0.731640636920929,
      "logps/chosen": -0.7611328363418579,
      "logps/rejected": -0.9722656011581421,
      "loss": 1.0304,
      "nll_loss": 0.9779297113418579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07609863579273224,
      "rewards/margins": 0.02121734619140625,
      "rewards/rejected": -0.09733887016773224,
      "step": 3390
    },
    {
      "epoch": 0.12741001667572277,
      "grad_norm": 1.433099794939882,
      "learning_rate": 1.3719886811400705e-06,
      "log_odds_chosen": 0.4092163145542145,
      "log_odds_ratio": -0.628125011920929,
      "logits/chosen": -0.819531261920929,
      "logits/rejected": -0.743945300579071,
      "logps/chosen": -0.732617199420929,
      "logps/rejected": -1.0183594226837158,
      "loss": 1.0312,
      "nll_loss": 0.9580078125,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07315673679113388,
      "rewards/margins": 0.028684234246611595,
      "rewards/rejected": -0.10189209133386612,
      "step": 3400
    },
    {
      "epoch": 0.12778475201888667,
      "grad_norm": 1.9813930689907795,
      "learning_rate": 1.3699754905014834e-06,
      "log_odds_chosen": 0.15311279892921448,
      "log_odds_ratio": -0.725390613079071,
      "logits/chosen": -0.8212890625,
      "logits/rejected": -0.7826172113418579,
      "logps/chosen": -0.750781238079071,
      "logps/rejected": -0.8798828125,
      "loss": 1.052,
      "nll_loss": 0.989062488079071,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.07509765774011612,
      "rewards/margins": 0.012919616885483265,
      "rewards/rejected": -0.08797607570886612,
      "step": 3410
    },
    {
      "epoch": 0.12815948736205054,
      "grad_norm": 1.4461666730789868,
      "learning_rate": 1.3679711361135388e-06,
      "log_odds_chosen": 0.29742431640625,
      "log_odds_ratio": -0.6607421636581421,
      "logits/chosen": -0.7666015625,
      "logits/rejected": -0.710742175579071,
      "logps/chosen": -0.723437488079071,
      "logps/rejected": -0.929492175579071,
      "loss": 1.0285,
      "nll_loss": 1.0246093273162842,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07236327975988388,
      "rewards/margins": 0.020700836554169655,
      "rewards/rejected": -0.09304199367761612,
      "step": 3420
    },
    {
      "epoch": 0.12853422270521445,
      "grad_norm": 1.3925076815748787,
      "learning_rate": 1.3659755535250212e-06,
      "log_odds_chosen": 0.4573120176792145,
      "log_odds_ratio": -0.609375,
      "logits/chosen": -0.8291015625,
      "logits/rejected": -0.798828125,
      "logps/chosen": -0.716796875,
      "logps/rejected": -1.0246093273162842,
      "loss": 1.0289,
      "nll_loss": 0.974609375,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07159423828125,
      "rewards/margins": 0.03085937537252903,
      "rewards/rejected": -0.10246582329273224,
      "step": 3430
    },
    {
      "epoch": 0.12890895804837832,
      "grad_norm": 1.3191892189429664,
      "learning_rate": 1.3639886789409469e-06,
      "log_odds_chosen": 0.29534912109375,
      "log_odds_ratio": -0.6670898199081421,
      "logits/chosen": -0.837695300579071,
      "logits/rejected": -0.7447265386581421,
      "logps/chosen": -0.736328125,
      "logps/rejected": -0.9253906011581421,
      "loss": 1.0246,
      "nll_loss": 0.99609375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07362060248851776,
      "rewards/margins": 0.01891174353659153,
      "rewards/rejected": -0.092529296875,
      "step": 3440
    },
    {
      "epoch": 0.12928369339154222,
      "grad_norm": 1.3928706618733364,
      "learning_rate": 1.3620104492139977e-06,
      "log_odds_chosen": 0.46165770292282104,
      "log_odds_ratio": -0.5877929925918579,
      "logits/chosen": -0.842578113079071,
      "logits/rejected": -0.7919921875,
      "logps/chosen": -0.6849609613418579,
      "logps/rejected": -0.985546886920929,
      "loss": 1.0087,
      "nll_loss": 0.9673827886581421,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06845702975988388,
      "rewards/margins": 0.03009796142578125,
      "rewards/rejected": -0.09858398139476776,
      "step": 3450
    },
    {
      "epoch": 0.12965842873470612,
      "grad_norm": 1.338829833429675,
      "learning_rate": 1.3600408018360918e-06,
      "log_odds_chosen": 0.3080810606479645,
      "log_odds_ratio": -0.6650390625,
      "logits/chosen": -0.813281238079071,
      "logits/rejected": -0.76123046875,
      "logps/chosen": -0.7691406011581421,
      "logps/rejected": -0.989453136920929,
      "loss": 1.0715,
      "nll_loss": 1.0146484375,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07680664211511612,
      "rewards/margins": 0.02215118333697319,
      "rewards/rejected": -0.09903564304113388,
      "step": 3460
    },
    {
      "epoch": 0.13003316407787,
      "grad_norm": 1.4023106367878078,
      "learning_rate": 1.3580796749300878e-06,
      "log_odds_chosen": 0.39167481660842896,
      "log_odds_ratio": -0.633593738079071,
      "logits/chosen": -0.8506835699081421,
      "logits/rejected": -0.7796875238418579,
      "logps/chosen": -0.737109363079071,
      "logps/rejected": -0.998242199420929,
      "loss": 1.0167,
      "nll_loss": 0.9937499761581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07371826469898224,
      "rewards/margins": 0.02617492713034153,
      "rewards/rejected": -0.09982910007238388,
      "step": 3470
    },
    {
      "epoch": 0.1304078994210339,
      "grad_norm": 1.3709833638840578,
      "learning_rate": 1.3561270072416209e-06,
      "log_odds_chosen": 0.29442137479782104,
      "log_odds_ratio": -0.6573241949081421,
      "logits/chosen": -0.843945324420929,
      "logits/rejected": -0.790820300579071,
      "logps/chosen": -0.7914062738418579,
      "logps/rejected": -0.989453136920929,
      "loss": 1.0266,
      "nll_loss": 0.981640636920929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07913818210363388,
      "rewards/margins": 0.01979980431497097,
      "rewards/rejected": -0.09896240383386612,
      "step": 3480
    },
    {
      "epoch": 0.1307826347641978,
      "grad_norm": 1.3155632513631141,
      "learning_rate": 1.3541827381310652e-06,
      "log_odds_chosen": 0.2837890684604645,
      "log_odds_ratio": -0.661328136920929,
      "logits/chosen": -0.8246093988418579,
      "logits/rejected": -0.773632824420929,
      "logps/chosen": -0.756054699420929,
      "logps/rejected": -0.953906238079071,
      "loss": 1.0354,
      "nll_loss": 0.9828125238418579,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07565917819738388,
      "rewards/margins": 0.019792938604950905,
      "rewards/rejected": -0.09537353366613388,
      "step": 3490
    },
    {
      "epoch": 0.13115737010736167,
      "grad_norm": 1.52101861703067,
      "learning_rate": 1.3522468075656264e-06,
      "log_odds_chosen": 0.36920166015625,
      "log_odds_ratio": -0.634082019329071,
      "logits/chosen": -0.830859363079071,
      "logits/rejected": -0.7632812261581421,
      "logps/chosen": -0.744140625,
      "logps/rejected": -0.9814453125,
      "loss": 1.0341,
      "nll_loss": 0.9378906488418579,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07437743991613388,
      "rewards/margins": 0.023767853155732155,
      "rewards/rejected": -0.09821777045726776,
      "step": 3500
    },
    {
      "epoch": 0.13153210545052557,
      "grad_norm": 1.4382283925331376,
      "learning_rate": 1.3503191561115553e-06,
      "log_odds_chosen": 0.46629637479782104,
      "log_odds_ratio": -0.5947265625,
      "logits/chosen": -0.801562488079071,
      "logits/rejected": -0.7601562738418579,
      "logps/chosen": -0.7210937738418579,
      "logps/rejected": -1.004296898841858,
      "loss": 1.0299,
      "nll_loss": 0.932421863079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07207031548023224,
      "rewards/margins": 0.02826385572552681,
      "rewards/rejected": -0.10036621242761612,
      "step": 3510
    },
    {
      "epoch": 0.13190684079368944,
      "grad_norm": 1.4522531536635177,
      "learning_rate": 1.348399724926484e-06,
      "log_odds_chosen": 0.35041505098342896,
      "log_odds_ratio": -0.626660168170929,
      "logits/chosen": -0.797656238079071,
      "logits/rejected": -0.727343738079071,
      "logps/chosen": -0.744140625,
      "logps/rejected": -0.9662109613418579,
      "loss": 1.0486,
      "nll_loss": 1.0294921398162842,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07441405951976776,
      "rewards/margins": 0.022223662585020065,
      "rewards/rejected": -0.09652099758386612,
      "step": 3520
    },
    {
      "epoch": 0.13228157613685335,
      "grad_norm": 1.716159362581383,
      "learning_rate": 1.346488455751882e-06,
      "log_odds_chosen": 0.46198731660842896,
      "log_odds_ratio": -0.6004883050918579,
      "logits/chosen": -0.8208984136581421,
      "logits/rejected": -0.7105468511581421,
      "logps/chosen": -0.741992175579071,
      "logps/rejected": -1.047460913658142,
      "loss": 1.0464,
      "nll_loss": 1.052343726158142,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07420654594898224,
      "rewards/margins": 0.03058624267578125,
      "rewards/rejected": -0.10472412407398224,
      "step": 3530
    },
    {
      "epoch": 0.13265631148001725,
      "grad_norm": 1.3539447529214195,
      "learning_rate": 1.3445852909056286e-06,
      "log_odds_chosen": 0.31456297636032104,
      "log_odds_ratio": -0.651074230670929,
      "logits/chosen": -0.8304687738418579,
      "logits/rejected": -0.7642577886581421,
      "logps/chosen": -0.7591797113418579,
      "logps/rejected": -0.9736328125,
      "loss": 1.0358,
      "nll_loss": 1.057226538658142,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07595214992761612,
      "rewards/margins": 0.02144470252096653,
      "rewards/rejected": -0.09737548977136612,
      "step": 3540
    },
    {
      "epoch": 0.13303104682318112,
      "grad_norm": 1.5130263646303688,
      "learning_rate": 1.3426901732747024e-06,
      "log_odds_chosen": 0.54638671875,
      "log_odds_ratio": -0.5826171636581421,
      "logits/chosen": -0.7982422113418579,
      "logits/rejected": -0.7032226324081421,
      "logps/chosen": -0.7242187261581421,
      "logps/rejected": -1.0841796398162842,
      "loss": 1.0471,
      "nll_loss": 0.981640636920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07243652641773224,
      "rewards/margins": 0.03590850904583931,
      "rewards/rejected": -0.10837402194738388,
      "step": 3550
    },
    {
      "epoch": 0.13340578216634502,
      "grad_norm": 1.3362395082861855,
      "learning_rate": 1.3408030463079818e-06,
      "log_odds_chosen": 0.25739747285842896,
      "log_odds_ratio": -0.6678711175918579,
      "logits/chosen": -0.841992199420929,
      "logits/rejected": -0.7679687738418579,
      "logps/chosen": -0.7734375,
      "logps/rejected": -0.9609375,
      "loss": 1.0497,
      "nll_loss": 0.968945324420929,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07740478217601776,
      "rewards/margins": 0.0186614990234375,
      "rewards/rejected": -0.0960693359375,
      "step": 3560
    },
    {
      "epoch": 0.13378051750950892,
      "grad_norm": 1.4761566512192363,
      "learning_rate": 1.3389238540091568e-06,
      "log_odds_chosen": 0.3655761778354645,
      "log_odds_ratio": -0.6329101324081421,
      "logits/chosen": -0.8187500238418579,
      "logits/rejected": -0.756054699420929,
      "logps/chosen": -0.720898449420929,
      "logps/rejected": -0.9517577886581421,
      "loss": 1.0254,
      "nll_loss": 0.944531261920929,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07197265326976776,
      "rewards/margins": 0.02305450476706028,
      "rewards/rejected": -0.09504394233226776,
      "step": 3570
    },
    {
      "epoch": 0.1341552528526728,
      "grad_norm": 1.4566250946732364,
      "learning_rate": 1.337052540929751e-06,
      "log_odds_chosen": 0.36723631620407104,
      "log_odds_ratio": -0.6294921636581421,
      "logits/chosen": -0.8238281011581421,
      "logits/rejected": -0.725390613079071,
      "logps/chosen": -0.730273425579071,
      "logps/rejected": -0.97265625,
      "loss": 1.0316,
      "nll_loss": 0.925000011920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07305908203125,
      "rewards/margins": 0.024263763800263405,
      "rewards/rejected": -0.09733887016773224,
      "step": 3580
    },
    {
      "epoch": 0.1345299881958367,
      "grad_norm": 1.3220712017598293,
      "learning_rate": 1.33518905216225e-06,
      "log_odds_chosen": 0.4074462950229645,
      "log_odds_ratio": -0.621386706829071,
      "logits/chosen": -0.839062511920929,
      "logits/rejected": -0.7890625,
      "logps/chosen": -0.702929675579071,
      "logps/rejected": -1.005468726158142,
      "loss": 1.0093,
      "nll_loss": 0.997851550579071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07020263373851776,
      "rewards/margins": 0.03032989427447319,
      "rewards/rejected": -0.10061035305261612,
      "step": 3590
    },
    {
      "epoch": 0.13490472353900057,
      "grad_norm": 1.4949156329113271,
      "learning_rate": 1.3333333333333332e-06,
      "log_odds_chosen": 0.28107911348342896,
      "log_odds_ratio": -0.6778320074081421,
      "logits/chosen": -0.8374999761581421,
      "logits/rejected": -0.7724609375,
      "logps/chosen": -0.749218761920929,
      "logps/rejected": -0.9462890625,
      "loss": 1.0384,
      "nll_loss": 0.964062511920929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.074951171875,
      "rewards/margins": 0.01965789869427681,
      "rewards/rejected": -0.09467773139476776,
      "step": 3600
    },
    {
      "epoch": 0.13527945888216447,
      "grad_norm": 1.345672455128302,
      "learning_rate": 1.3314853305972122e-06,
      "log_odds_chosen": 0.38543701171875,
      "log_odds_ratio": -0.624316394329071,
      "logits/chosen": -0.781445324420929,
      "logits/rejected": -0.6953125,
      "logps/chosen": -0.725390613079071,
      "logps/rejected": -0.9908202886581421,
      "loss": 1.0407,
      "nll_loss": 0.9970703125,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07258300483226776,
      "rewards/margins": 0.02642059326171875,
      "rewards/rejected": -0.09907226264476776,
      "step": 3610
    },
    {
      "epoch": 0.13565419422532837,
      "grad_norm": 1.3163379100143133,
      "learning_rate": 1.3296449906290671e-06,
      "log_odds_chosen": 0.46027833223342896,
      "log_odds_ratio": -0.602343738079071,
      "logits/chosen": -0.822070300579071,
      "logits/rejected": -0.7861328125,
      "logps/chosen": -0.728515625,
      "logps/rejected": -1.0314452648162842,
      "loss": 1.0427,
      "nll_loss": 0.9261718988418579,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07283935695886612,
      "rewards/margins": 0.03036956861615181,
      "rewards/rejected": -0.10317382961511612,
      "step": 3620
    },
    {
      "epoch": 0.13602892956849225,
      "grad_norm": 1.5075954977643606,
      "learning_rate": 1.3278122606185844e-06,
      "log_odds_chosen": 0.34290772676467896,
      "log_odds_ratio": -0.639941394329071,
      "logits/chosen": -0.8150390386581421,
      "logits/rejected": -0.797656238079071,
      "logps/chosen": -0.7198241949081421,
      "logps/rejected": -0.949023425579071,
      "loss": 1.0419,
      "nll_loss": 0.961132824420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07197265326976776,
      "rewards/margins": 0.022899627685546875,
      "rewards/rejected": -0.09489746391773224,
      "step": 3630
    },
    {
      "epoch": 0.13640366491165615,
      "grad_norm": 1.3488073266856888,
      "learning_rate": 1.3259870882635918e-06,
      "log_odds_chosen": 0.25041502714157104,
      "log_odds_ratio": -0.691699206829071,
      "logits/chosen": -0.8681640625,
      "logits/rejected": -0.785351574420929,
      "logps/chosen": -0.7671874761581421,
      "logps/rejected": -0.936718761920929,
      "loss": 1.0428,
      "nll_loss": 0.9326171875,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07674560695886612,
      "rewards/margins": 0.01700744591653347,
      "rewards/rejected": -0.09376220405101776,
      "step": 3640
    },
    {
      "epoch": 0.13677840025482002,
      "grad_norm": 1.5109216424473435,
      "learning_rate": 1.3241694217637886e-06,
      "log_odds_chosen": 0.5709472894668579,
      "log_odds_ratio": -0.5728515386581421,
      "logits/chosen": -0.8427734375,
      "logits/rejected": -0.741015613079071,
      "logps/chosen": -0.7035156488418579,
      "logps/rejected": -1.083593726158142,
      "loss": 1.033,
      "nll_loss": 0.9478515386581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07041015475988388,
      "rewards/margins": 0.037933349609375,
      "rewards/rejected": -0.10832519829273224,
      "step": 3650
    },
    {
      "epoch": 0.13715313559798392,
      "grad_norm": 1.4929980039594493,
      "learning_rate": 1.3223592098145723e-06,
      "log_odds_chosen": 0.532482922077179,
      "log_odds_ratio": -0.5918945074081421,
      "logits/chosen": -0.8607422113418579,
      "logits/rejected": -0.7544921636581421,
      "logps/chosen": -0.7314453125,
      "logps/rejected": -1.1150391101837158,
      "loss": 1.0463,
      "nll_loss": 0.996874988079071,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07315673679113388,
      "rewards/margins": 0.0383148193359375,
      "rewards/rejected": -0.11140136420726776,
      "step": 3660
    },
    {
      "epoch": 0.13752787094114782,
      "grad_norm": 1.3676830938227817,
      "learning_rate": 1.3205564016009555e-06,
      "log_odds_chosen": 0.40068358182907104,
      "log_odds_ratio": -0.604785144329071,
      "logits/chosen": -0.8343750238418579,
      "logits/rejected": -0.7564452886581421,
      "logps/chosen": -0.716992199420929,
      "logps/rejected": -0.9750000238418579,
      "loss": 1.0305,
      "nll_loss": 0.9400390386581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07171630859375,
      "rewards/margins": 0.02590484544634819,
      "rewards/rejected": -0.09758301079273224,
      "step": 3670
    },
    {
      "epoch": 0.1379026062843117,
      "grad_norm": 1.3983394159629317,
      "learning_rate": 1.318760946791574e-06,
      "log_odds_chosen": 0.38868409395217896,
      "log_odds_ratio": -0.6312500238418579,
      "logits/chosen": -0.783984363079071,
      "logits/rejected": -0.7470703125,
      "logps/chosen": -0.73388671875,
      "logps/rejected": -1.005468726158142,
      "loss": 1.039,
      "nll_loss": 1.0447266101837158,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07332763820886612,
      "rewards/margins": 0.02717742882668972,
      "rewards/rejected": -0.10052490234375,
      "step": 3680
    },
    {
      "epoch": 0.1382773416274756,
      "grad_norm": 1.4297707473297936,
      "learning_rate": 1.316972795532786e-06,
      "log_odds_chosen": 0.3338256776332855,
      "log_odds_ratio": -0.643261730670929,
      "logits/chosen": -0.806445300579071,
      "logits/rejected": -0.7601562738418579,
      "logps/chosen": -0.75244140625,
      "logps/rejected": -0.9652343988418579,
      "loss": 1.0608,
      "nll_loss": 0.9701172113418579,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07528076320886612,
      "rewards/margins": 0.021310806274414062,
      "rewards/rejected": -0.09653320163488388,
      "step": 3690
    },
    {
      "epoch": 0.1386520769706395,
      "grad_norm": 1.3506650836155463,
      "learning_rate": 1.3151918984428582e-06,
      "log_odds_chosen": 0.539990246295929,
      "log_odds_ratio": -0.5663086175918579,
      "logits/chosen": -0.8597656488418579,
      "logits/rejected": -0.739062488079071,
      "logps/chosen": -0.6869140863418579,
      "logps/rejected": -1.035742163658142,
      "loss": 1.0472,
      "nll_loss": 0.9642578363418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06868896633386612,
      "rewards/margins": 0.03497924655675888,
      "rewards/rejected": -0.10362549126148224,
      "step": 3700
    },
    {
      "epoch": 0.13902681231380337,
      "grad_norm": 1.3737570360783824,
      "learning_rate": 1.313418206606237e-06,
      "log_odds_chosen": 0.39312744140625,
      "log_odds_ratio": -0.632128894329071,
      "logits/chosen": -0.787304699420929,
      "logits/rejected": -0.741406261920929,
      "logps/chosen": -0.770800769329071,
      "logps/rejected": -1.0431640148162842,
      "loss": 1.0318,
      "nll_loss": 1.03515625,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07705078274011612,
      "rewards/margins": 0.0272979736328125,
      "rewards/rejected": -0.10440673679113388,
      "step": 3710
    },
    {
      "epoch": 0.13940154765696727,
      "grad_norm": 1.3997929175165282,
      "learning_rate": 1.3116516715679057e-06,
      "log_odds_chosen": 0.4538330137729645,
      "log_odds_ratio": -0.60400390625,
      "logits/chosen": -0.810351550579071,
      "logits/rejected": -0.701953113079071,
      "logps/chosen": -0.7236328125,
      "logps/rejected": -1.016992211341858,
      "loss": 1.0563,
      "nll_loss": 1.0246093273162842,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07241211086511612,
      "rewards/margins": 0.02933349646627903,
      "rewards/rejected": -0.10170898586511612,
      "step": 3720
    },
    {
      "epoch": 0.13977628300013115,
      "grad_norm": 1.4822147007641422,
      "learning_rate": 1.3098922453278258e-06,
      "log_odds_chosen": 0.4449462890625,
      "log_odds_ratio": -0.6058593988418579,
      "logits/chosen": -0.8145507574081421,
      "logits/rejected": -0.737011730670929,
      "logps/chosen": -0.7740234136581421,
      "logps/rejected": -1.076757788658142,
      "loss": 1.0267,
      "nll_loss": 0.976367175579071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07740478217601776,
      "rewards/margins": 0.0301666259765625,
      "rewards/rejected": -0.1075439453125,
      "step": 3730
    },
    {
      "epoch": 0.14015101834329505,
      "grad_norm": 1.3949462541966722,
      "learning_rate": 1.3081398803354573e-06,
      "log_odds_chosen": 0.510693371295929,
      "log_odds_ratio": -0.625048816204071,
      "logits/chosen": -0.8462890386581421,
      "logits/rejected": -0.7696288824081421,
      "logps/chosen": -0.786425769329071,
      "logps/rejected": -1.1472656726837158,
      "loss": 1.0342,
      "nll_loss": 0.989062488079071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07872314751148224,
      "rewards/margins": 0.03598785400390625,
      "rewards/rejected": -0.11469726264476776,
      "step": 3740
    },
    {
      "epoch": 0.14052575368645895,
      "grad_norm": 1.3637226052129898,
      "learning_rate": 1.3063945294843617e-06,
      "log_odds_chosen": 0.34376221895217896,
      "log_odds_ratio": -0.6439453363418579,
      "logits/chosen": -0.8121093511581421,
      "logits/rejected": -0.749218761920929,
      "logps/chosen": -0.7626953125,
      "logps/rejected": -1.001562476158142,
      "loss": 1.0367,
      "nll_loss": 1.0197265148162842,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07619629055261612,
      "rewards/margins": 0.02392578125,
      "rewards/rejected": -0.10019531100988388,
      "step": 3750
    },
    {
      "epoch": 0.14090048902962282,
      "grad_norm": 1.3780139844388228,
      "learning_rate": 1.3046561461068843e-06,
      "log_odds_chosen": 0.41398924589157104,
      "log_odds_ratio": -0.626757800579071,
      "logits/chosen": -0.787304699420929,
      "logits/rejected": -0.7339843511581421,
      "logps/chosen": -0.739453136920929,
      "logps/rejected": -1.0222656726837158,
      "loss": 1.0349,
      "nll_loss": 0.9443359375,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07402344048023224,
      "rewards/margins": 0.02824554406106472,
      "rewards/rejected": -0.10220947116613388,
      "step": 3760
    },
    {
      "epoch": 0.14127522437278672,
      "grad_norm": 1.5679056457796723,
      "learning_rate": 1.3029246839689124e-06,
      "log_odds_chosen": 0.3877197206020355,
      "log_odds_ratio": -0.6407226324081421,
      "logits/chosen": -0.7857421636581421,
      "logits/rejected": -0.741015613079071,
      "logps/chosen": -0.7554687261581421,
      "logps/rejected": -1.002343773841858,
      "loss": 1.0514,
      "nll_loss": 0.9800781011581421,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07552490383386612,
      "rewards/margins": 0.02465667761862278,
      "rewards/rejected": -0.10026855766773224,
      "step": 3770
    },
    {
      "epoch": 0.1416499597159506,
      "grad_norm": 1.4065088786543678,
      "learning_rate": 1.3012000972647109e-06,
      "log_odds_chosen": 0.26829832792282104,
      "log_odds_ratio": -0.675976574420929,
      "logits/chosen": -0.8558593988418579,
      "logits/rejected": -0.7744140625,
      "logps/chosen": -0.7099609375,
      "logps/rejected": -0.9068359136581421,
      "loss": 1.0328,
      "nll_loss": 0.9775390625,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.07099609076976776,
      "rewards/margins": 0.01963653601706028,
      "rewards/rejected": -0.09071044623851776,
      "step": 3780
    },
    {
      "epoch": 0.1420246950591145,
      "grad_norm": 1.4314878674053224,
      "learning_rate": 1.299482340611832e-06,
      "log_odds_chosen": 0.38641357421875,
      "log_odds_ratio": -0.6348632574081421,
      "logits/chosen": -0.808398425579071,
      "logits/rejected": -0.7420898675918579,
      "logps/chosen": -0.7572265863418579,
      "logps/rejected": -1.0027344226837158,
      "loss": 1.05,
      "nll_loss": 1.007421851158142,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07569579780101776,
      "rewards/margins": 0.0246124267578125,
      "rewards/rejected": -0.10023193061351776,
      "step": 3790
    },
    {
      "epoch": 0.1423994304022784,
      "grad_norm": 1.3920934217465544,
      "learning_rate": 1.2977713690461003e-06,
      "log_odds_chosen": 0.3974609375,
      "log_odds_ratio": -0.6275390386581421,
      "logits/chosen": -0.791210949420929,
      "logits/rejected": -0.719531238079071,
      "logps/chosen": -0.692675769329071,
      "logps/rejected": -0.947265625,
      "loss": 1.0697,
      "nll_loss": 1.012304663658142,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06934814155101776,
      "rewards/margins": 0.02535552904009819,
      "rewards/rejected": -0.0947265625,
      "step": 3800
    },
    {
      "epoch": 0.14277416574544227,
      "grad_norm": 1.477477497166206,
      "learning_rate": 1.296067138016669e-06,
      "log_odds_chosen": 0.5390625,
      "log_odds_ratio": -0.5892578363418579,
      "logits/chosen": -0.855273425579071,
      "logits/rejected": -0.775585949420929,
      "logps/chosen": -0.7113281488418579,
      "logps/rejected": -1.0732421875,
      "loss": 1.0215,
      "nll_loss": 0.942578136920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07113037258386612,
      "rewards/margins": 0.03623657301068306,
      "rewards/rejected": -0.10747070610523224,
      "step": 3810
    },
    {
      "epoch": 0.14314890108860617,
      "grad_norm": 1.4303849648548144,
      "learning_rate": 1.294369603381147e-06,
      "log_odds_chosen": 0.5321899652481079,
      "log_odds_ratio": -0.5751953125,
      "logits/chosen": -0.848437488079071,
      "logits/rejected": -0.746289074420929,
      "logps/chosen": -0.71533203125,
      "logps/rejected": -1.047460913658142,
      "loss": 1.0306,
      "nll_loss": 0.90625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07145996391773224,
      "rewards/margins": 0.03331603854894638,
      "rewards/rejected": -0.10469970852136612,
      "step": 3820
    },
    {
      "epoch": 0.14352363643177007,
      "grad_norm": 1.436553003694088,
      "learning_rate": 1.2926787214007981e-06,
      "log_odds_chosen": 0.39471435546875,
      "log_odds_ratio": -0.616894543170929,
      "logits/chosen": -0.839062511920929,
      "logits/rejected": -0.7671874761581421,
      "logps/chosen": -0.7173827886581421,
      "logps/rejected": -0.9712890386581421,
      "loss": 1.0312,
      "nll_loss": 0.954882800579071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07176513969898224,
      "rewards/margins": 0.02550811693072319,
      "rewards/rejected": -0.09721679985523224,
      "step": 3830
    },
    {
      "epoch": 0.14389837177493395,
      "grad_norm": 1.3891739407811645,
      "learning_rate": 1.2909944487358056e-06,
      "log_odds_chosen": 0.3313354551792145,
      "log_odds_ratio": -0.6533203125,
      "logits/chosen": -0.8935546875,
      "logits/rejected": -0.794726550579071,
      "logps/chosen": -0.7347656488418579,
      "logps/rejected": -0.9466797113418579,
      "loss": 1.0141,
      "nll_loss": 0.9496093988418579,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07343749701976776,
      "rewards/margins": 0.02115936204791069,
      "rewards/rejected": -0.09465332329273224,
      "step": 3840
    },
    {
      "epoch": 0.14427310711809785,
      "grad_norm": 1.3503365385730663,
      "learning_rate": 1.2893167424406084e-06,
      "log_odds_chosen": 0.2706054747104645,
      "log_odds_ratio": -0.670214831829071,
      "logits/chosen": -0.8404296636581421,
      "logits/rejected": -0.7953125238418579,
      "logps/chosen": -0.7162109613418579,
      "logps/rejected": -0.889453113079071,
      "loss": 1.0632,
      "nll_loss": 0.9292968511581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07160644233226776,
      "rewards/margins": 0.017458343878388405,
      "rewards/rejected": -0.08903808891773224,
      "step": 3850
    },
    {
      "epoch": 0.14464784246126172,
      "grad_norm": 1.3873512565568498,
      "learning_rate": 1.2876455599593008e-06,
      "log_odds_chosen": 0.3470458984375,
      "log_odds_ratio": -0.640625,
      "logits/chosen": -0.8412109613418579,
      "logits/rejected": -0.790234386920929,
      "logps/chosen": -0.7015625238418579,
      "logps/rejected": -0.921679675579071,
      "loss": 1.0355,
      "nll_loss": 0.9332031011581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07015381008386612,
      "rewards/margins": 0.02206573449075222,
      "rewards/rejected": -0.09217528998851776,
      "step": 3860
    },
    {
      "epoch": 0.14502257780442562,
      "grad_norm": 1.3983277765321478,
      "learning_rate": 1.285980859121099e-06,
      "log_odds_chosen": 0.5597289800643921,
      "log_odds_ratio": -0.5958007574081421,
      "logits/chosen": -0.806640625,
      "logits/rejected": -0.740429699420929,
      "logps/chosen": -0.725390613079071,
      "logps/rejected": -1.0945312976837158,
      "loss": 1.0448,
      "nll_loss": 1.029296875,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07258300483226776,
      "rewards/margins": 0.03683318942785263,
      "rewards/rejected": -0.10942383110523224,
      "step": 3870
    },
    {
      "epoch": 0.14539731314758952,
      "grad_norm": 1.4657296711992784,
      "learning_rate": 1.2843225981358712e-06,
      "log_odds_chosen": 0.37543946504592896,
      "log_odds_ratio": -0.6089843511581421,
      "logits/chosen": -0.840039074420929,
      "logits/rejected": -0.757031261920929,
      "logps/chosen": -0.6953125,
      "logps/rejected": -0.916796863079071,
      "loss": 1.0206,
      "nll_loss": 0.947265625,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06948242336511612,
      "rewards/margins": 0.02218170091509819,
      "rewards/rejected": -0.09169922024011612,
      "step": 3880
    },
    {
      "epoch": 0.1457720484907534,
      "grad_norm": 1.331098512484356,
      "learning_rate": 1.2826707355897317e-06,
      "log_odds_chosen": 0.58984375,
      "log_odds_ratio": -0.56591796875,
      "logits/chosen": -0.801074206829071,
      "logits/rejected": -0.6983398199081421,
      "logps/chosen": -0.687792956829071,
      "logps/rejected": -1.0857422351837158,
      "loss": 1.0291,
      "nll_loss": 1.015234351158142,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06875000149011612,
      "rewards/margins": 0.03988800197839737,
      "rewards/rejected": -0.10856933891773224,
      "step": 3890
    },
    {
      "epoch": 0.1461467838339173,
      "grad_norm": 1.4015714477829244,
      "learning_rate": 1.281025230440697e-06,
      "log_odds_chosen": 0.34552001953125,
      "log_odds_ratio": -0.6449218988418579,
      "logits/chosen": -0.814648449420929,
      "logits/rejected": -0.7554687261581421,
      "logps/chosen": -0.698437511920929,
      "logps/rejected": -0.926953136920929,
      "loss": 1.0416,
      "nll_loss": 0.958203136920929,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.06978759914636612,
      "rewards/margins": 0.022832488641142845,
      "rewards/rejected": -0.09267578274011612,
      "step": 3900
    },
    {
      "epoch": 0.1465215191770812,
      "grad_norm": 1.3914053328824945,
      "learning_rate": 1.2793860420144025e-06,
      "log_odds_chosen": 0.25152587890625,
      "log_odds_ratio": -0.6849609613418579,
      "logits/chosen": -0.820117175579071,
      "logits/rejected": -0.7880859375,
      "logps/chosen": -0.7796875238418579,
      "logps/rejected": -0.96484375,
      "loss": 1.0246,
      "nll_loss": 0.9986327886581421,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07795409858226776,
      "rewards/margins": 0.01848297193646431,
      "rewards/rejected": -0.09645996242761612,
      "step": 3910
    },
    {
      "epoch": 0.14689625452024507,
      "grad_norm": 1.450133020403644,
      "learning_rate": 1.2777531299998798e-06,
      "log_odds_chosen": 0.46965330839157104,
      "log_odds_ratio": -0.5923827886581421,
      "logits/chosen": -0.816210925579071,
      "logits/rejected": -0.744335949420929,
      "logps/chosen": -0.7330077886581421,
      "logps/rejected": -1.046289086341858,
      "loss": 1.0231,
      "nll_loss": 0.9248046875,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07322998344898224,
      "rewards/margins": 0.03132019191980362,
      "rewards/rejected": -0.10450439155101776,
      "step": 3920
    },
    {
      "epoch": 0.14727098986340897,
      "grad_norm": 1.3686811064706508,
      "learning_rate": 1.2761264544453928e-06,
      "log_odds_chosen": 0.531176745891571,
      "log_odds_ratio": -0.573925793170929,
      "logits/chosen": -0.8033202886581421,
      "logits/rejected": -0.6805664300918579,
      "logps/chosen": -0.6796875,
      "logps/rejected": -1.0291016101837158,
      "loss": 1.0189,
      "nll_loss": 0.9263671636581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06796874850988388,
      "rewards/margins": 0.03490753099322319,
      "rewards/rejected": -0.10295410454273224,
      "step": 3930
    },
    {
      "epoch": 0.14764572520657285,
      "grad_norm": 1.4531156143751387,
      "learning_rate": 1.2745059757543324e-06,
      "log_odds_chosen": 0.35895997285842896,
      "log_odds_ratio": -0.651074230670929,
      "logits/chosen": -0.818359375,
      "logits/rejected": -0.7548828125,
      "logps/chosen": -0.71875,
      "logps/rejected": -0.9638671875,
      "loss": 1.0172,
      "nll_loss": 0.9541015625,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0718994140625,
      "rewards/margins": 0.02456054650247097,
      "rewards/rejected": -0.09641113132238388,
      "step": 3940
    },
    {
      "epoch": 0.14802046054973675,
      "grad_norm": 1.445916425146402,
      "learning_rate": 1.272891654681168e-06,
      "log_odds_chosen": 0.582043468952179,
      "log_odds_ratio": -0.5801757574081421,
      "logits/chosen": -0.799023449420929,
      "logits/rejected": -0.7002929449081421,
      "logps/chosen": -0.717968761920929,
      "logps/rejected": -1.119531273841858,
      "loss": 1.0384,
      "nll_loss": 0.9849609136581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07174072414636612,
      "rewards/margins": 0.040076445788145065,
      "rewards/rejected": -0.11191406100988388,
      "step": 3950
    },
    {
      "epoch": 0.14839519589290065,
      "grad_norm": 1.3214176409382157,
      "learning_rate": 1.2712834523274563e-06,
      "log_odds_chosen": 0.4051757752895355,
      "log_odds_ratio": -0.6200195550918579,
      "logits/chosen": -0.796093761920929,
      "logits/rejected": -0.7337890863418579,
      "logps/chosen": -0.7216796875,
      "logps/rejected": -0.987109363079071,
      "loss": 1.0287,
      "nll_loss": 1.0115234851837158,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07215575873851776,
      "rewards/margins": 0.02661743201315403,
      "rewards/rejected": -0.09877929836511612,
      "step": 3960
    },
    {
      "epoch": 0.14876993123606452,
      "grad_norm": 1.531342020901493,
      "learning_rate": 1.2696813301379032e-06,
      "log_odds_chosen": 0.62158203125,
      "log_odds_ratio": -0.5443359613418579,
      "logits/chosen": -0.816210925579071,
      "logits/rejected": -0.7523437738418579,
      "logps/chosen": -0.692675769329071,
      "logps/rejected": -1.0908203125,
      "loss": 1.0226,
      "nll_loss": 0.9365234375,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06926269829273224,
      "rewards/margins": 0.03981933742761612,
      "rewards/rejected": -0.10905762016773224,
      "step": 3970
    },
    {
      "epoch": 0.14914466657922842,
      "grad_norm": 1.4063570286294333,
      "learning_rate": 1.2680852498964829e-06,
      "log_odds_chosen": 0.40526121854782104,
      "log_odds_ratio": -0.640625,
      "logits/chosen": -0.787109375,
      "logits/rejected": -0.704785168170929,
      "logps/chosen": -0.7476562261581421,
      "logps/rejected": -1.0437500476837158,
      "loss": 1.0072,
      "nll_loss": 0.957812488079071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07479248195886612,
      "rewards/margins": 0.02964477613568306,
      "rewards/rejected": -0.10441894829273224,
      "step": 3980
    },
    {
      "epoch": 0.1495194019223923,
      "grad_norm": 1.3913062093044695,
      "learning_rate": 1.266495173722607e-06,
      "log_odds_chosen": 0.3758789002895355,
      "log_odds_ratio": -0.611621081829071,
      "logits/chosen": -0.8130859136581421,
      "logits/rejected": -0.7822265625,
      "logps/chosen": -0.652539074420929,
      "logps/rejected": -0.884570300579071,
      "loss": 1.0319,
      "nll_loss": 0.9322265386581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06527099758386612,
      "rewards/margins": 0.02321472205221653,
      "rewards/rejected": -0.08847656100988388,
      "step": 3990
    },
    {
      "epoch": 0.1498941372655562,
      "grad_norm": 1.3062554895343845,
      "learning_rate": 1.2649110640673517e-06,
      "log_odds_chosen": 0.4199585020542145,
      "log_odds_ratio": -0.6205078363418579,
      "logits/chosen": -0.8128906488418579,
      "logits/rejected": -0.7548828125,
      "logps/chosen": -0.7083984613418579,
      "logps/rejected": -0.9990234375,
      "loss": 1.0359,
      "nll_loss": 0.9916015863418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07081298530101776,
      "rewards/margins": 0.02911376953125,
      "rewards/rejected": -0.09992675483226776,
      "step": 4000
    },
    {
      "epoch": 0.1502688726087201,
      "grad_norm": 1.503656491198069,
      "learning_rate": 1.2633328837097308e-06,
      "log_odds_chosen": 0.524792492389679,
      "log_odds_ratio": -0.583691418170929,
      "logits/chosen": -0.8177734613418579,
      "logits/rejected": -0.7318359613418579,
      "logps/chosen": -0.717968761920929,
      "logps/rejected": -1.053125023841858,
      "loss": 1.033,
      "nll_loss": 0.9554687738418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07185058295726776,
      "rewards/margins": 0.03347931057214737,
      "rewards/rejected": -0.10537109524011612,
      "step": 4010
    },
    {
      "epoch": 0.15064360795188397,
      "grad_norm": 1.450618875376577,
      "learning_rate": 1.2617605957530233e-06,
      "log_odds_chosen": 0.35588377714157104,
      "log_odds_ratio": -0.636035144329071,
      "logits/chosen": -0.845507800579071,
      "logits/rejected": -0.7738281488418579,
      "logps/chosen": -0.7669922113418579,
      "logps/rejected": -0.980273425579071,
      "loss": 1.0238,
      "nll_loss": 1.012304663658142,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07666015625,
      "rewards/margins": 0.02138366736471653,
      "rewards/rejected": -0.09808349609375,
      "step": 4020
    },
    {
      "epoch": 0.15101834329504787,
      "grad_norm": 1.4047574794892244,
      "learning_rate": 1.2601941636211516e-06,
      "log_odds_chosen": 0.47331541776657104,
      "log_odds_ratio": -0.623828113079071,
      "logits/chosen": -0.7982422113418579,
      "logits/rejected": -0.767578125,
      "logps/chosen": -0.719921886920929,
      "logps/rejected": -1.033789038658142,
      "loss": 1.0413,
      "nll_loss": 0.9283202886581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07199706882238388,
      "rewards/margins": 0.031330108642578125,
      "rewards/rejected": -0.10332031548023224,
      "step": 4030
    },
    {
      "epoch": 0.15139307863821178,
      "grad_norm": 1.33671985953357,
      "learning_rate": 1.2586335510551052e-06,
      "log_odds_chosen": 0.3323730528354645,
      "log_odds_ratio": -0.658203125,
      "logits/chosen": -0.797070324420929,
      "logits/rejected": -0.7353515625,
      "logps/chosen": -0.7470703125,
      "logps/rejected": -0.9794921875,
      "loss": 1.0127,
      "nll_loss": 1.004492163658142,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07470703125,
      "rewards/margins": 0.02324066124856472,
      "rewards/rejected": -0.097900390625,
      "step": 4040
    },
    {
      "epoch": 0.15176781398137565,
      "grad_norm": 1.429944760729478,
      "learning_rate": 1.2570787221094177e-06,
      "log_odds_chosen": 0.41120606660842896,
      "log_odds_ratio": -0.620410144329071,
      "logits/chosen": -0.850390613079071,
      "logits/rejected": -0.771289050579071,
      "logps/chosen": -0.698535144329071,
      "logps/rejected": -0.973828136920929,
      "loss": 1.0323,
      "nll_loss": 0.9498046636581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06988525390625,
      "rewards/margins": 0.02749175950884819,
      "rewards/rejected": -0.09733887016773224,
      "step": 4050
    },
    {
      "epoch": 0.15214254932453955,
      "grad_norm": 1.4797487401228786,
      "learning_rate": 1.255529641148689e-06,
      "log_odds_chosen": 0.57745361328125,
      "log_odds_ratio": -0.572949230670929,
      "logits/chosen": -0.8677734136581421,
      "logits/rejected": -0.7875000238418579,
      "logps/chosen": -0.7171875238418579,
      "logps/rejected": -1.108789086341858,
      "loss": 1.0129,
      "nll_loss": 0.944140613079071,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07172851264476776,
      "rewards/margins": 0.03914032131433487,
      "rewards/rejected": -0.11083984375,
      "step": 4060
    },
    {
      "epoch": 0.15251728466770342,
      "grad_norm": 1.4311394224604588,
      "learning_rate": 1.2539862728441536e-06,
      "log_odds_chosen": 0.4922729432582855,
      "log_odds_ratio": -0.5782226324081421,
      "logits/chosen": -0.7935546636581421,
      "logits/rejected": -0.7376953363418579,
      "logps/chosen": -0.6680663824081421,
      "logps/rejected": -0.976367175579071,
      "loss": 1.0261,
      "nll_loss": 0.9488281011581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06680908054113388,
      "rewards/margins": 0.03090209886431694,
      "rewards/rejected": -0.09771728515625,
      "step": 4070
    },
    {
      "epoch": 0.15289202001086732,
      "grad_norm": 1.4494925910300667,
      "learning_rate": 1.252448582170299e-06,
      "log_odds_chosen": 0.31135863065719604,
      "log_odds_ratio": -0.674023449420929,
      "logits/chosen": -0.8720703125,
      "logits/rejected": -0.7982422113418579,
      "logps/chosen": -0.7339843511581421,
      "logps/rejected": -0.9595702886581421,
      "loss": 1.0292,
      "nll_loss": 0.926953136920929,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07341308891773224,
      "rewards/margins": 0.02253570593893528,
      "rewards/rejected": -0.09593506157398224,
      "step": 4080
    },
    {
      "epoch": 0.15326675535403123,
      "grad_norm": 1.4099266850431709,
      "learning_rate": 1.2509165344015243e-06,
      "log_odds_chosen": 0.538317859172821,
      "log_odds_ratio": -0.56982421875,
      "logits/chosen": -0.805859386920929,
      "logits/rejected": -0.7328125238418579,
      "logps/chosen": -0.7251952886581421,
      "logps/rejected": -1.085351586341858,
      "loss": 1.0167,
      "nll_loss": 0.9488281011581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07247314602136612,
      "rewards/margins": 0.03599701076745987,
      "rewards/rejected": -0.1085205078125,
      "step": 4090
    },
    {
      "epoch": 0.1536414906971951,
      "grad_norm": 1.6340485782035048,
      "learning_rate": 1.2493900951088486e-06,
      "log_odds_chosen": 0.41986083984375,
      "log_odds_ratio": -0.606249988079071,
      "logits/chosen": -0.817578136920929,
      "logits/rejected": -0.74951171875,
      "logps/chosen": -0.7738281488418579,
      "logps/rejected": -1.047460913658142,
      "loss": 1.0309,
      "nll_loss": 0.966015636920929,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07740478217601776,
      "rewards/margins": 0.02736205980181694,
      "rewards/rejected": -0.10476074367761612,
      "step": 4100
    },
    {
      "epoch": 0.154016226040359,
      "grad_norm": 1.4656923659130283,
      "learning_rate": 1.2478692301566601e-06,
      "log_odds_chosen": 0.4156127870082855,
      "log_odds_ratio": -0.611132800579071,
      "logits/chosen": -0.749804675579071,
      "logits/rejected": -0.706835925579071,
      "logps/chosen": -0.710156261920929,
      "logps/rejected": -0.9775390625,
      "loss": 1.0224,
      "nll_loss": 0.9498046636581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07100830227136612,
      "rewards/margins": 0.026702880859375,
      "rewards/rejected": -0.09770508110523224,
      "step": 4110
    },
    {
      "epoch": 0.15439096138352287,
      "grad_norm": 1.548273142430971,
      "learning_rate": 1.2463539056995116e-06,
      "log_odds_chosen": 0.4007812440395355,
      "log_odds_ratio": -0.613476574420929,
      "logits/chosen": -0.8130859136581421,
      "logits/rejected": -0.740429699420929,
      "logps/chosen": -0.697265625,
      "logps/rejected": -0.967968761920929,
      "loss": 1.0398,
      "nll_loss": 1.0232422351837158,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06967773288488388,
      "rewards/margins": 0.027062226086854935,
      "rewards/rejected": -0.0968017578125,
      "step": 4120
    },
    {
      "epoch": 0.15476569672668677,
      "grad_norm": 1.4878154227127165,
      "learning_rate": 1.2448440881789541e-06,
      "log_odds_chosen": 0.3198486268520355,
      "log_odds_ratio": -0.64306640625,
      "logits/chosen": -0.806640625,
      "logits/rejected": -0.7320312261581421,
      "logps/chosen": -0.7447265386581421,
      "logps/rejected": -0.9632812738418579,
      "loss": 1.0188,
      "nll_loss": 0.9521484375,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07454834133386612,
      "rewards/margins": 0.021808624267578125,
      "rewards/rejected": -0.09633789211511612,
      "step": 4130
    },
    {
      "epoch": 0.15514043206985068,
      "grad_norm": 1.457380797769298,
      "learning_rate": 1.2433397443204184e-06,
      "log_odds_chosen": 0.4485229551792145,
      "log_odds_ratio": -0.616406261920929,
      "logits/chosen": -0.8257812261581421,
      "logits/rejected": -0.7265625,
      "logps/chosen": -0.707812488079071,
      "logps/rejected": -1.0080077648162842,
      "loss": 1.0256,
      "nll_loss": 0.903515636920929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07077636569738388,
      "rewards/margins": 0.03001708909869194,
      "rewards/rejected": -0.10087890923023224,
      "step": 4140
    },
    {
      "epoch": 0.15551516741301455,
      "grad_norm": 1.411033538218913,
      "learning_rate": 1.2418408411301324e-06,
      "log_odds_chosen": 0.38029783964157104,
      "log_odds_ratio": -0.6034179925918579,
      "logits/chosen": -0.853320300579071,
      "logits/rejected": -0.7669922113418579,
      "logps/chosen": -0.7533203363418579,
      "logps/rejected": -0.990039050579071,
      "loss": 1.024,
      "nll_loss": 0.9300781488418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07528076320886612,
      "rewards/margins": 0.02369537390768528,
      "rewards/rejected": -0.09904785454273224,
      "step": 4150
    },
    {
      "epoch": 0.15588990275617845,
      "grad_norm": 1.4401765262310855,
      "learning_rate": 1.2403473458920844e-06,
      "log_odds_chosen": 0.30217283964157104,
      "log_odds_ratio": -0.6473633050918579,
      "logits/chosen": -0.8324218988418579,
      "logits/rejected": -0.767773449420929,
      "logps/chosen": -0.7255859375,
      "logps/rejected": -0.9066406488418579,
      "loss": 1.036,
      "nll_loss": 1.0087890625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07259521633386612,
      "rewards/margins": 0.01812286302447319,
      "rewards/rejected": -0.09067382663488388,
      "step": 4160
    },
    {
      "epoch": 0.15626463809934235,
      "grad_norm": 1.4252322641096642,
      "learning_rate": 1.2388592261650217e-06,
      "log_odds_chosen": 0.29399412870407104,
      "log_odds_ratio": -0.68017578125,
      "logits/chosen": -0.783398449420929,
      "logits/rejected": -0.7564452886581421,
      "logps/chosen": -0.7579101324081421,
      "logps/rejected": -0.938671886920929,
      "loss": 1.051,
      "nll_loss": 1.0029296875,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.07575683295726776,
      "rewards/margins": 0.018076324835419655,
      "rewards/rejected": -0.09381103515625,
      "step": 4170
    },
    {
      "epoch": 0.15663937344250622,
      "grad_norm": 1.3986464312942886,
      "learning_rate": 1.2373764497794918e-06,
      "log_odds_chosen": 0.39484864473342896,
      "log_odds_ratio": -0.6143554449081421,
      "logits/chosen": -0.744921863079071,
      "logits/rejected": -0.6898437738418579,
      "logps/chosen": -0.720507800579071,
      "logps/rejected": -0.9888671636581421,
      "loss": 1.0053,
      "nll_loss": 0.96484375,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07203368842601776,
      "rewards/margins": 0.026815032586455345,
      "rewards/rejected": -0.09890136867761612,
      "step": 4180
    },
    {
      "epoch": 0.15701410878567013,
      "grad_norm": 1.3946464614037555,
      "learning_rate": 1.2358989848349217e-06,
      "log_odds_chosen": 0.546887218952179,
      "log_odds_ratio": -0.57080078125,
      "logits/chosen": -0.8177734613418579,
      "logits/rejected": -0.720898449420929,
      "logps/chosen": -0.7203124761581421,
      "logps/rejected": -1.0810546875,
      "loss": 1.021,
      "nll_loss": 0.9140625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07194824516773224,
      "rewards/margins": 0.036144256591796875,
      "rewards/rejected": -0.1080322265625,
      "step": 4190
    },
    {
      "epoch": 0.157388844128834,
      "grad_norm": 1.4480920844510548,
      "learning_rate": 1.2344267996967353e-06,
      "log_odds_chosen": 0.38701170682907104,
      "log_odds_ratio": -0.619140625,
      "logits/chosen": -0.811328113079071,
      "logits/rejected": -0.75341796875,
      "logps/chosen": -0.68994140625,
      "logps/rejected": -0.932421863079071,
      "loss": 1.0077,
      "nll_loss": 0.8998047113418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06904296576976776,
      "rewards/margins": 0.024167632684111595,
      "rewards/rejected": -0.09323730319738388,
      "step": 4200
    },
    {
      "epoch": 0.1577635794719979,
      "grad_norm": 1.5551205714396803,
      "learning_rate": 1.2329598629935076e-06,
      "log_odds_chosen": 0.38878172636032104,
      "log_odds_ratio": -0.6416991949081421,
      "logits/chosen": -0.829296886920929,
      "logits/rejected": -0.732421875,
      "logps/chosen": -0.754687488079071,
      "logps/rejected": -1.020117163658142,
      "loss": 1.0223,
      "nll_loss": 0.944531261920929,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07546386867761612,
      "rewards/margins": 0.02654266357421875,
      "rewards/rejected": -0.10198974609375,
      "step": 4210
    },
    {
      "epoch": 0.1581383148151618,
      "grad_norm": 1.338923810350434,
      "learning_rate": 1.2314981436141583e-06,
      "log_odds_chosen": 0.22910156846046448,
      "log_odds_ratio": -0.6957031488418579,
      "logits/chosen": -0.8099609613418579,
      "logits/rejected": -0.7548828125,
      "logps/chosen": -0.7314453125,
      "logps/rejected": -0.888867199420929,
      "loss": 1.0301,
      "nll_loss": 0.9585937261581421,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.07315673679113388,
      "rewards/margins": 0.015772247686982155,
      "rewards/rejected": -0.08885498344898224,
      "step": 4220
    },
    {
      "epoch": 0.15851305015832567,
      "grad_norm": 1.67286370992021,
      "learning_rate": 1.2300416107051802e-06,
      "log_odds_chosen": 0.41254884004592896,
      "log_odds_ratio": -0.6371093988418579,
      "logits/chosen": -0.800585925579071,
      "logits/rejected": -0.7359374761581421,
      "logps/chosen": -0.7386718988418579,
      "logps/rejected": -1.0187499523162842,
      "loss": 1.0253,
      "nll_loss": 0.9175781011581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0738525390625,
      "rewards/margins": 0.027997588738799095,
      "rewards/rejected": -0.10185547173023224,
      "step": 4230
    },
    {
      "epoch": 0.15888778550148958,
      "grad_norm": 1.4252504958588472,
      "learning_rate": 1.2285902336679024e-06,
      "log_odds_chosen": 0.44188231229782104,
      "log_odds_ratio": -0.616406261920929,
      "logits/chosen": -0.828906238079071,
      "logits/rejected": -0.7894531488418579,
      "logps/chosen": -0.7406250238418579,
      "logps/rejected": -1.0265624523162842,
      "loss": 1.0255,
      "nll_loss": 0.901562511920929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07407226413488388,
      "rewards/margins": 0.02864837646484375,
      "rewards/rejected": -0.10275878757238388,
      "step": 4240
    },
    {
      "epoch": 0.15926252084465348,
      "grad_norm": 1.415161756635946,
      "learning_rate": 1.2271439821557926e-06,
      "log_odds_chosen": 0.39368897676467896,
      "log_odds_ratio": -0.6219726800918579,
      "logits/chosen": -0.8013671636581421,
      "logits/rejected": -0.731152355670929,
      "logps/chosen": -0.731640636920929,
      "logps/rejected": -0.9927734136581421,
      "loss": 1.0292,
      "nll_loss": 0.934374988079071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07316894829273224,
      "rewards/margins": 0.026184845715761185,
      "rewards/rejected": -0.09938964992761612,
      "step": 4250
    },
    {
      "epoch": 0.15963725618781735,
      "grad_norm": 1.3557801398031177,
      "learning_rate": 1.225702826071791e-06,
      "log_odds_chosen": 0.45721435546875,
      "log_odds_ratio": -0.5658203363418579,
      "logits/chosen": -0.811328113079071,
      "logits/rejected": -0.7269531488418579,
      "logps/chosen": -0.658203125,
      "logps/rejected": -0.9423828125,
      "loss": 1.0246,
      "nll_loss": 0.9419921636581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06578369438648224,
      "rewards/margins": 0.02843017503619194,
      "rewards/rejected": -0.09423828125,
      "step": 4260
    },
    {
      "epoch": 0.16001199153098125,
      "grad_norm": 1.445900753309891,
      "learning_rate": 1.2242667355656797e-06,
      "log_odds_chosen": 0.41064453125,
      "log_odds_ratio": -0.6142578125,
      "logits/chosen": -0.8355468511581421,
      "logits/rejected": -0.7447265386581421,
      "logps/chosen": -0.7124999761581421,
      "logps/rejected": -0.980761706829071,
      "loss": 1.0339,
      "nll_loss": 0.9917968511581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07126464694738388,
      "rewards/margins": 0.02683410607278347,
      "rewards/rejected": -0.09804687649011612,
      "step": 4270
    },
    {
      "epoch": 0.16038672687414512,
      "grad_norm": 1.6378779771552119,
      "learning_rate": 1.2228356810314862e-06,
      "log_odds_chosen": 0.4164794981479645,
      "log_odds_ratio": -0.6226562261581421,
      "logits/chosen": -0.8271484375,
      "logits/rejected": -0.7310546636581421,
      "logps/chosen": -0.681445300579071,
      "logps/rejected": -0.9595702886581421,
      "loss": 1.0345,
      "nll_loss": 0.977734386920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06809081882238388,
      "rewards/margins": 0.027799606323242188,
      "rewards/rejected": -0.095947265625,
      "step": 4280
    },
    {
      "epoch": 0.16076146221730903,
      "grad_norm": 1.4063535505299827,
      "learning_rate": 1.2214096331049186e-06,
      "log_odds_chosen": 0.503063976764679,
      "log_odds_ratio": -0.600781261920929,
      "logits/chosen": -0.860546886920929,
      "logits/rejected": -0.728320300579071,
      "logps/chosen": -0.706738293170929,
      "logps/rejected": -1.04296875,
      "loss": 1.0461,
      "nll_loss": 0.9732421636581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07072754204273224,
      "rewards/margins": 0.03354949876666069,
      "rewards/rejected": -0.10423584282398224,
      "step": 4290
    },
    {
      "epoch": 0.16113619756047293,
      "grad_norm": 1.317558780986736,
      "learning_rate": 1.2199885626608373e-06,
      "log_odds_chosen": 0.3826049864292145,
      "log_odds_ratio": -0.624218761920929,
      "logits/chosen": -0.760937511920929,
      "logits/rejected": -0.71435546875,
      "logps/chosen": -0.732617199420929,
      "logps/rejected": -0.979296863079071,
      "loss": 1.0379,
      "nll_loss": 1.017187476158142,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07330322265625,
      "rewards/margins": 0.02462616004049778,
      "rewards/rejected": -0.09794922173023224,
      "step": 4300
    },
    {
      "epoch": 0.1615109329036368,
      "grad_norm": 1.4092027108767469,
      "learning_rate": 1.2185724408107546e-06,
      "log_odds_chosen": 0.3385376036167145,
      "log_odds_ratio": -0.640625,
      "logits/chosen": -0.7964843511581421,
      "logits/rejected": -0.7314453125,
      "logps/chosen": -0.697070300579071,
      "logps/rejected": -0.9193359613418579,
      "loss": 0.9966,
      "nll_loss": 0.8941406011581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06967773288488388,
      "rewards/margins": 0.0222930908203125,
      "rewards/rejected": -0.09199218451976776,
      "step": 4310
    },
    {
      "epoch": 0.1618856682468007,
      "grad_norm": 1.4520639056947278,
      "learning_rate": 1.2171612389003689e-06,
      "log_odds_chosen": 0.4250244200229645,
      "log_odds_ratio": -0.611523449420929,
      "logits/chosen": -0.825976550579071,
      "logits/rejected": -0.707714855670929,
      "logps/chosen": -0.6898437738418579,
      "logps/rejected": -0.987109363079071,
      "loss": 1.0229,
      "nll_loss": 0.932421863079071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06900634616613388,
      "rewards/margins": 0.02971496619284153,
      "rewards/rejected": -0.09866943210363388,
      "step": 4320
    },
    {
      "epoch": 0.16226040358996457,
      "grad_norm": 1.4119052372940457,
      "learning_rate": 1.2157549285071297e-06,
      "log_odds_chosen": 0.48304444551467896,
      "log_odds_ratio": -0.6263672113418579,
      "logits/chosen": -0.785351574420929,
      "logits/rejected": -0.723828136920929,
      "logps/chosen": -0.736523449420929,
      "logps/rejected": -1.0662109851837158,
      "loss": 1.0284,
      "nll_loss": 0.96875,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.0736083984375,
      "rewards/margins": 0.03296051174402237,
      "rewards/rejected": -0.10651855170726776,
      "step": 4330
    },
    {
      "epoch": 0.16263513893312848,
      "grad_norm": 1.4728551968502608,
      "learning_rate": 1.2143534814378327e-06,
      "log_odds_chosen": 0.49494630098342896,
      "log_odds_ratio": -0.5948241949081421,
      "logits/chosen": -0.858593761920929,
      "logits/rejected": -0.764453113079071,
      "logps/chosen": -0.684374988079071,
      "logps/rejected": -1.0251953601837158,
      "loss": 1.0089,
      "nll_loss": 0.88671875,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0684814453125,
      "rewards/margins": 0.034113310277462006,
      "rewards/rejected": -0.1025390625,
      "step": 4340
    },
    {
      "epoch": 0.16300987427629238,
      "grad_norm": 1.4191928848799757,
      "learning_rate": 1.2129568697262454e-06,
      "log_odds_chosen": 0.4547119140625,
      "log_odds_ratio": -0.596875011920929,
      "logits/chosen": -0.824023425579071,
      "logits/rejected": -0.715527355670929,
      "logps/chosen": -0.692187488079071,
      "logps/rejected": -1.001367211341858,
      "loss": 1.0272,
      "nll_loss": 0.9166015386581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06927490234375,
      "rewards/margins": 0.03086090087890625,
      "rewards/rejected": -0.10008545219898224,
      "step": 4350
    },
    {
      "epoch": 0.16338460961945625,
      "grad_norm": 1.3004433503039274,
      "learning_rate": 1.2115650656307653e-06,
      "log_odds_chosen": 0.43574219942092896,
      "log_odds_ratio": -0.626757800579071,
      "logits/chosen": -0.7925781011581421,
      "logits/rejected": -0.680371105670929,
      "logps/chosen": -0.7259765863418579,
      "logps/rejected": -1.0310547351837158,
      "loss": 1.0077,
      "nll_loss": 0.971875011920929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07254638522863388,
      "rewards/margins": 0.03049621544778347,
      "rewards/rejected": -0.10317382961511612,
      "step": 4360
    },
    {
      "epoch": 0.16375934496262015,
      "grad_norm": 1.5864684467914232,
      "learning_rate": 1.210178041632103e-06,
      "log_odds_chosen": 0.32684326171875,
      "log_odds_ratio": -0.640625,
      "logits/chosen": -0.866015613079071,
      "logits/rejected": -0.767578125,
      "logps/chosen": -0.766406238079071,
      "logps/rejected": -0.9583984613418579,
      "loss": 1.017,
      "nll_loss": 0.964062511920929,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07658691704273224,
      "rewards/margins": 0.01932983472943306,
      "rewards/rejected": -0.09589843451976776,
      "step": 4370
    },
    {
      "epoch": 0.16413408030578405,
      "grad_norm": 1.5197993186981218,
      "learning_rate": 1.2087957704309988e-06,
      "log_odds_chosen": 0.503857433795929,
      "log_odds_ratio": -0.5967773199081421,
      "logits/chosen": -0.814257800579071,
      "logits/rejected": -0.758984386920929,
      "logps/chosen": -0.664257824420929,
      "logps/rejected": -0.986328125,
      "loss": 1.0144,
      "nll_loss": 0.9017578363418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06643066555261612,
      "rewards/margins": 0.03220520168542862,
      "rewards/rejected": -0.09865722805261612,
      "step": 4380
    },
    {
      "epoch": 0.16450881564894793,
      "grad_norm": 1.3159733444256776,
      "learning_rate": 1.2074182249459642e-06,
      "log_odds_chosen": 0.45732420682907104,
      "log_odds_ratio": -0.5873047113418579,
      "logits/chosen": -0.8150390386581421,
      "logits/rejected": -0.75,
      "logps/chosen": -0.6880859136581421,
      "logps/rejected": -1.001562476158142,
      "loss": 1.017,
      "nll_loss": 0.949023425579071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06885986030101776,
      "rewards/margins": 0.03126983717083931,
      "rewards/rejected": -0.10009765625,
      "step": 4390
    },
    {
      "epoch": 0.16488355099211183,
      "grad_norm": 1.4711815681326021,
      "learning_rate": 1.2060453783110545e-06,
      "log_odds_chosen": 0.3551025390625,
      "log_odds_ratio": -0.6407226324081421,
      "logits/chosen": -0.8333984613418579,
      "logits/rejected": -0.7582031488418579,
      "logps/chosen": -0.709179699420929,
      "logps/rejected": -0.9609375,
      "loss": 1.0152,
      "nll_loss": 0.905078113079071,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07093505561351776,
      "rewards/margins": 0.02520599402487278,
      "rewards/rejected": -0.09614257514476776,
      "step": 4400
    },
    {
      "epoch": 0.1652582863352757,
      "grad_norm": 1.2954841156723436,
      "learning_rate": 1.2046772038736682e-06,
      "log_odds_chosen": 0.28795164823532104,
      "log_odds_ratio": -0.66796875,
      "logits/chosen": -0.8031250238418579,
      "logits/rejected": -0.731249988079071,
      "logps/chosen": -0.759570300579071,
      "logps/rejected": -0.9677734375,
      "loss": 1.0181,
      "nll_loss": 1.034570336341858,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07598876953125,
      "rewards/margins": 0.020782470703125,
      "rewards/rejected": -0.09672851860523224,
      "step": 4410
    },
    {
      "epoch": 0.1656330216784396,
      "grad_norm": 1.4120870780888737,
      "learning_rate": 1.2033136751923736e-06,
      "log_odds_chosen": 0.3365234434604645,
      "log_odds_ratio": -0.644238293170929,
      "logits/chosen": -0.813671886920929,
      "logits/rejected": -0.7220703363418579,
      "logps/chosen": -0.745312511920929,
      "logps/rejected": -0.970898449420929,
      "loss": 1.0396,
      "nll_loss": 0.9712890386581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07453612983226776,
      "rewards/margins": 0.02257232740521431,
      "rewards/rejected": -0.09707031399011612,
      "step": 4420
    },
    {
      "epoch": 0.1660077570216035,
      "grad_norm": 1.4109765959062353,
      "learning_rate": 1.201954766034762e-06,
      "log_odds_chosen": 0.5154784917831421,
      "log_odds_ratio": -0.5794922113418579,
      "logits/chosen": -0.8099609613418579,
      "logits/rejected": -0.7528320550918579,
      "logps/chosen": -0.677929699420929,
      "logps/rejected": -1.016015648841858,
      "loss": 1.0087,
      "nll_loss": 0.9283202886581421,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06784667819738388,
      "rewards/margins": 0.03374633938074112,
      "rewards/rejected": -0.10161133110523224,
      "step": 4430
    },
    {
      "epoch": 0.16638249236476738,
      "grad_norm": 1.460206190197764,
      "learning_rate": 1.2006004503753285e-06,
      "log_odds_chosen": 0.3975463807582855,
      "log_odds_ratio": -0.6373046636581421,
      "logits/chosen": -0.8671875,
      "logits/rejected": -0.786328136920929,
      "logps/chosen": -0.7457031011581421,
      "logps/rejected": -0.9916015863418579,
      "loss": 1.0143,
      "nll_loss": 0.978515625,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07454834133386612,
      "rewards/margins": 0.02460784837603569,
      "rewards/rejected": -0.09916992485523224,
      "step": 4440
    },
    {
      "epoch": 0.16675722770793128,
      "grad_norm": 1.4876881695441841,
      "learning_rate": 1.1992507023933782e-06,
      "log_odds_chosen": 0.3500122129917145,
      "log_odds_ratio": -0.6494140625,
      "logits/chosen": -0.824023425579071,
      "logits/rejected": -0.782421886920929,
      "logps/chosen": -0.738476574420929,
      "logps/rejected": -0.9966796636581421,
      "loss": 1.0152,
      "nll_loss": 0.9189453125,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07382812350988388,
      "rewards/margins": 0.025858307257294655,
      "rewards/rejected": -0.09971924126148224,
      "step": 4450
    },
    {
      "epoch": 0.16713196305109515,
      "grad_norm": 1.3997293229870738,
      "learning_rate": 1.1979054964709597e-06,
      "log_odds_chosen": 0.38037109375,
      "log_odds_ratio": -0.6250976324081421,
      "logits/chosen": -0.8128906488418579,
      "logits/rejected": -0.745312511920929,
      "logps/chosen": -0.694140613079071,
      "logps/rejected": -0.9417968988418579,
      "loss": 1.0136,
      "nll_loss": 0.873046875,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.06943359225988388,
      "rewards/margins": 0.024688720703125,
      "rewards/rejected": -0.09405517578125,
      "step": 4460
    },
    {
      "epoch": 0.16750669839425905,
      "grad_norm": 1.403703306786951,
      "learning_rate": 1.1965648071908207e-06,
      "log_odds_chosen": 0.3755859434604645,
      "log_odds_ratio": -0.624218761920929,
      "logits/chosen": -0.834765613079071,
      "logits/rejected": -0.755664050579071,
      "logps/chosen": -0.748242199420929,
      "logps/rejected": -1.0080077648162842,
      "loss": 1.0183,
      "nll_loss": 0.9593750238418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07487793266773224,
      "rewards/margins": 0.02600708045065403,
      "rewards/rejected": -0.10084228217601776,
      "step": 4470
    },
    {
      "epoch": 0.16788143373742295,
      "grad_norm": 1.5039625709355833,
      "learning_rate": 1.1952286093343935e-06,
      "log_odds_chosen": 0.36357420682907104,
      "log_odds_ratio": -0.620410144329071,
      "logits/chosen": -0.832812488079071,
      "logits/rejected": -0.7689453363418579,
      "logps/chosen": -0.7266601324081421,
      "logps/rejected": -0.9619140625,
      "loss": 1.0254,
      "nll_loss": 0.939453125,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07265625149011612,
      "rewards/margins": 0.02350769005715847,
      "rewards/rejected": -0.09616699069738388,
      "step": 4480
    },
    {
      "epoch": 0.16825616908058683,
      "grad_norm": 1.4023993174774243,
      "learning_rate": 1.1938968778798005e-06,
      "log_odds_chosen": 0.3463378846645355,
      "log_odds_ratio": -0.64697265625,
      "logits/chosen": -0.7900390625,
      "logits/rejected": -0.73828125,
      "logps/chosen": -0.7066406011581421,
      "logps/rejected": -0.9214843511581421,
      "loss": 1.0235,
      "nll_loss": 0.9609375,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07062987983226776,
      "rewards/margins": 0.02149658277630806,
      "rewards/rejected": -0.0921630859375,
      "step": 4490
    },
    {
      "epoch": 0.16863090442375073,
      "grad_norm": 1.5066632780075462,
      "learning_rate": 1.1925695879998878e-06,
      "log_odds_chosen": 0.38096922636032104,
      "log_odds_ratio": -0.644726574420929,
      "logits/chosen": -0.793749988079071,
      "logits/rejected": -0.704296886920929,
      "logps/chosen": -0.7158203125,
      "logps/rejected": -0.9873046875,
      "loss": 1.0457,
      "nll_loss": 0.9478515386581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07161865383386612,
      "rewards/margins": 0.02714996412396431,
      "rewards/rejected": -0.09873046725988388,
      "step": 4500
    },
    {
      "epoch": 0.16900563976691463,
      "grad_norm": 1.376921660399924,
      "learning_rate": 1.1912467150602794e-06,
      "log_odds_chosen": 0.42756348848342896,
      "log_odds_ratio": -0.602343738079071,
      "logits/chosen": -0.773242175579071,
      "logits/rejected": -0.707226574420929,
      "logps/chosen": -0.65234375,
      "logps/rejected": -0.9234374761581421,
      "loss": 1.0008,
      "nll_loss": 0.908984363079071,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06523437798023224,
      "rewards/margins": 0.02714691124856472,
      "rewards/rejected": -0.09234619140625,
      "step": 4510
    },
    {
      "epoch": 0.1693803751100785,
      "grad_norm": 1.34671724322411,
      "learning_rate": 1.189928234617459e-06,
      "log_odds_chosen": 0.3268066346645355,
      "log_odds_ratio": -0.6314452886581421,
      "logits/chosen": -0.7972656488418579,
      "logits/rejected": -0.722851574420929,
      "logps/chosen": -0.708203136920929,
      "logps/rejected": -0.922070324420929,
      "loss": 1.0222,
      "nll_loss": 0.9476562738418579,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07087402045726776,
      "rewards/margins": 0.02139587327837944,
      "rewards/rejected": -0.09223632514476776,
      "step": 4520
    },
    {
      "epoch": 0.1697551104532424,
      "grad_norm": 1.5002640720431595,
      "learning_rate": 1.1886141224168716e-06,
      "log_odds_chosen": 0.531384289264679,
      "log_odds_ratio": -0.5887695550918579,
      "logits/chosen": -0.8121093511581421,
      "logits/rejected": -0.743945300579071,
      "logps/chosen": -0.723437488079071,
      "logps/rejected": -1.096289038658142,
      "loss": 1.0205,
      "nll_loss": 0.9491211175918579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07237549126148224,
      "rewards/margins": 0.03720702975988388,
      "rewards/rejected": -0.10959472507238388,
      "step": 4530
    },
    {
      "epoch": 0.17012984579640628,
      "grad_norm": 1.2917637855489457,
      "learning_rate": 1.1873043543910495e-06,
      "log_odds_chosen": 0.552014172077179,
      "log_odds_ratio": -0.5595703125,
      "logits/chosen": -0.785937488079071,
      "logits/rejected": -0.724414050579071,
      "logps/chosen": -0.673828125,
      "logps/rejected": -1.016210913658142,
      "loss": 0.9963,
      "nll_loss": 0.8929687738418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06743164360523224,
      "rewards/margins": 0.03418578952550888,
      "rewards/rejected": -0.10166015475988388,
      "step": 4540
    },
    {
      "epoch": 0.17050458113957018,
      "grad_norm": 1.450324213312085,
      "learning_rate": 1.1859989066577617e-06,
      "log_odds_chosen": 0.4853515625,
      "log_odds_ratio": -0.5799804925918579,
      "logits/chosen": -0.778515636920929,
      "logits/rejected": -0.735156238079071,
      "logps/chosen": -0.701171875,
      "logps/rejected": -1.0164062976837158,
      "loss": 1.0048,
      "nll_loss": 0.8935546875,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07004394382238388,
      "rewards/margins": 0.03153533861041069,
      "rewards/rejected": -0.10152588039636612,
      "step": 4550
    },
    {
      "epoch": 0.17087931648273408,
      "grad_norm": 1.3014332250325684,
      "learning_rate": 1.1846977555181846e-06,
      "log_odds_chosen": 0.503039538860321,
      "log_odds_ratio": -0.587109386920929,
      "logits/chosen": -0.7875000238418579,
      "logits/rejected": -0.7212890386581421,
      "logps/chosen": -0.6845703125,
      "logps/rejected": -1.0242187976837158,
      "loss": 1.0195,
      "nll_loss": 0.916015625,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.0684814453125,
      "rewards/margins": 0.0339202880859375,
      "rewards/rejected": -0.10255126655101776,
      "step": 4560
    },
    {
      "epoch": 0.17125405182589795,
      "grad_norm": 1.3515529055145639,
      "learning_rate": 1.1834008774550946e-06,
      "log_odds_chosen": 0.704821765422821,
      "log_odds_ratio": -0.5079101324081421,
      "logits/chosen": -0.8275390863418579,
      "logits/rejected": -0.7281249761581421,
      "logps/chosen": -0.71875,
      "logps/rejected": -1.190820336341858,
      "loss": 1.0128,
      "nll_loss": 0.947460949420929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07191161811351776,
      "rewards/margins": 0.04720153659582138,
      "rewards/rejected": -0.11906738579273224,
      "step": 4570
    },
    {
      "epoch": 0.17162878716906185,
      "grad_norm": 1.4608817087282855,
      "learning_rate": 1.1821082491310835e-06,
      "log_odds_chosen": 0.50762939453125,
      "log_odds_ratio": -0.604296863079071,
      "logits/chosen": -0.7914062738418579,
      "logits/rejected": -0.7328125238418579,
      "logps/chosen": -0.705273449420929,
      "logps/rejected": -1.054101586341858,
      "loss": 1.0372,
      "nll_loss": 0.968554675579071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.070556640625,
      "rewards/margins": 0.03481445461511612,
      "rewards/rejected": -0.10551758110523224,
      "step": 4580
    },
    {
      "epoch": 0.17200352251222575,
      "grad_norm": 1.5806055622904676,
      "learning_rate": 1.1808198473867937e-06,
      "log_odds_chosen": 0.514843761920929,
      "log_odds_ratio": -0.5889648199081421,
      "logits/chosen": -0.765625,
      "logits/rejected": -0.688671886920929,
      "logps/chosen": -0.7093750238418579,
      "logps/rejected": -1.048242211341858,
      "loss": 1.0212,
      "nll_loss": 0.9593750238418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07093505561351776,
      "rewards/margins": 0.03387413173913956,
      "rewards/rejected": -0.10482177883386612,
      "step": 4590
    },
    {
      "epoch": 0.17237825785538963,
      "grad_norm": 1.6260571033251165,
      "learning_rate": 1.179535649239177e-06,
      "log_odds_chosen": 0.2847656309604645,
      "log_odds_ratio": -0.6630859375,
      "logits/chosen": -0.7884765863418579,
      "logits/rejected": -0.711718738079071,
      "logps/chosen": -0.7572265863418579,
      "logps/rejected": -0.9593750238418579,
      "loss": 1.0167,
      "nll_loss": 0.9951171875,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07573242485523224,
      "rewards/margins": 0.02025604248046875,
      "rewards/rejected": -0.095947265625,
      "step": 4600
    },
    {
      "epoch": 0.17275299319855353,
      "grad_norm": 1.4815835022505406,
      "learning_rate": 1.178255631879771e-06,
      "log_odds_chosen": 0.37464600801467896,
      "log_odds_ratio": -0.6533203125,
      "logits/chosen": -0.8115234375,
      "logits/rejected": -0.7275390625,
      "logps/chosen": -0.755859375,
      "logps/rejected": -1.0261719226837158,
      "loss": 0.9986,
      "nll_loss": 0.950976550579071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07557372748851776,
      "rewards/margins": 0.027087783440947533,
      "rewards/rejected": -0.10270996391773224,
      "step": 4610
    },
    {
      "epoch": 0.1731277285417174,
      "grad_norm": 1.4996219255036913,
      "learning_rate": 1.1769797726729992e-06,
      "log_odds_chosen": 0.40519410371780396,
      "log_odds_ratio": -0.615527331829071,
      "logits/chosen": -0.800585925579071,
      "logits/rejected": -0.7386718988418579,
      "logps/chosen": -0.6859375238418579,
      "logps/rejected": -0.949023425579071,
      "loss": 1.0125,
      "nll_loss": 0.955859363079071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06861571967601776,
      "rewards/margins": 0.02634887769818306,
      "rewards/rejected": -0.094970703125,
      "step": 4620
    },
    {
      "epoch": 0.1735024638848813,
      "grad_norm": 1.4645121705702047,
      "learning_rate": 1.1757080491544881e-06,
      "log_odds_chosen": 0.4435791075229645,
      "log_odds_ratio": -0.6337890625,
      "logits/chosen": -0.7935546636581421,
      "logits/rejected": -0.7398437261581421,
      "logps/chosen": -0.71630859375,
      "logps/rejected": -1.039453148841858,
      "loss": 1.0062,
      "nll_loss": 0.9496093988418579,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07164306938648224,
      "rewards/margins": 0.03233032301068306,
      "rewards/rejected": -0.10393066704273224,
      "step": 4630
    },
    {
      "epoch": 0.1738771992280452,
      "grad_norm": 1.3658206297663578,
      "learning_rate": 1.1744404390294068e-06,
      "log_odds_chosen": 0.551892101764679,
      "log_odds_ratio": -0.597851574420929,
      "logits/chosen": -0.7855468988418579,
      "logits/rejected": -0.7066406011581421,
      "logps/chosen": -0.743359386920929,
      "logps/rejected": -1.1396484375,
      "loss": 1.0293,
      "nll_loss": 0.9603515863418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07436523586511612,
      "rewards/margins": 0.03949584811925888,
      "rewards/rejected": -0.11394043266773224,
      "step": 4640
    },
    {
      "epoch": 0.17425193457120908,
      "grad_norm": 1.3959950682602738,
      "learning_rate": 1.1731769201708264e-06,
      "log_odds_chosen": 0.41114503145217896,
      "log_odds_ratio": -0.6275390386581421,
      "logits/chosen": -0.8011718988418579,
      "logits/rejected": -0.7509765625,
      "logps/chosen": -0.7457031011581421,
      "logps/rejected": -0.999804675579071,
      "loss": 1.0128,
      "nll_loss": 0.964648425579071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07456054538488388,
      "rewards/margins": 0.025368500500917435,
      "rewards/rejected": -0.09987793117761612,
      "step": 4650
    },
    {
      "epoch": 0.17462666991437298,
      "grad_norm": 1.48578861254211,
      "learning_rate": 1.1719174706180952e-06,
      "log_odds_chosen": 0.42479246854782104,
      "log_odds_ratio": -0.6236327886581421,
      "logits/chosen": -0.8648437261581421,
      "logits/rejected": -0.765625,
      "logps/chosen": -0.6953125,
      "logps/rejected": -0.9830077886581421,
      "loss": 1.0015,
      "nll_loss": 0.8929687738418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06951904296875,
      "rewards/margins": 0.02878418006002903,
      "rewards/rejected": -0.09824218600988388,
      "step": 4660
    },
    {
      "epoch": 0.17500140525753685,
      "grad_norm": 1.3764107890586061,
      "learning_rate": 1.1706620685752386e-06,
      "log_odds_chosen": 0.4056030213832855,
      "log_odds_ratio": -0.626269519329071,
      "logits/chosen": -0.82421875,
      "logits/rejected": -0.7427734136581421,
      "logps/chosen": -0.7066406011581421,
      "logps/rejected": -0.971484363079071,
      "loss": 1.0272,
      "nll_loss": 0.9664062261581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0706787109375,
      "rewards/margins": 0.02647705003619194,
      "rewards/rejected": -0.09714355319738388,
      "step": 4670
    },
    {
      "epoch": 0.17537614060070075,
      "grad_norm": 1.3959581668112349,
      "learning_rate": 1.1694106924093723e-06,
      "log_odds_chosen": 0.42974853515625,
      "log_odds_ratio": -0.617382824420929,
      "logits/chosen": -0.7623046636581421,
      "logits/rejected": -0.7251952886581421,
      "logps/chosen": -0.710156261920929,
      "logps/rejected": -1.008203148841858,
      "loss": 1.025,
      "nll_loss": 0.946093738079071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07097168266773224,
      "rewards/margins": 0.02987365797162056,
      "rewards/rejected": -0.100830078125,
      "step": 4680
    },
    {
      "epoch": 0.17575087594386465,
      "grad_norm": 1.5651144988719077,
      "learning_rate": 1.1681633206491381e-06,
      "log_odds_chosen": 0.4397949278354645,
      "log_odds_ratio": -0.5985351800918579,
      "logits/chosen": -0.7596679925918579,
      "logits/rejected": -0.69921875,
      "logps/chosen": -0.716796875,
      "logps/rejected": -1.007226586341858,
      "loss": 1.009,
      "nll_loss": 0.955859363079071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0716552734375,
      "rewards/margins": 0.02904968336224556,
      "rewards/rejected": -0.10067138820886612,
      "step": 4690
    },
    {
      "epoch": 0.17612561128702853,
      "grad_norm": 1.3909408681960396,
      "learning_rate": 1.1669199319831564e-06,
      "log_odds_chosen": 0.3963867127895355,
      "log_odds_ratio": -0.638378918170929,
      "logits/chosen": -0.786328136920929,
      "logits/rejected": -0.7071288824081421,
      "logps/chosen": -0.6900390386581421,
      "logps/rejected": -0.958984375,
      "loss": 1.0278,
      "nll_loss": 0.992968738079071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06907959282398224,
      "rewards/margins": 0.02695159986615181,
      "rewards/rejected": -0.09602050483226776,
      "step": 4700
    },
    {
      "epoch": 0.17650034663019243,
      "grad_norm": 1.506396616074277,
      "learning_rate": 1.1656805052584958e-06,
      "log_odds_chosen": 0.4712890684604645,
      "log_odds_ratio": -0.610546886920929,
      "logits/chosen": -0.869921863079071,
      "logits/rejected": -0.791210949420929,
      "logps/chosen": -0.7210937738418579,
      "logps/rejected": -1.036523461341858,
      "loss": 1.0016,
      "nll_loss": 0.881054699420929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07211913913488388,
      "rewards/margins": 0.03148956224322319,
      "rewards/rejected": -0.10361327975988388,
      "step": 4710
    },
    {
      "epoch": 0.17687508197335633,
      "grad_norm": 1.3527516405702997,
      "learning_rate": 1.164445019479164e-06,
      "log_odds_chosen": 0.19790038466453552,
      "log_odds_ratio": -0.7066406011581421,
      "logits/chosen": -0.8060547113418579,
      "logits/rejected": -0.743945300579071,
      "logps/chosen": -0.7818359136581421,
      "logps/rejected": -0.9261718988418579,
      "loss": 1.0208,
      "nll_loss": 1.00390625,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07817383110523224,
      "rewards/margins": 0.01436462439596653,
      "rewards/rejected": -0.09259033203125,
      "step": 4720
    },
    {
      "epoch": 0.1772498173165202,
      "grad_norm": 1.5074142081006987,
      "learning_rate": 1.1632134538046105e-06,
      "log_odds_chosen": 0.5184081792831421,
      "log_odds_ratio": -0.6075195074081421,
      "logits/chosen": -0.805859386920929,
      "logits/rejected": -0.736328125,
      "logps/chosen": -0.742382824420929,
      "logps/rejected": -1.1160156726837158,
      "loss": 1.0043,
      "nll_loss": 0.915820300579071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07429198920726776,
      "rewards/margins": 0.03733367845416069,
      "rewards/rejected": -0.11159668117761612,
      "step": 4730
    },
    {
      "epoch": 0.1776245526596841,
      "grad_norm": 1.46796449395377,
      "learning_rate": 1.1619857875482536e-06,
      "log_odds_chosen": 0.6576172113418579,
      "log_odds_ratio": -0.544726550579071,
      "logits/chosen": -0.7847656011581421,
      "logits/rejected": -0.6874023675918579,
      "logps/chosen": -0.7083984613418579,
      "logps/rejected": -1.150781273841858,
      "loss": 1.0011,
      "nll_loss": 0.9466797113418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07086181640625,
      "rewards/margins": 0.04420013353228569,
      "rewards/rejected": -0.11506347358226776,
      "step": 4740
    },
    {
      "epoch": 0.17799928800284798,
      "grad_norm": 1.3961133054125037,
      "learning_rate": 1.1607620001760185e-06,
      "log_odds_chosen": 0.28474122285842896,
      "log_odds_ratio": -0.669140636920929,
      "logits/chosen": -0.8062499761581421,
      "logits/rejected": -0.7789062261581421,
      "logps/chosen": -0.73828125,
      "logps/rejected": -0.9242187738418579,
      "loss": 1.0127,
      "nll_loss": 0.911914050579071,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07384033501148224,
      "rewards/margins": 0.018549347296357155,
      "rewards/rejected": -0.09241943061351776,
      "step": 4750
    },
    {
      "epoch": 0.17837402334601188,
      "grad_norm": 1.5010481185618596,
      "learning_rate": 1.1595420713048968e-06,
      "log_odds_chosen": 0.2679199278354645,
      "log_odds_ratio": -0.679394543170929,
      "logits/chosen": -0.838671863079071,
      "logits/rejected": -0.7591797113418579,
      "logps/chosen": -0.7759765386581421,
      "logps/rejected": -0.960156261920929,
      "loss": 1.0,
      "nll_loss": 0.960156261920929,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07757568359375,
      "rewards/margins": 0.01844329759478569,
      "rewards/rejected": -0.09599609673023224,
      "step": 4760
    },
    {
      "epoch": 0.17874875868917578,
      "grad_norm": 1.4764700231821972,
      "learning_rate": 1.1583259807015182e-06,
      "log_odds_chosen": 0.35417479276657104,
      "log_odds_ratio": -0.623730480670929,
      "logits/chosen": -0.7867187261581421,
      "logits/rejected": -0.712695300579071,
      "logps/chosen": -0.744335949420929,
      "logps/rejected": -0.9673827886581421,
      "loss": 0.9975,
      "nll_loss": 0.9410156011581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07441405951976776,
      "rewards/margins": 0.022307585924863815,
      "rewards/rejected": -0.09666748344898224,
      "step": 4770
    },
    {
      "epoch": 0.17912349403233965,
      "grad_norm": 1.4041078405117182,
      "learning_rate": 1.1571137082807434e-06,
      "log_odds_chosen": 0.3780761659145355,
      "log_odds_ratio": -0.641796886920929,
      "logits/chosen": -0.7835937738418579,
      "logits/rejected": -0.6826171875,
      "logps/chosen": -0.716796875,
      "logps/rejected": -0.955078125,
      "loss": 1.0146,
      "nll_loss": 0.982421875,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07167968899011612,
      "rewards/margins": 0.02387237548828125,
      "rewards/rejected": -0.09548339992761612,
      "step": 4780
    },
    {
      "epoch": 0.17949822937550355,
      "grad_norm": 1.4339740556775926,
      "learning_rate": 1.155905234104269e-06,
      "log_odds_chosen": 0.549914538860321,
      "log_odds_ratio": -0.576855480670929,
      "logits/chosen": -0.822070300579071,
      "logits/rejected": -0.7408202886581421,
      "logps/chosen": -0.68701171875,
      "logps/rejected": -1.051171898841858,
      "loss": 1.0222,
      "nll_loss": 0.953320324420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06873778998851776,
      "rewards/margins": 0.036420442163944244,
      "rewards/rejected": -0.10512695461511612,
      "step": 4790
    },
    {
      "epoch": 0.17987296471866743,
      "grad_norm": 1.584413862943699,
      "learning_rate": 1.1547005383792516e-06,
      "log_odds_chosen": 0.35980224609375,
      "log_odds_ratio": -0.6353515386581421,
      "logits/chosen": -0.801953136920929,
      "logits/rejected": -0.706835925579071,
      "logps/chosen": -0.767578125,
      "logps/rejected": -1.023046851158142,
      "loss": 0.9918,
      "nll_loss": 0.955859363079071,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07673339545726776,
      "rewards/margins": 0.0255279541015625,
      "rewards/rejected": -0.10222168266773224,
      "step": 4800
    },
    {
      "epoch": 0.18024770006183133,
      "grad_norm": 1.4248083851239572,
      "learning_rate": 1.1534996014569446e-06,
      "log_odds_chosen": 0.525927722454071,
      "log_odds_ratio": -0.6123046875,
      "logits/chosen": -0.7818359136581421,
      "logits/rejected": -0.703125,
      "logps/chosen": -0.733203113079071,
      "logps/rejected": -1.0908203125,
      "loss": 1.0002,
      "nll_loss": 0.978515625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07332763820886612,
      "rewards/margins": 0.03574409335851669,
      "rewards/rejected": -0.10906982421875,
      "step": 4810
    },
    {
      "epoch": 0.18062243540499523,
      "grad_norm": 1.4487374227626042,
      "learning_rate": 1.1523024038313547e-06,
      "log_odds_chosen": 0.35957032442092896,
      "log_odds_ratio": -0.6556640863418579,
      "logits/chosen": -0.8179687261581421,
      "logits/rejected": -0.728515625,
      "logps/chosen": -0.7171875238418579,
      "logps/rejected": -0.969531238079071,
      "loss": 1.0214,
      "nll_loss": 0.9638671875,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.0716552734375,
      "rewards/margins": 0.02523345872759819,
      "rewards/rejected": -0.09689941257238388,
      "step": 4820
    },
    {
      "epoch": 0.1809971707481591,
      "grad_norm": 1.4901343148996566,
      "learning_rate": 1.1511089261379083e-06,
      "log_odds_chosen": 0.44970703125,
      "log_odds_ratio": -0.6297851800918579,
      "logits/chosen": -0.786816418170929,
      "logits/rejected": -0.6734374761581421,
      "logps/chosen": -0.763476550579071,
      "logps/rejected": -1.060937523841858,
      "loss": 1.0275,
      "nll_loss": 1.0529296398162842,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07634277641773224,
      "rewards/margins": 0.029750823974609375,
      "rewards/rejected": -0.1060791015625,
      "step": 4830
    },
    {
      "epoch": 0.181371906091323,
      "grad_norm": 1.5854888986836688,
      "learning_rate": 1.149919149152138e-06,
      "log_odds_chosen": 0.630780041217804,
      "log_odds_ratio": -0.55224609375,
      "logits/chosen": -0.789257824420929,
      "logits/rejected": -0.7103515863418579,
      "logps/chosen": -0.691601574420929,
      "logps/rejected": -1.1130859851837158,
      "loss": 1.0325,
      "nll_loss": 1.034570336341858,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06918945163488388,
      "rewards/margins": 0.042111970484256744,
      "rewards/rejected": -0.11125488579273224,
      "step": 4840
    },
    {
      "epoch": 0.1817466414344869,
      "grad_norm": 1.4517007238470008,
      "learning_rate": 1.148733053788381e-06,
      "log_odds_chosen": 0.5247558355331421,
      "log_odds_ratio": -0.5624023675918579,
      "logits/chosen": -0.796875,
      "logits/rejected": -0.747265636920929,
      "logps/chosen": -0.676953136920929,
      "logps/rejected": -0.9833984375,
      "loss": 0.9953,
      "nll_loss": 0.9203125238418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06772460788488388,
      "rewards/margins": 0.03060760535299778,
      "rewards/rejected": -0.09836425632238388,
      "step": 4850
    },
    {
      "epoch": 0.18212137677765078,
      "grad_norm": 1.358993758242819,
      "learning_rate": 1.1475506210984938e-06,
      "log_odds_chosen": 0.37016600370407104,
      "log_odds_ratio": -0.634570300579071,
      "logits/chosen": -0.795703113079071,
      "logits/rejected": -0.714550793170929,
      "logps/chosen": -0.755078136920929,
      "logps/rejected": -1.0251953601837158,
      "loss": 1.0383,
      "nll_loss": 0.9517577886581421,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07554931938648224,
      "rewards/margins": 0.02699127234518528,
      "rewards/rejected": -0.10249023139476776,
      "step": 4860
    },
    {
      "epoch": 0.18249611212081468,
      "grad_norm": 1.4504627355188944,
      "learning_rate": 1.1463718322705807e-06,
      "log_odds_chosen": 0.3818115293979645,
      "log_odds_ratio": -0.63232421875,
      "logits/chosen": -0.7880859375,
      "logits/rejected": -0.713671863079071,
      "logps/chosen": -0.7416015863418579,
      "logps/rejected": -0.986523449420929,
      "loss": 1.0132,
      "nll_loss": 0.9722656011581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07414551079273224,
      "rewards/margins": 0.024495696648955345,
      "rewards/rejected": -0.09865722805261612,
      "step": 4870
    },
    {
      "epoch": 0.18287084746397855,
      "grad_norm": 1.4547816453575877,
      "learning_rate": 1.1451966686277364e-06,
      "log_odds_chosen": 0.517529308795929,
      "log_odds_ratio": -0.5775390863418579,
      "logits/chosen": -0.825976550579071,
      "logits/rejected": -0.708691418170929,
      "logps/chosen": -0.702832043170929,
      "logps/rejected": -1.039453148841858,
      "loss": 1.0086,
      "nll_loss": 0.9087890386581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07033691555261612,
      "rewards/margins": 0.03369140625,
      "rewards/rejected": -0.10397949069738388,
      "step": 4880
    },
    {
      "epoch": 0.18324558280714245,
      "grad_norm": 1.5253794621509411,
      "learning_rate": 1.1440251116268034e-06,
      "log_odds_chosen": 0.39210206270217896,
      "log_odds_ratio": -0.626660168170929,
      "logits/chosen": -0.774218738079071,
      "logits/rejected": -0.683789074420929,
      "logps/chosen": -0.7552734613418579,
      "logps/rejected": -1.013085961341858,
      "loss": 1.019,
      "nll_loss": 1.011132836341858,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.0755615234375,
      "rewards/margins": 0.025739287957549095,
      "rewards/rejected": -0.10134277492761612,
      "step": 4890
    },
    {
      "epoch": 0.18362031815030636,
      "grad_norm": 1.4870596168105208,
      "learning_rate": 1.1428571428571428e-06,
      "log_odds_chosen": 0.48430174589157104,
      "log_odds_ratio": -0.591796875,
      "logits/chosen": -0.821484386920929,
      "logits/rejected": -0.7093750238418579,
      "logps/chosen": -0.711718738079071,
      "logps/rejected": -1.041015625,
      "loss": 1.0344,
      "nll_loss": 0.932421863079071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07127685844898224,
      "rewards/margins": 0.03288726881146431,
      "rewards/rejected": -0.10407714545726776,
      "step": 4900
    },
    {
      "epoch": 0.18399505349347023,
      "grad_norm": 1.4582995152907543,
      "learning_rate": 1.14169274403942e-06,
      "log_odds_chosen": 0.4102783203125,
      "log_odds_ratio": -0.6273437738418579,
      "logits/chosen": -0.766796886920929,
      "logits/rejected": -0.707714855670929,
      "logps/chosen": -0.7578125,
      "logps/rejected": -1.042382836341858,
      "loss": 1.0255,
      "nll_loss": 1.0029296875,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07569579780101776,
      "rewards/margins": 0.02843627892434597,
      "rewards/rejected": -0.10419921576976776,
      "step": 4910
    },
    {
      "epoch": 0.18436978883663413,
      "grad_norm": 1.4155847703838387,
      "learning_rate": 1.140531897024402e-06,
      "log_odds_chosen": 0.44581300020217896,
      "log_odds_ratio": -0.613964855670929,
      "logits/chosen": -0.8285156488418579,
      "logits/rejected": -0.743945300579071,
      "logps/chosen": -0.704296886920929,
      "logps/rejected": -1.008203148841858,
      "loss": 1.018,
      "nll_loss": 0.908398449420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07042236626148224,
      "rewards/margins": 0.03042449988424778,
      "rewards/rejected": -0.10081787407398224,
      "step": 4920
    },
    {
      "epoch": 0.18474452417979803,
      "grad_norm": 1.484441789295811,
      "learning_rate": 1.13937458379177e-06,
      "log_odds_chosen": 0.5240722894668579,
      "log_odds_ratio": -0.5894531011581421,
      "logits/chosen": -0.792773425579071,
      "logits/rejected": -0.6923828125,
      "logps/chosen": -0.710644543170929,
      "logps/rejected": -1.0714843273162842,
      "loss": 1.0117,
      "nll_loss": 0.908984363079071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07110595703125,
      "rewards/margins": 0.03607177734375,
      "rewards/rejected": -0.10722656548023224,
      "step": 4930
    },
    {
      "epoch": 0.1851192595229619,
      "grad_norm": 1.671986322698223,
      "learning_rate": 1.1382207864489444e-06,
      "log_odds_chosen": 0.46647948026657104,
      "log_odds_ratio": -0.629589855670929,
      "logits/chosen": -0.797656238079071,
      "logits/rejected": -0.735156238079071,
      "logps/chosen": -0.7529296875,
      "logps/rejected": -1.076562523841858,
      "loss": 1.0149,
      "nll_loss": 0.9849609136581421,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.0753173828125,
      "rewards/margins": 0.03232269361615181,
      "rewards/rejected": -0.10769043117761612,
      "step": 4940
    },
    {
      "epoch": 0.1854939948661258,
      "grad_norm": 1.430830577905321,
      "learning_rate": 1.1370704872299223e-06,
      "log_odds_chosen": 0.32696533203125,
      "log_odds_ratio": -0.6908203363418579,
      "logits/chosen": -0.790234386920929,
      "logits/rejected": -0.701171875,
      "logps/chosen": -0.7916015386581421,
      "logps/rejected": -1.0392577648162842,
      "loss": 1.0208,
      "nll_loss": 0.969531238079071,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07913818210363388,
      "rewards/margins": 0.02482757531106472,
      "rewards/rejected": -0.10397949069738388,
      "step": 4950
    },
    {
      "epoch": 0.18586873020928968,
      "grad_norm": 1.401756716683372,
      "learning_rate": 1.1359236684941295e-06,
      "log_odds_chosen": 0.407958984375,
      "log_odds_ratio": -0.6177734136581421,
      "logits/chosen": -0.7890625,
      "logits/rejected": -0.7030273675918579,
      "logps/chosen": -0.73193359375,
      "logps/rejected": -0.986328125,
      "loss": 1.0058,
      "nll_loss": 0.9644531011581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07315673679113388,
      "rewards/margins": 0.02552337571978569,
      "rewards/rejected": -0.09865722805261612,
      "step": 4960
    },
    {
      "epoch": 0.18624346555245358,
      "grad_norm": 1.4433978408384998,
      "learning_rate": 1.1347803127252839e-06,
      "log_odds_chosen": 0.3365478515625,
      "log_odds_ratio": -0.6327148675918579,
      "logits/chosen": -0.8193359375,
      "logits/rejected": -0.724609375,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -0.931835949420929,
      "loss": 0.9964,
      "nll_loss": 0.922070324420929,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07169189304113388,
      "rewards/margins": 0.02150726318359375,
      "rewards/rejected": -0.09323730319738388,
      "step": 4970
    },
    {
      "epoch": 0.18661820089561748,
      "grad_norm": 1.4820506929521844,
      "learning_rate": 1.1336404025302715e-06,
      "log_odds_chosen": 0.4490600526332855,
      "log_odds_ratio": -0.6234375238418579,
      "logits/chosen": -0.7886718511581421,
      "logits/rejected": -0.7318359613418579,
      "logps/chosen": -0.716015636920929,
      "logps/rejected": -1.0310547351837158,
      "loss": 1.01,
      "nll_loss": 0.98046875,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07164306938648224,
      "rewards/margins": 0.03142700344324112,
      "rewards/rejected": -0.10307617485523224,
      "step": 4980
    },
    {
      "epoch": 0.18699293623878135,
      "grad_norm": 1.4613853365769054,
      "learning_rate": 1.1325039206380352e-06,
      "log_odds_chosen": 0.25067138671875,
      "log_odds_ratio": -0.693164050579071,
      "logits/chosen": -0.7666015625,
      "logits/rejected": -0.7159179449081421,
      "logps/chosen": -0.741992175579071,
      "logps/rejected": -0.923632800579071,
      "loss": 1.0261,
      "nll_loss": 0.987109363079071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07421875,
      "rewards/margins": 0.01814727857708931,
      "rewards/rejected": -0.09235839545726776,
      "step": 4990
    },
    {
      "epoch": 0.18736767158194526,
      "grad_norm": 1.4708074419370973,
      "learning_rate": 1.131370849898476e-06,
      "log_odds_chosen": 0.5504394769668579,
      "log_odds_ratio": -0.570019543170929,
      "logits/chosen": -0.7984374761581421,
      "logits/rejected": -0.7162109613418579,
      "logps/chosen": -0.694531261920929,
      "logps/rejected": -1.0535156726837158,
      "loss": 1.0158,
      "nll_loss": 0.940234363079071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06943359225988388,
      "rewards/margins": 0.03588562086224556,
      "rewards/rejected": -0.1053466796875,
      "step": 5000
    },
    {
      "epoch": 0.18736767158194526,
      "eval_log_odds_chosen": 0.37588492035865784,
      "eval_log_odds_ratio": -0.6365241408348083,
      "eval_logits/chosen": -0.7865331768989563,
      "eval_logits/rejected": -0.709992527961731,
      "eval_logps/chosen": -0.7526043653488159,
      "eval_logps/rejected": -1.0089200735092163,
      "eval_loss": 1.0987300872802734,
      "eval_nll_loss": 1.0370804071426392,
      "eval_rewards/accuracies": 0.5867329835891724,
      "eval_rewards/chosen": -0.07525432854890823,
      "eval_rewards/margins": 0.025626344606280327,
      "eval_rewards/rejected": -0.10088168829679489,
      "eval_runtime": 2456.0739,
      "eval_samples_per_second": 77.286,
      "eval_steps_per_second": 1.208,
      "step": 5000
    },
    {
      "epoch": 0.18774240692510913,
      "grad_norm": 1.3240842326668232,
      "learning_rate": 1.130241173281366e-06,
      "log_odds_chosen": 0.41326904296875,
      "log_odds_ratio": -0.6351562738418579,
      "logits/chosen": -0.744921863079071,
      "logits/rejected": -0.674511730670929,
      "logps/chosen": -0.730664074420929,
      "logps/rejected": -0.9896484613418579,
      "loss": 0.9867,
      "nll_loss": 0.950390636920929,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07308349758386612,
      "rewards/margins": 0.025847624987363815,
      "rewards/rejected": -0.09895019233226776,
      "step": 5010
    },
    {
      "epoch": 0.18811714226827303,
      "grad_norm": 1.4202000346723023,
      "learning_rate": 1.1291148738752732e-06,
      "log_odds_chosen": 0.39306640625,
      "log_odds_ratio": -0.653027355670929,
      "logits/chosen": -0.843554675579071,
      "logits/rejected": -0.7818359136581421,
      "logps/chosen": -0.730175793170929,
      "logps/rejected": -0.975390613079071,
      "loss": 1.0132,
      "nll_loss": 0.9126952886581421,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07297363132238388,
      "rewards/margins": 0.02457885816693306,
      "rewards/rejected": -0.09757079929113388,
      "step": 5020
    },
    {
      "epoch": 0.18849187761143693,
      "grad_norm": 1.4419584322752133,
      "learning_rate": 1.1279919348864981e-06,
      "log_odds_chosen": 0.3125,
      "log_odds_ratio": -0.6292968988418579,
      "logits/chosen": -0.826171875,
      "logits/rejected": -0.741406261920929,
      "logps/chosen": -0.7457031011581421,
      "logps/rejected": -0.938671886920929,
      "loss": 1.0106,
      "nll_loss": 0.966601550579071,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07451172173023224,
      "rewards/margins": 0.01930389367043972,
      "rewards/rejected": -0.09389648586511612,
      "step": 5030
    },
    {
      "epoch": 0.1888666129546008,
      "grad_norm": 1.376441685198369,
      "learning_rate": 1.126872339638022e-06,
      "log_odds_chosen": 0.3451385498046875,
      "log_odds_ratio": -0.6377929449081421,
      "logits/chosen": -0.7320312261581421,
      "logits/rejected": -0.6812499761581421,
      "logps/chosen": -0.690234363079071,
      "logps/rejected": -0.9125000238418579,
      "loss": 1.0202,
      "nll_loss": 1.013085961341858,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06898193061351776,
      "rewards/margins": 0.022216033190488815,
      "rewards/rejected": -0.09123535454273224,
      "step": 5040
    },
    {
      "epoch": 0.1892413482977647,
      "grad_norm": 1.3392651146182422,
      "learning_rate": 1.1257560715684668e-06,
      "log_odds_chosen": 0.3859619200229645,
      "log_odds_ratio": -0.6426757574081421,
      "logits/chosen": -0.8324218988418579,
      "logits/rejected": -0.737109363079071,
      "logps/chosen": -0.709179699420929,
      "logps/rejected": -0.96435546875,
      "loss": 1.0085,
      "nll_loss": 0.9205077886581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07086181640625,
      "rewards/margins": 0.02558288536965847,
      "rewards/rejected": -0.09642334282398224,
      "step": 5050
    },
    {
      "epoch": 0.1896160836409286,
      "grad_norm": 1.4818403397271551,
      "learning_rate": 1.1246431142310665e-06,
      "log_odds_chosen": 0.3198608458042145,
      "log_odds_ratio": -0.643750011920929,
      "logits/chosen": -0.8089843988418579,
      "logits/rejected": -0.742871105670929,
      "logps/chosen": -0.7569335699081421,
      "logps/rejected": -0.982226550579071,
      "loss": 0.991,
      "nll_loss": 0.9390624761581421,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07568359375,
      "rewards/margins": 0.02252807654440403,
      "rewards/rejected": -0.09824218600988388,
      "step": 5060
    },
    {
      "epoch": 0.18999081898409248,
      "grad_norm": 1.4224537276623648,
      "learning_rate": 1.1235334512926484e-06,
      "log_odds_chosen": 0.43977051973342896,
      "log_odds_ratio": -0.6463867425918579,
      "logits/chosen": -0.8082031011581421,
      "logits/rejected": -0.710742175579071,
      "logps/chosen": -0.737988293170929,
      "logps/rejected": -1.0363280773162842,
      "loss": 0.9985,
      "nll_loss": 0.939648449420929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07387695461511612,
      "rewards/margins": 0.02983856201171875,
      "rewards/rejected": -0.10366211086511612,
      "step": 5070
    },
    {
      "epoch": 0.19036555432725638,
      "grad_norm": 1.4179042345711725,
      "learning_rate": 1.1224270665326274e-06,
      "log_odds_chosen": 0.43964844942092896,
      "log_odds_ratio": -0.605273425579071,
      "logits/chosen": -0.8326171636581421,
      "logits/rejected": -0.7769531011581421,
      "logps/chosen": -0.67724609375,
      "logps/rejected": -0.969531238079071,
      "loss": 1.0137,
      "nll_loss": 0.888867199420929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06770019233226776,
      "rewards/margins": 0.02926330640912056,
      "rewards/rejected": -0.09694824367761612,
      "step": 5080
    },
    {
      "epoch": 0.19074028967042025,
      "grad_norm": 1.4763527966917396,
      "learning_rate": 1.12132394384201e-06,
      "log_odds_chosen": 0.3319946229457855,
      "log_odds_ratio": -0.6666015386581421,
      "logits/chosen": -0.843554675579071,
      "logits/rejected": -0.767578125,
      "logps/chosen": -0.7337890863418579,
      "logps/rejected": -0.9564453363418579,
      "loss": 1.0273,
      "nll_loss": 0.930859386920929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0733642578125,
      "rewards/margins": 0.02229461632668972,
      "rewards/rejected": -0.09562988579273224,
      "step": 5090
    },
    {
      "epoch": 0.19111502501358416,
      "grad_norm": 1.5001000406223022,
      "learning_rate": 1.1202240672224076e-06,
      "log_odds_chosen": 0.4443115293979645,
      "log_odds_ratio": -0.602832019329071,
      "logits/chosen": -0.8101562261581421,
      "logits/rejected": -0.7464843988418579,
      "logps/chosen": -0.731249988079071,
      "logps/rejected": -1.021093726158142,
      "loss": 1.0315,
      "nll_loss": 0.9593750238418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07308349758386612,
      "rewards/margins": 0.028980255126953125,
      "rewards/rejected": -0.10214843600988388,
      "step": 5100
    },
    {
      "epoch": 0.19148976035674806,
      "grad_norm": 1.5089832340098073,
      "learning_rate": 1.1191274207850654e-06,
      "log_odds_chosen": 0.533337414264679,
      "log_odds_ratio": -0.563671886920929,
      "logits/chosen": -0.8011718988418579,
      "logits/rejected": -0.702929675579071,
      "logps/chosen": -0.7529296875,
      "logps/rejected": -1.0998046398162842,
      "loss": 1.0063,
      "nll_loss": 1.0066406726837158,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07535400241613388,
      "rewards/margins": 0.03471069410443306,
      "rewards/rejected": -0.1099853515625,
      "step": 5110
    },
    {
      "epoch": 0.19186449569991193,
      "grad_norm": 1.4058265101054341,
      "learning_rate": 1.1180339887498948e-06,
      "log_odds_chosen": 0.4199462831020355,
      "log_odds_ratio": -0.6226562261581421,
      "logits/chosen": -0.7847656011581421,
      "logits/rejected": -0.724804699420929,
      "logps/chosen": -0.7060546875,
      "logps/rejected": -0.9751952886581421,
      "loss": 1.0103,
      "nll_loss": 0.926562488079071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07056884467601776,
      "rewards/margins": 0.02688903734087944,
      "rewards/rejected": -0.09752197563648224,
      "step": 5120
    },
    {
      "epoch": 0.19223923104307583,
      "grad_norm": 1.4162838471604924,
      "learning_rate": 1.1169437554445213e-06,
      "log_odds_chosen": 0.2978759706020355,
      "log_odds_ratio": -0.6709960699081421,
      "logits/chosen": -0.787890613079071,
      "logits/rejected": -0.7310546636581421,
      "logps/chosen": -0.7435547113418579,
      "logps/rejected": -0.967968761920929,
      "loss": 1.004,
      "nll_loss": 1.0050780773162842,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.0743408203125,
      "rewards/margins": 0.0223846435546875,
      "rewards/rejected": -0.09671630710363388,
      "step": 5130
    },
    {
      "epoch": 0.1926139663862397,
      "grad_norm": 1.3503154872333527,
      "learning_rate": 1.1158567053033413e-06,
      "log_odds_chosen": 0.49107664823532104,
      "log_odds_ratio": -0.610156238079071,
      "logits/chosen": -0.841992199420929,
      "logits/rejected": -0.7583984136581421,
      "logps/chosen": -0.729687511920929,
      "logps/rejected": -1.0408203601837158,
      "loss": 0.9946,
      "nll_loss": 0.947070300579071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07297363132238388,
      "rewards/margins": 0.03104248084127903,
      "rewards/rejected": -0.10407714545726776,
      "step": 5140
    },
    {
      "epoch": 0.1929887017294036,
      "grad_norm": 1.4267174399172637,
      "learning_rate": 1.1147728228665882e-06,
      "log_odds_chosen": 0.41535645723342896,
      "log_odds_ratio": -0.610058605670929,
      "logits/chosen": -0.8382812738418579,
      "logits/rejected": -0.7411133050918579,
      "logps/chosen": -0.7236328125,
      "logps/rejected": -1.0105469226837158,
      "loss": 1.0323,
      "nll_loss": 1.016210913658142,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07237549126148224,
      "rewards/margins": 0.028656005859375,
      "rewards/rejected": -0.10100097954273224,
      "step": 5150
    },
    {
      "epoch": 0.1933634370725675,
      "grad_norm": 1.489092544896369,
      "learning_rate": 1.1136920927794092e-06,
      "log_odds_chosen": 0.23480224609375,
      "log_odds_ratio": -0.68505859375,
      "logits/chosen": -0.804882824420929,
      "logits/rejected": -0.734375,
      "logps/chosen": -0.734570324420929,
      "logps/rejected": -0.8857421875,
      "loss": 0.991,
      "nll_loss": 0.901171863079071,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.07344970852136612,
      "rewards/margins": 0.01515274029225111,
      "rewards/rejected": -0.08854980766773224,
      "step": 5160
    },
    {
      "epoch": 0.19373817241573138,
      "grad_norm": 1.4365357114172064,
      "learning_rate": 1.1126144997909508e-06,
      "log_odds_chosen": 0.37871092557907104,
      "log_odds_ratio": -0.6258789300918579,
      "logits/chosen": -0.767578125,
      "logits/rejected": -0.678515613079071,
      "logps/chosen": -0.7354491949081421,
      "logps/rejected": -0.9736328125,
      "loss": 1.0287,
      "nll_loss": 1.013671875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.073486328125,
      "rewards/margins": 0.02387695387005806,
      "rewards/rejected": -0.0972900390625,
      "step": 5170
    },
    {
      "epoch": 0.19411290775889528,
      "grad_norm": 1.6062706384054746,
      "learning_rate": 1.1115400287534568e-06,
      "log_odds_chosen": 0.3957275450229645,
      "log_odds_ratio": -0.611621081829071,
      "logits/chosen": -0.7476562261581421,
      "logits/rejected": -0.671679675579071,
      "logps/chosen": -0.6963866949081421,
      "logps/rejected": -0.9375,
      "loss": 1.0227,
      "nll_loss": 0.9085937738418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06968994438648224,
      "rewards/margins": 0.02406463585793972,
      "rewards/rejected": -0.09375,
      "step": 5180
    },
    {
      "epoch": 0.19448764310205918,
      "grad_norm": 1.4229038816073951,
      "learning_rate": 1.110468664621372e-06,
      "log_odds_chosen": 0.35688477754592896,
      "log_odds_ratio": -0.634082019329071,
      "logits/chosen": -0.8505859375,
      "logits/rejected": -0.790820300579071,
      "logps/chosen": -0.774609386920929,
      "logps/rejected": -0.9964843988418579,
      "loss": 1.02,
      "nll_loss": 0.9652343988418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.0775146484375,
      "rewards/margins": 0.02214355394244194,
      "rewards/rejected": -0.09956054389476776,
      "step": 5190
    },
    {
      "epoch": 0.19486237844522306,
      "grad_norm": 2.6538345518181368,
      "learning_rate": 1.1094003924504583e-06,
      "log_odds_chosen": 0.43214112520217896,
      "log_odds_ratio": -0.618359386920929,
      "logits/chosen": -0.833984375,
      "logits/rejected": -0.792187511920929,
      "logps/chosen": -0.7422851324081421,
      "logps/rejected": -1.025976538658142,
      "loss": 1.0196,
      "nll_loss": 0.970703125,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07418213039636612,
      "rewards/margins": 0.02828369103372097,
      "rewards/rejected": -0.10252685844898224,
      "step": 5200
    },
    {
      "epoch": 0.19523711378838696,
      "grad_norm": 1.4723895103776234,
      "learning_rate": 1.1083351973969191e-06,
      "log_odds_chosen": 0.42381590604782104,
      "log_odds_ratio": -0.6039062738418579,
      "logits/chosen": -0.8763672113418579,
      "logits/rejected": -0.766796886920929,
      "logps/chosen": -0.674609363079071,
      "logps/rejected": -0.9609375,
      "loss": 1.003,
      "nll_loss": 0.925000011920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06743164360523224,
      "rewards/margins": 0.02863464318215847,
      "rewards/rejected": -0.09602050483226776,
      "step": 5210
    },
    {
      "epoch": 0.19561184913155083,
      "grad_norm": 1.5530091575315892,
      "learning_rate": 1.107273064716533e-06,
      "log_odds_chosen": 0.3341064453125,
      "log_odds_ratio": -0.627246081829071,
      "logits/chosen": -0.8121093511581421,
      "logits/rejected": -0.7181640863418579,
      "logps/chosen": -0.7203124761581421,
      "logps/rejected": -0.931640625,
      "loss": 1.0288,
      "nll_loss": 0.9775390625,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07193603366613388,
      "rewards/margins": 0.02121429517865181,
      "rewards/rejected": -0.09312744438648224,
      "step": 5220
    },
    {
      "epoch": 0.19598658447471473,
      "grad_norm": 1.627611772370195,
      "learning_rate": 1.1062139797637962e-06,
      "log_odds_chosen": 0.34465330839157104,
      "log_odds_ratio": -0.634960949420929,
      "logits/chosen": -0.740039050579071,
      "logits/rejected": -0.6787109375,
      "logps/chosen": -0.735156238079071,
      "logps/rejected": -0.9449218511581421,
      "loss": 1.0315,
      "nll_loss": 0.9443359375,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07351074367761612,
      "rewards/margins": 0.02105102501809597,
      "rewards/rejected": -0.09456787258386612,
      "step": 5230
    },
    {
      "epoch": 0.19636131981787863,
      "grad_norm": 1.5154149295386667,
      "learning_rate": 1.1051579279910751e-06,
      "log_odds_chosen": 0.37891846895217896,
      "log_odds_ratio": -0.63330078125,
      "logits/chosen": -0.7308593988418579,
      "logits/rejected": -0.6307617425918579,
      "logps/chosen": -0.7621093988418579,
      "logps/rejected": -1.0115234851837158,
      "loss": 1.024,
      "nll_loss": 1.0087890625,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07618407905101776,
      "rewards/margins": 0.02500152587890625,
      "rewards/rejected": -0.10117187350988388,
      "step": 5240
    },
    {
      "epoch": 0.1967360551610425,
      "grad_norm": 1.4211930317469625,
      "learning_rate": 1.1041048949477667e-06,
      "log_odds_chosen": 0.5107055902481079,
      "log_odds_ratio": -0.586621105670929,
      "logits/chosen": -0.819531261920929,
      "logits/rejected": -0.687695324420929,
      "logps/chosen": -0.7134765386581421,
      "logps/rejected": -1.0525391101837158,
      "loss": 1.0195,
      "nll_loss": 0.9437500238418579,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07131347805261612,
      "rewards/margins": 0.03393401950597763,
      "rewards/rejected": -0.10529784858226776,
      "step": 5250
    },
    {
      "epoch": 0.1971107905042064,
      "grad_norm": 1.386417204965951,
      "learning_rate": 1.1030548662794673e-06,
      "log_odds_chosen": 0.4070068299770355,
      "log_odds_ratio": -0.614550769329071,
      "logits/chosen": -0.7060546875,
      "logits/rejected": -0.633007824420929,
      "logps/chosen": -0.7250000238418579,
      "logps/rejected": -0.9837890863418579,
      "loss": 0.9997,
      "nll_loss": 0.9800781011581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07247314602136612,
      "rewards/margins": 0.02583618089556694,
      "rewards/rejected": -0.09824218600988388,
      "step": 5260
    },
    {
      "epoch": 0.1974855258473703,
      "grad_norm": 1.5546443808899741,
      "learning_rate": 1.102007827727152e-06,
      "log_odds_chosen": 0.35393065214157104,
      "log_odds_ratio": -0.6376953125,
      "logits/chosen": -0.8050781488418579,
      "logits/rejected": -0.7291015386581421,
      "logps/chosen": -0.7574218511581421,
      "logps/rejected": -1.003515601158142,
      "loss": 1.0044,
      "nll_loss": 0.9027343988418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07564697414636612,
      "rewards/margins": 0.02467498742043972,
      "rewards/rejected": -0.10037841647863388,
      "step": 5270
    },
    {
      "epoch": 0.19786026119053418,
      "grad_norm": 1.488953919209301,
      "learning_rate": 1.1009637651263607e-06,
      "log_odds_chosen": 0.3305419981479645,
      "log_odds_ratio": -0.659863293170929,
      "logits/chosen": -0.7236328125,
      "logits/rejected": -0.6693359613418579,
      "logps/chosen": -0.760937511920929,
      "logps/rejected": -0.982226550579071,
      "loss": 1.0108,
      "nll_loss": 0.956835925579071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07613525539636612,
      "rewards/margins": 0.022064208984375,
      "rewards/rejected": -0.09824218600988388,
      "step": 5280
    },
    {
      "epoch": 0.19823499653369808,
      "grad_norm": 1.545243969038594,
      "learning_rate": 1.0999226644063927e-06,
      "log_odds_chosen": 0.45404052734375,
      "log_odds_ratio": -0.617968738079071,
      "logits/chosen": -0.7177734375,
      "logits/rejected": -0.645800769329071,
      "logps/chosen": -0.7421875,
      "logps/rejected": -1.0615234375,
      "loss": 1.0209,
      "nll_loss": 0.966015636920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07419433444738388,
      "rewards/margins": 0.031964875757694244,
      "rewards/rejected": -0.10625000298023224,
      "step": 5290
    },
    {
      "epoch": 0.19860973187686196,
      "grad_norm": 1.4481381967471094,
      "learning_rate": 1.0988845115895123e-06,
      "log_odds_chosen": 0.4030395448207855,
      "log_odds_ratio": -0.6112304925918579,
      "logits/chosen": -0.7076171636581421,
      "logits/rejected": -0.6537109613418579,
      "logps/chosen": -0.759765625,
      "logps/rejected": -1.0476562976837158,
      "loss": 1.0323,
      "nll_loss": 1.0226562023162842,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07596435397863388,
      "rewards/margins": 0.028755951672792435,
      "rewards/rejected": -0.10476074367761612,
      "step": 5300
    },
    {
      "epoch": 0.19898446722002586,
      "grad_norm": 1.6284119967646056,
      "learning_rate": 1.0978492927901574e-06,
      "log_odds_chosen": 0.3265624940395355,
      "log_odds_ratio": -0.653613269329071,
      "logits/chosen": -0.7416015863418579,
      "logits/rejected": -0.669140636920929,
      "logps/chosen": -0.7759765386581421,
      "logps/rejected": -1.008398413658142,
      "loss": 1.0191,
      "nll_loss": 1.083984375,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07755126804113388,
      "rewards/margins": 0.0233306884765625,
      "rewards/rejected": -0.10092773288488388,
      "step": 5310
    },
    {
      "epoch": 0.19935920256318976,
      "grad_norm": 1.5131045474665754,
      "learning_rate": 1.0968169942141634e-06,
      "log_odds_chosen": 0.42021483182907104,
      "log_odds_ratio": -0.6136718988418579,
      "logits/chosen": -0.8042968511581421,
      "logits/rejected": -0.7230468988418579,
      "logps/chosen": -0.701953113079071,
      "logps/rejected": -0.9837890863418579,
      "loss": 0.9935,
      "nll_loss": 0.911914050579071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0701904296875,
      "rewards/margins": 0.02821655198931694,
      "rewards/rejected": -0.09843750298023224,
      "step": 5320
    },
    {
      "epoch": 0.19973393790635363,
      "grad_norm": 1.5515990045594084,
      "learning_rate": 1.0957876021579874e-06,
      "log_odds_chosen": 0.36419677734375,
      "log_odds_ratio": -0.62744140625,
      "logits/chosen": -0.747851550579071,
      "logits/rejected": -0.6683593988418579,
      "logps/chosen": -0.739453136920929,
      "logps/rejected": -0.9898437261581421,
      "loss": 1.0151,
      "nll_loss": 0.934765636920929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07395019382238388,
      "rewards/margins": 0.024980926886200905,
      "rewards/rejected": -0.098876953125,
      "step": 5330
    },
    {
      "epoch": 0.20010867324951753,
      "grad_norm": 1.5951843822269294,
      "learning_rate": 1.0947611030079466e-06,
      "log_odds_chosen": 0.32476806640625,
      "log_odds_ratio": -0.64404296875,
      "logits/chosen": -0.73291015625,
      "logits/rejected": -0.670214831829071,
      "logps/chosen": -0.72265625,
      "logps/rejected": -0.9439452886581421,
      "loss": 1.0086,
      "nll_loss": 0.8939453363418579,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.072265625,
      "rewards/margins": 0.02213287353515625,
      "rewards/rejected": -0.09443359076976776,
      "step": 5340
    },
    {
      "epoch": 0.2004834085926814,
      "grad_norm": 1.4971444824415558,
      "learning_rate": 1.0937374832394612e-06,
      "log_odds_chosen": 0.5105835199356079,
      "log_odds_ratio": -0.5746093988418579,
      "logits/chosen": -0.787890613079071,
      "logits/rejected": -0.667773425579071,
      "logps/chosen": -0.7408202886581421,
      "logps/rejected": -1.062890648841858,
      "loss": 1.0063,
      "nll_loss": 0.9482421875,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07410888373851776,
      "rewards/margins": 0.03223419189453125,
      "rewards/rejected": -0.10627441108226776,
      "step": 5350
    },
    {
      "epoch": 0.2008581439358453,
      "grad_norm": 1.474359303365608,
      "learning_rate": 1.092716729416306e-06,
      "log_odds_chosen": 0.23476561903953552,
      "log_odds_ratio": -0.6761718988418579,
      "logits/chosen": -0.818164050579071,
      "logits/rejected": -0.7021484375,
      "logps/chosen": -0.733203113079071,
      "logps/rejected": -0.8658202886581421,
      "loss": 0.9954,
      "nll_loss": 0.987500011920929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07333984225988388,
      "rewards/margins": 0.01330718956887722,
      "rewards/rejected": -0.08659668266773224,
      "step": 5360
    },
    {
      "epoch": 0.2012328792790092,
      "grad_norm": 1.4886587871064383,
      "learning_rate": 1.0916988281898703e-06,
      "log_odds_chosen": 0.32746583223342896,
      "log_odds_ratio": -0.66455078125,
      "logits/chosen": -0.7406250238418579,
      "logits/rejected": -0.67919921875,
      "logps/chosen": -0.6802734136581421,
      "logps/rejected": -0.89453125,
      "loss": 0.9979,
      "nll_loss": 0.900585949420929,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.06801757961511612,
      "rewards/margins": 0.021475981920957565,
      "rewards/rejected": -0.08946533501148224,
      "step": 5370
    },
    {
      "epoch": 0.20160761462217308,
      "grad_norm": 1.475188013285329,
      "learning_rate": 1.0906837662984237e-06,
      "log_odds_chosen": 0.4634155333042145,
      "log_odds_ratio": -0.595898449420929,
      "logits/chosen": -0.763671875,
      "logits/rejected": -0.636523425579071,
      "logps/chosen": -0.696093738079071,
      "logps/rejected": -0.9906250238418579,
      "loss": 1.0105,
      "nll_loss": 0.957226574420929,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06965331733226776,
      "rewards/margins": 0.02943115308880806,
      "rewards/rejected": -0.09901122748851776,
      "step": 5380
    },
    {
      "epoch": 0.20198234996533698,
      "grad_norm": 1.5947947502539124,
      "learning_rate": 1.089671530566391e-06,
      "log_odds_chosen": 0.41302490234375,
      "log_odds_ratio": -0.61083984375,
      "logits/chosen": -0.7447265386581421,
      "logits/rejected": -0.6556640863418579,
      "logps/chosen": -0.6966797113418579,
      "logps/rejected": -0.971484363079071,
      "loss": 0.998,
      "nll_loss": 0.961132824420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06960449367761612,
      "rewards/margins": 0.02759857103228569,
      "rewards/rejected": -0.09724120795726776,
      "step": 5390
    },
    {
      "epoch": 0.20235708530850088,
      "grad_norm": 1.5653734291838861,
      "learning_rate": 1.0886621079036346e-06,
      "log_odds_chosen": 0.553698718547821,
      "log_odds_ratio": -0.583300769329071,
      "logits/chosen": -0.784960925579071,
      "logits/rejected": -0.6947265863418579,
      "logps/chosen": -0.7041991949081421,
      "logps/rejected": -1.066992163658142,
      "loss": 1.0364,
      "nll_loss": 0.906445324420929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07041015475988388,
      "rewards/margins": 0.03632201999425888,
      "rewards/rejected": -0.10679931938648224,
      "step": 5400
    },
    {
      "epoch": 0.20273182065166476,
      "grad_norm": 1.5153083554554272,
      "learning_rate": 1.0876554853047417e-06,
      "log_odds_chosen": 0.4747070372104645,
      "log_odds_ratio": -0.6143554449081421,
      "logits/chosen": -0.774218738079071,
      "logits/rejected": -0.6781250238418579,
      "logps/chosen": -0.746289074420929,
      "logps/rejected": -1.0613281726837158,
      "loss": 0.9929,
      "nll_loss": 0.9496093988418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07469482719898224,
      "rewards/margins": 0.031504057347774506,
      "rewards/rejected": -0.10617675632238388,
      "step": 5410
    },
    {
      "epoch": 0.20310655599482866,
      "grad_norm": 1.4072726254554946,
      "learning_rate": 1.0866516498483225e-06,
      "log_odds_chosen": 0.4104858338832855,
      "log_odds_ratio": -0.612011730670929,
      "logits/chosen": -0.7835937738418579,
      "logits/rejected": -0.710742175579071,
      "logps/chosen": -0.6875,
      "logps/rejected": -0.9476562738418579,
      "loss": 1.0207,
      "nll_loss": 0.87890625,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06875000149011612,
      "rewards/margins": 0.02599334716796875,
      "rewards/rejected": -0.0947265625,
      "step": 5420
    },
    {
      "epoch": 0.20348129133799253,
      "grad_norm": 1.4919871768481998,
      "learning_rate": 1.0856505886963116e-06,
      "log_odds_chosen": 0.38618165254592896,
      "log_odds_ratio": -0.634570300579071,
      "logits/chosen": -0.7914062738418579,
      "logits/rejected": -0.6744140386581421,
      "logps/chosen": -0.7520507574081421,
      "logps/rejected": -1.0146484375,
      "loss": 1.0102,
      "nll_loss": 0.9574218988418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07517089694738388,
      "rewards/margins": 0.026194000616669655,
      "rewards/rejected": -0.10136719048023224,
      "step": 5430
    },
    {
      "epoch": 0.20385602668115643,
      "grad_norm": 1.4968533456089825,
      "learning_rate": 1.0846522890932808e-06,
      "log_odds_chosen": 0.4017089903354645,
      "log_odds_ratio": -0.6156250238418579,
      "logits/chosen": -0.803515613079071,
      "logits/rejected": -0.699414074420929,
      "logps/chosen": -0.7357422113418579,
      "logps/rejected": -1.002539038658142,
      "loss": 0.9938,
      "nll_loss": 0.9410156011581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07353515923023224,
      "rewards/margins": 0.02670898474752903,
      "rewards/rejected": -0.10024414211511612,
      "step": 5440
    },
    {
      "epoch": 0.20423076202432033,
      "grad_norm": 1.525119537368698,
      "learning_rate": 1.0836567383657542e-06,
      "log_odds_chosen": 0.22633056342601776,
      "log_odds_ratio": -0.688281238079071,
      "logits/chosen": -0.747851550579071,
      "logits/rejected": -0.7040039300918579,
      "logps/chosen": -0.776562511920929,
      "logps/rejected": -0.908007800579071,
      "loss": 1.0137,
      "nll_loss": 0.9869140386581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07762451469898224,
      "rewards/margins": 0.013111114501953125,
      "rewards/rejected": -0.09075927734375,
      "step": 5450
    },
    {
      "epoch": 0.2046054973674842,
      "grad_norm": 1.4586562868237907,
      "learning_rate": 1.0826639239215334e-06,
      "log_odds_chosen": 0.25164794921875,
      "log_odds_ratio": -0.683300793170929,
      "logits/chosen": -0.727734386920929,
      "logits/rejected": -0.6670898199081421,
      "logps/chosen": -0.7611328363418579,
      "logps/rejected": -0.940234363079071,
      "loss": 1.0194,
      "nll_loss": 0.9769531488418579,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.07613525539636612,
      "rewards/margins": 0.017867278307676315,
      "rewards/rejected": -0.093994140625,
      "step": 5460
    },
    {
      "epoch": 0.2049802327106481,
      "grad_norm": 1.5436113489710273,
      "learning_rate": 1.0816738332490292e-06,
      "log_odds_chosen": 0.36126708984375,
      "log_odds_ratio": -0.636914074420929,
      "logits/chosen": -0.8080078363418579,
      "logits/rejected": -0.726367175579071,
      "logps/chosen": -0.7108398675918579,
      "logps/rejected": -0.9623047113418579,
      "loss": 1.0128,
      "nll_loss": 0.947460949420929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07103271782398224,
      "rewards/margins": 0.02509002760052681,
      "rewards/rejected": -0.09616699069738388,
      "step": 5470
    },
    {
      "epoch": 0.20535496805381198,
      "grad_norm": 1.5532051090402255,
      "learning_rate": 1.0806864539165982e-06,
      "log_odds_chosen": 0.4537353515625,
      "log_odds_ratio": -0.626171886920929,
      "logits/chosen": -0.7679687738418579,
      "logits/rejected": -0.705273449420929,
      "logps/chosen": -0.7787109613418579,
      "logps/rejected": -1.085351586341858,
      "loss": 1.023,
      "nll_loss": 1.022851586341858,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07786865532398224,
      "rewards/margins": 0.03069610521197319,
      "rewards/rejected": -0.10854492336511612,
      "step": 5480
    },
    {
      "epoch": 0.20572970339697588,
      "grad_norm": 1.603011450928993,
      "learning_rate": 1.0797017735718878e-06,
      "log_odds_chosen": 0.21556396782398224,
      "log_odds_ratio": -0.665820300579071,
      "logits/chosen": -0.7476562261581421,
      "logits/rejected": -0.6978515386581421,
      "logps/chosen": -0.769726574420929,
      "logps/rejected": -0.9125000238418579,
      "loss": 1.0087,
      "nll_loss": 0.9453125,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07698974758386612,
      "rewards/margins": 0.01426086388528347,
      "rewards/rejected": -0.09121093899011612,
      "step": 5490
    },
    {
      "epoch": 0.20610443874013978,
      "grad_norm": 1.5311090480769391,
      "learning_rate": 1.0787197799411874e-06,
      "log_odds_chosen": 0.3153076171875,
      "log_odds_ratio": -0.6748046875,
      "logits/chosen": -0.8056640625,
      "logits/rejected": -0.727343738079071,
      "logps/chosen": -0.7544921636581421,
      "logps/rejected": -0.9722656011581421,
      "loss": 1.0077,
      "nll_loss": 0.949999988079071,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07550048828125,
      "rewards/margins": 0.02175750769674778,
      "rewards/rejected": -0.09719238430261612,
      "step": 5500
    },
    {
      "epoch": 0.20647917408330366,
      "grad_norm": 1.5276188204484633,
      "learning_rate": 1.0777404608287846e-06,
      "log_odds_chosen": 0.514514148235321,
      "log_odds_ratio": -0.5835937261581421,
      "logits/chosen": -0.7867187261581421,
      "logits/rejected": -0.7203124761581421,
      "logps/chosen": -0.7109375,
      "logps/rejected": -1.0554687976837158,
      "loss": 1.0125,
      "nll_loss": 0.9603515863418579,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07105712592601776,
      "rewards/margins": 0.03452148288488388,
      "rewards/rejected": -0.1055908203125,
      "step": 5510
    },
    {
      "epoch": 0.20685390942646756,
      "grad_norm": 1.5576958737518514,
      "learning_rate": 1.0767638041163309e-06,
      "log_odds_chosen": 0.39165037870407104,
      "log_odds_ratio": -0.627734363079071,
      "logits/chosen": -0.769726574420929,
      "logits/rejected": -0.698437511920929,
      "logps/chosen": -0.716601550579071,
      "logps/rejected": -0.9833984375,
      "loss": 1.012,
      "nll_loss": 0.88671875,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07164306938648224,
      "rewards/margins": 0.02663726732134819,
      "rewards/rejected": -0.09831543266773224,
      "step": 5520
    },
    {
      "epoch": 0.20722864476963146,
      "grad_norm": 1.45690827484755,
      "learning_rate": 1.0757897977622107e-06,
      "log_odds_chosen": 0.38569337129592896,
      "log_odds_ratio": -0.625,
      "logits/chosen": -0.7567383050918579,
      "logits/rejected": -0.7242187261581421,
      "logps/chosen": -0.7298828363418579,
      "logps/rejected": -0.9761718511581421,
      "loss": 1.006,
      "nll_loss": 0.952343761920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07303466647863388,
      "rewards/margins": 0.024582672864198685,
      "rewards/rejected": -0.09758301079273224,
      "step": 5530
    },
    {
      "epoch": 0.20760338011279533,
      "grad_norm": 1.5998317714031944,
      "learning_rate": 1.074818429800918e-06,
      "log_odds_chosen": 0.3721069395542145,
      "log_odds_ratio": -0.639843761920929,
      "logits/chosen": -0.757617175579071,
      "logits/rejected": -0.709765613079071,
      "logps/chosen": -0.685742199420929,
      "logps/rejected": -0.9183593988418579,
      "loss": 0.9916,
      "nll_loss": 0.9330078363418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.068603515625,
      "rewards/margins": 0.023357391357421875,
      "rewards/rejected": -0.09184570610523224,
      "step": 5540
    },
    {
      "epoch": 0.20797811545595923,
      "grad_norm": 1.4154607318478718,
      "learning_rate": 1.073849688342439e-06,
      "log_odds_chosen": 0.42816162109375,
      "log_odds_ratio": -0.605761706829071,
      "logits/chosen": -0.7728515863418579,
      "logits/rejected": -0.682910144329071,
      "logps/chosen": -0.701367199420929,
      "logps/rejected": -0.9876953363418579,
      "loss": 0.9753,
      "nll_loss": 0.9595702886581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07016601413488388,
      "rewards/margins": 0.02865753136575222,
      "rewards/rejected": -0.098876953125,
      "step": 5550
    },
    {
      "epoch": 0.2083528507991231,
      "grad_norm": 1.4467105978637427,
      "learning_rate": 1.07288356157164e-06,
      "log_odds_chosen": 0.41517335176467896,
      "log_odds_ratio": -0.61279296875,
      "logits/chosen": -0.8251953125,
      "logits/rejected": -0.7328125238418579,
      "logps/chosen": -0.727343738079071,
      "logps/rejected": -1.020898461341858,
      "loss": 0.9757,
      "nll_loss": 0.898632824420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07272949069738388,
      "rewards/margins": 0.029389191418886185,
      "rewards/rejected": -0.10209961235523224,
      "step": 5560
    },
    {
      "epoch": 0.208727586142287,
      "grad_norm": 1.657304799079679,
      "learning_rate": 1.0719200377476648e-06,
      "log_odds_chosen": 0.31367188692092896,
      "log_odds_ratio": -0.66259765625,
      "logits/chosen": -0.7757812738418579,
      "logits/rejected": -0.693164050579071,
      "logps/chosen": -0.7076171636581421,
      "logps/rejected": -0.9150390625,
      "loss": 1.0146,
      "nll_loss": 0.9378906488418579,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07078857719898224,
      "rewards/margins": 0.02075347863137722,
      "rewards/rejected": -0.09146728366613388,
      "step": 5570
    },
    {
      "epoch": 0.2091023214854509,
      "grad_norm": 1.4973708042216782,
      "learning_rate": 1.0709591052033317e-06,
      "log_odds_chosen": 0.43537598848342896,
      "log_odds_ratio": -0.6119140386581421,
      "logits/chosen": -0.8095703125,
      "logits/rejected": -0.723437488079071,
      "logps/chosen": -0.673828125,
      "logps/rejected": -0.9740234613418579,
      "loss": 1.021,
      "nll_loss": 0.927539050579071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06737060844898224,
      "rewards/margins": 0.02991485595703125,
      "rewards/rejected": -0.09733887016773224,
      "step": 5580
    },
    {
      "epoch": 0.20947705682861478,
      "grad_norm": 1.3494261479087373,
      "learning_rate": 1.0700007523445435e-06,
      "log_odds_chosen": 0.5643554925918579,
      "log_odds_ratio": -0.5401366949081421,
      "logits/chosen": -0.814648449420929,
      "logits/rejected": -0.702441394329071,
      "logps/chosen": -0.677734375,
      "logps/rejected": -1.0099608898162842,
      "loss": 0.9743,
      "nll_loss": 0.90625,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06779785454273224,
      "rewards/margins": 0.03311767429113388,
      "rewards/rejected": -0.10100097954273224,
      "step": 5590
    },
    {
      "epoch": 0.20985179217177868,
      "grad_norm": 1.5420820227709149,
      "learning_rate": 1.0690449676496976e-06,
      "log_odds_chosen": 0.4448486268520355,
      "log_odds_ratio": -0.6112304925918579,
      "logits/chosen": -0.762890636920929,
      "logits/rejected": -0.673828125,
      "logps/chosen": -0.748828113079071,
      "logps/rejected": -1.025781273841858,
      "loss": 1.0204,
      "nll_loss": 0.9951171875,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07497558742761612,
      "rewards/margins": 0.027727508917450905,
      "rewards/rejected": -0.10261230170726776,
      "step": 5600
    },
    {
      "epoch": 0.21022652751494258,
      "grad_norm": 1.397601166513192,
      "learning_rate": 1.0680917396691054e-06,
      "log_odds_chosen": 0.35175782442092896,
      "log_odds_ratio": -0.642871081829071,
      "logits/chosen": -0.7748047113418579,
      "logits/rejected": -0.654003918170929,
      "logps/chosen": -0.717578113079071,
      "logps/rejected": -0.943164050579071,
      "loss": 1.0248,
      "nll_loss": 0.966601550579071,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07172851264476776,
      "rewards/margins": 0.02262420579791069,
      "rewards/rejected": -0.0943603515625,
      "step": 5610
    },
    {
      "epoch": 0.21060126285810646,
      "grad_norm": 1.4415621723230712,
      "learning_rate": 1.0671410570244164e-06,
      "log_odds_chosen": 0.3832641541957855,
      "log_odds_ratio": -0.6610351800918579,
      "logits/chosen": -0.7445312738418579,
      "logits/rejected": -0.6949218511581421,
      "logps/chosen": -0.746874988079071,
      "logps/rejected": -1.009374976158142,
      "loss": 1.0065,
      "nll_loss": 0.931640625,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07468261569738388,
      "rewards/margins": 0.02629852294921875,
      "rewards/rejected": -0.10100097954273224,
      "step": 5620
    },
    {
      "epoch": 0.21097599820127036,
      "grad_norm": 1.6291515824867797,
      "learning_rate": 1.0661929084080466e-06,
      "log_odds_chosen": 0.46708983182907104,
      "log_odds_ratio": -0.597851574420929,
      "logits/chosen": -0.75,
      "logits/rejected": -0.656445324420929,
      "logps/chosen": -0.70849609375,
      "logps/rejected": -0.9937499761581421,
      "loss": 1.0046,
      "nll_loss": 0.9580078125,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07082519680261612,
      "rewards/margins": 0.02853851392865181,
      "rewards/rejected": -0.09940185397863388,
      "step": 5630
    },
    {
      "epoch": 0.21135073354443423,
      "grad_norm": 1.5383647889250327,
      "learning_rate": 1.0652472825826149e-06,
      "log_odds_chosen": 0.47795408964157104,
      "log_odds_ratio": -0.6104491949081421,
      "logits/chosen": -0.7408202886581421,
      "logits/rejected": -0.6690429449081421,
      "logps/chosen": -0.708984375,
      "logps/rejected": -1.0167968273162842,
      "loss": 1.0002,
      "nll_loss": 0.915820300579071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07082519680261612,
      "rewards/margins": 0.030774688348174095,
      "rewards/rejected": -0.1016845703125,
      "step": 5640
    },
    {
      "epoch": 0.21172546888759813,
      "grad_norm": 1.4972097674542846,
      "learning_rate": 1.0643041683803828e-06,
      "log_odds_chosen": 0.530517578125,
      "log_odds_ratio": -0.571093738079071,
      "logits/chosen": -0.8160156011581421,
      "logits/rejected": -0.724804699420929,
      "logps/chosen": -0.7041015625,
      "logps/rejected": -1.0470702648162842,
      "loss": 0.9956,
      "nll_loss": 0.8910156488418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0704345703125,
      "rewards/margins": 0.034236907958984375,
      "rewards/rejected": -0.104736328125,
      "step": 5650
    },
    {
      "epoch": 0.21210020423076203,
      "grad_norm": 1.5507381003451315,
      "learning_rate": 1.063363554702701e-06,
      "log_odds_chosen": 0.5732421875,
      "log_odds_ratio": -0.5726562738418579,
      "logits/chosen": -0.779492199420929,
      "logits/rejected": -0.69775390625,
      "logps/chosen": -0.671875,
      "logps/rejected": -1.0499999523162842,
      "loss": 0.9906,
      "nll_loss": 0.9537109136581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06717529147863388,
      "rewards/margins": 0.03790435940027237,
      "rewards/rejected": -0.10501708835363388,
      "step": 5660
    },
    {
      "epoch": 0.2124749395739259,
      "grad_norm": 1.457349049774407,
      "learning_rate": 1.0624254305194609e-06,
      "log_odds_chosen": 0.34624022245407104,
      "log_odds_ratio": -0.6368163824081421,
      "logits/chosen": -0.855273425579071,
      "logits/rejected": -0.723437488079071,
      "logps/chosen": -0.7171875238418579,
      "logps/rejected": -0.9390624761581421,
      "loss": 0.9994,
      "nll_loss": 0.916210949420929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07178954780101776,
      "rewards/margins": 0.02215118333697319,
      "rewards/rejected": -0.09392090141773224,
      "step": 5670
    },
    {
      "epoch": 0.2128496749170898,
      "grad_norm": 1.6067989823736093,
      "learning_rate": 1.0614897848685505e-06,
      "log_odds_chosen": 0.32441407442092896,
      "log_odds_ratio": -0.647167980670929,
      "logits/chosen": -0.787109375,
      "logits/rejected": -0.672167956829071,
      "logps/chosen": -0.7027343511581421,
      "logps/rejected": -0.921875,
      "loss": 0.9954,
      "nll_loss": 0.9671875238418579,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07020263373851776,
      "rewards/margins": 0.02190094068646431,
      "rewards/rejected": -0.09215088188648224,
      "step": 5680
    },
    {
      "epoch": 0.21322441026025368,
      "grad_norm": 1.6073548708361605,
      "learning_rate": 1.0605566068553173e-06,
      "log_odds_chosen": 0.533642590045929,
      "log_odds_ratio": -0.585253894329071,
      "logits/chosen": -0.779296875,
      "logits/rejected": -0.6787109375,
      "logps/chosen": -0.7562500238418579,
      "logps/rejected": -1.111328125,
      "loss": 1.0149,
      "nll_loss": 0.9136718511581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07568359375,
      "rewards/margins": 0.03560485690832138,
      "rewards/rejected": -0.11115722358226776,
      "step": 5690
    },
    {
      "epoch": 0.21359914560341758,
      "grad_norm": 1.4712968091451017,
      "learning_rate": 1.0596258856520351e-06,
      "log_odds_chosen": 0.46605223417282104,
      "log_odds_ratio": -0.591503918170929,
      "logits/chosen": -0.8541015386581421,
      "logits/rejected": -0.7652343511581421,
      "logps/chosen": -0.7054687738418579,
      "logps/rejected": -1.0187499523162842,
      "loss": 1.0016,
      "nll_loss": 0.848437488079071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07054443657398224,
      "rewards/margins": 0.031305693089962006,
      "rewards/rejected": -0.10175780951976776,
      "step": 5700
    },
    {
      "epoch": 0.21397388094658149,
      "grad_norm": 1.4918221664802203,
      "learning_rate": 1.0586976104973764e-06,
      "log_odds_chosen": 0.43970948457717896,
      "log_odds_ratio": -0.60498046875,
      "logits/chosen": -0.7919921875,
      "logits/rejected": -0.7216796875,
      "logps/chosen": -0.775390625,
      "logps/rejected": -1.064062476158142,
      "loss": 1.0156,
      "nll_loss": 0.983593761920929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07749023288488388,
      "rewards/margins": 0.02873992919921875,
      "rewards/rejected": -0.1063232421875,
      "step": 5710
    },
    {
      "epoch": 0.21434861628974536,
      "grad_norm": 1.558707564372212,
      "learning_rate": 1.05777177069589e-06,
      "log_odds_chosen": 0.3391357362270355,
      "log_odds_ratio": -0.6475585699081421,
      "logits/chosen": -0.756054699420929,
      "logits/rejected": -0.720703125,
      "logps/chosen": -0.720898449420929,
      "logps/rejected": -0.9322265386581421,
      "loss": 1.0014,
      "nll_loss": 0.9232422113418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07207031548023224,
      "rewards/margins": 0.02114410325884819,
      "rewards/rejected": -0.09318847954273224,
      "step": 5720
    },
    {
      "epoch": 0.21472335163290926,
      "grad_norm": 1.5616751453223996,
      "learning_rate": 1.0568483556174834e-06,
      "log_odds_chosen": 0.49397581815719604,
      "log_odds_ratio": -0.5845702886581421,
      "logits/chosen": -0.804492175579071,
      "logits/rejected": -0.6982421875,
      "logps/chosen": -0.723437488079071,
      "logps/rejected": -1.0625,
      "loss": 0.9934,
      "nll_loss": 0.9439452886581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07235107570886612,
      "rewards/margins": 0.03389129787683487,
      "rewards/rejected": -0.10627441108226776,
      "step": 5730
    },
    {
      "epoch": 0.21509808697607316,
      "grad_norm": 1.5774234613969529,
      "learning_rate": 1.055927354696909e-06,
      "log_odds_chosen": 0.39265137910842896,
      "log_odds_ratio": -0.6485351324081421,
      "logits/chosen": -0.8013671636581421,
      "logits/rejected": -0.7220703363418579,
      "logps/chosen": -0.7408202886581421,
      "logps/rejected": -1.020898461341858,
      "loss": 1.0083,
      "nll_loss": 0.9986327886581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07410888373851776,
      "rewards/margins": 0.02806396409869194,
      "rewards/rejected": -0.10212402045726776,
      "step": 5740
    },
    {
      "epoch": 0.21547282231923703,
      "grad_norm": 1.6969140068204376,
      "learning_rate": 1.0550087574332592e-06,
      "log_odds_chosen": 0.46221923828125,
      "log_odds_ratio": -0.5960937738418579,
      "logits/chosen": -0.7451171875,
      "logits/rejected": -0.676562488079071,
      "logps/chosen": -0.704882800579071,
      "logps/rejected": -1.001562476158142,
      "loss": 1.0089,
      "nll_loss": 0.9722656011581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07052002102136612,
      "rewards/margins": 0.02959594689309597,
      "rewards/rejected": -0.10007324069738388,
      "step": 5750
    },
    {
      "epoch": 0.21584755766240094,
      "grad_norm": 1.6207676259264963,
      "learning_rate": 1.0540925533894598e-06,
      "log_odds_chosen": 0.4939331114292145,
      "log_odds_ratio": -0.5843750238418579,
      "logits/chosen": -0.8013671636581421,
      "logits/rejected": -0.694140613079071,
      "logps/chosen": -0.6820312738418579,
      "logps/rejected": -1.0021483898162842,
      "loss": 0.9928,
      "nll_loss": 0.880078136920929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.068115234375,
      "rewards/margins": 0.032109834253787994,
      "rewards/rejected": -0.10019531100988388,
      "step": 5760
    },
    {
      "epoch": 0.2162222930055648,
      "grad_norm": 1.3177053339492906,
      "learning_rate": 1.053178732191775e-06,
      "log_odds_chosen": 0.4389892518520355,
      "log_odds_ratio": -0.5990234613418579,
      "logits/chosen": -0.7525390386581421,
      "logits/rejected": -0.69287109375,
      "logps/chosen": -0.6651366949081421,
      "logps/rejected": -0.9462890625,
      "loss": 1.0064,
      "nll_loss": 0.944531261920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06644286960363388,
      "rewards/margins": 0.028116608038544655,
      "rewards/rejected": -0.09458007663488388,
      "step": 5770
    },
    {
      "epoch": 0.2165970283487287,
      "grad_norm": 1.4761219288800165,
      "learning_rate": 1.0522672835293127e-06,
      "log_odds_chosen": 0.25904542207717896,
      "log_odds_ratio": -0.6675781011581421,
      "logits/chosen": -0.8101562261581421,
      "logits/rejected": -0.7412109375,
      "logps/chosen": -0.7308593988418579,
      "logps/rejected": -0.8912109136581421,
      "loss": 0.9932,
      "nll_loss": 0.930468738079071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07308349758386612,
      "rewards/margins": 0.01602630689740181,
      "rewards/rejected": -0.08908691257238388,
      "step": 5780
    },
    {
      "epoch": 0.2169717636918926,
      "grad_norm": 1.4129560915093013,
      "learning_rate": 1.0513581971535365e-06,
      "log_odds_chosen": 0.38804930448532104,
      "log_odds_ratio": -0.6534179449081421,
      "logits/chosen": -0.774218738079071,
      "logits/rejected": -0.6932617425918579,
      "logps/chosen": -0.7685546875,
      "logps/rejected": -1.0390625,
      "loss": 0.9919,
      "nll_loss": 0.9683593511581421,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07685546576976776,
      "rewards/margins": 0.026998138055205345,
      "rewards/rejected": -0.10385742038488388,
      "step": 5790
    },
    {
      "epoch": 0.21734649903505648,
      "grad_norm": 1.5084894025587174,
      "learning_rate": 1.0504514628777803e-06,
      "log_odds_chosen": 0.3934692442417145,
      "log_odds_ratio": -0.64892578125,
      "logits/chosen": -0.798828125,
      "logits/rejected": -0.7314453125,
      "logps/chosen": -0.8021484613418579,
      "logps/rejected": -1.078125,
      "loss": 0.9989,
      "nll_loss": 0.9898437261581421,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08022461086511612,
      "rewards/margins": 0.02757415734231472,
      "rewards/rejected": -0.10776367038488388,
      "step": 5800
    },
    {
      "epoch": 0.21772123437822039,
      "grad_norm": 1.6207370774676593,
      "learning_rate": 1.0495470705767713e-06,
      "log_odds_chosen": 0.23820801079273224,
      "log_odds_ratio": -0.6993163824081421,
      "logits/chosen": -0.7798827886581421,
      "logits/rejected": -0.71435546875,
      "logps/chosen": -0.696484386920929,
      "logps/rejected": -0.855273425579071,
      "loss": 0.9837,
      "nll_loss": 0.901562511920929,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.06961669772863388,
      "rewards/margins": 0.01586151123046875,
      "rewards/rejected": -0.08554687350988388,
      "step": 5810
    },
    {
      "epoch": 0.21809596972138426,
      "grad_norm": 1.5481388168815635,
      "learning_rate": 1.0486450101861527e-06,
      "log_odds_chosen": 0.29106444120407104,
      "log_odds_ratio": -0.6905273199081421,
      "logits/chosen": -0.778613269329071,
      "logits/rejected": -0.7291015386581421,
      "logps/chosen": -0.7738281488418579,
      "logps/rejected": -0.993359386920929,
      "loss": 0.9794,
      "nll_loss": 0.9449218511581421,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07734374701976776,
      "rewards/margins": 0.02199096605181694,
      "rewards/rejected": -0.09940185397863388,
      "step": 5820
    },
    {
      "epoch": 0.21847070506454816,
      "grad_norm": 1.5020281889281257,
      "learning_rate": 1.0477452717020143e-06,
      "log_odds_chosen": 0.29884034395217896,
      "log_odds_ratio": -0.678515613079071,
      "logits/chosen": -0.796582043170929,
      "logits/rejected": -0.729687511920929,
      "logps/chosen": -0.7461913824081421,
      "logps/rejected": -0.944140613079071,
      "loss": 0.9829,
      "nll_loss": 0.9248046875,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.0745849609375,
      "rewards/margins": 0.01982421800494194,
      "rewards/rejected": -0.09443359076976776,
      "step": 5830
    },
    {
      "epoch": 0.21884544040771206,
      "grad_norm": 1.4980425470691225,
      "learning_rate": 1.0468478451804272e-06,
      "log_odds_chosen": 0.4291015565395355,
      "log_odds_ratio": -0.616992175579071,
      "logits/chosen": -0.782421886920929,
      "logits/rejected": -0.708203136920929,
      "logps/chosen": -0.6976562738418579,
      "logps/rejected": -0.997851550579071,
      "loss": 0.9897,
      "nll_loss": 0.917187511920929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06984863430261612,
      "rewards/margins": 0.029996490105986595,
      "rewards/rejected": -0.09978027641773224,
      "step": 5840
    },
    {
      "epoch": 0.21922017575087593,
      "grad_norm": 1.6613963671388272,
      "learning_rate": 1.0459527207369814e-06,
      "log_odds_chosen": 0.4384765625,
      "log_odds_ratio": -0.6307617425918579,
      "logits/chosen": -0.755664050579071,
      "logits/rejected": -0.6937500238418579,
      "logps/chosen": -0.70556640625,
      "logps/rejected": -1.0031249523162842,
      "loss": 1.012,
      "nll_loss": 0.937304675579071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07053222507238388,
      "rewards/margins": 0.02973327599465847,
      "rewards/rejected": -0.10035400092601776,
      "step": 5850
    },
    {
      "epoch": 0.21959491109403984,
      "grad_norm": 1.5097285493015438,
      "learning_rate": 1.0450598885463281e-06,
      "log_odds_chosen": 0.3309570252895355,
      "log_odds_ratio": -0.6285156011581421,
      "logits/chosen": -0.8570312261581421,
      "logits/rejected": -0.783203125,
      "logps/chosen": -0.681835949420929,
      "logps/rejected": -0.883984386920929,
      "loss": 0.9902,
      "nll_loss": 0.84765625,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06815185397863388,
      "rewards/margins": 0.020241547375917435,
      "rewards/rejected": -0.08841552585363388,
      "step": 5860
    },
    {
      "epoch": 0.21996964643720374,
      "grad_norm": 1.4168449604446438,
      "learning_rate": 1.0441693388417282e-06,
      "log_odds_chosen": 0.46806639432907104,
      "log_odds_ratio": -0.6092773675918579,
      "logits/chosen": -0.8013671636581421,
      "logits/rejected": -0.683886706829071,
      "logps/chosen": -0.7275390625,
      "logps/rejected": -1.0595703125,
      "loss": 0.9762,
      "nll_loss": 0.9419921636581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07270507514476776,
      "rewards/margins": 0.03327484056353569,
      "rewards/rejected": -0.10595703125,
      "step": 5870
    },
    {
      "epoch": 0.2203443817803676,
      "grad_norm": 1.4620014142528812,
      "learning_rate": 1.0432810619146023e-06,
      "log_odds_chosen": 0.42835694551467896,
      "log_odds_ratio": -0.60791015625,
      "logits/chosen": -0.812304675579071,
      "logits/rejected": -0.7627929449081421,
      "logps/chosen": -0.7510741949081421,
      "logps/rejected": -1.023046851158142,
      "loss": 1.0004,
      "nll_loss": 0.9466797113418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07513427734375,
      "rewards/margins": 0.02728576585650444,
      "rewards/rejected": -0.10244140774011612,
      "step": 5880
    },
    {
      "epoch": 0.2207191171235315,
      "grad_norm": 1.443053620776168,
      "learning_rate": 1.042395048114086e-06,
      "log_odds_chosen": 0.49664306640625,
      "log_odds_ratio": -0.601269543170929,
      "logits/chosen": -0.7884765863418579,
      "logits/rejected": -0.7060546875,
      "logps/chosen": -0.6917968988418579,
      "logps/rejected": -1.0265624523162842,
      "loss": 1.0049,
      "nll_loss": 0.9468749761581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06916503608226776,
      "rewards/margins": 0.03348236158490181,
      "rewards/rejected": -0.10261230170726776,
      "step": 5890
    },
    {
      "epoch": 0.22109385246669538,
      "grad_norm": 1.4170210368035905,
      "learning_rate": 1.041511287846591e-06,
      "log_odds_chosen": 0.41704100370407104,
      "log_odds_ratio": -0.614550769329071,
      "logits/chosen": -0.8277343511581421,
      "logits/rejected": -0.7396484613418579,
      "logps/chosen": -0.7494140863418579,
      "logps/rejected": -1.000585913658142,
      "loss": 0.9852,
      "nll_loss": 0.951171875,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07492675632238388,
      "rewards/margins": 0.02518310584127903,
      "rewards/rejected": -0.10014648735523224,
      "step": 5900
    },
    {
      "epoch": 0.22146858780985929,
      "grad_norm": 1.3955274205314065,
      "learning_rate": 1.0406297715753674e-06,
      "log_odds_chosen": 0.556323230266571,
      "log_odds_ratio": -0.5716797113418579,
      "logits/chosen": -0.8023437261581421,
      "logits/rejected": -0.694140613079071,
      "logps/chosen": -0.6939452886581421,
      "logps/rejected": -1.0486328601837158,
      "loss": 0.9906,
      "nll_loss": 0.8919922113418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06934814155101776,
      "rewards/margins": 0.035494230687618256,
      "rewards/rejected": -0.10493163764476776,
      "step": 5910
    },
    {
      "epoch": 0.2218433231530232,
      "grad_norm": 1.6592412077365388,
      "learning_rate": 1.0397504898200726e-06,
      "log_odds_chosen": 0.31289368867874146,
      "log_odds_ratio": -0.65478515625,
      "logits/chosen": -0.7992187738418579,
      "logits/rejected": -0.703320324420929,
      "logps/chosen": -0.695605456829071,
      "logps/rejected": -0.928515613079071,
      "loss": 0.9826,
      "nll_loss": 0.9330078363418579,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.06951904296875,
      "rewards/margins": 0.023288344964385033,
      "rewards/rejected": -0.09281005710363388,
      "step": 5920
    },
    {
      "epoch": 0.22221805849618706,
      "grad_norm": 1.458484248703689,
      "learning_rate": 1.0388734331563415e-06,
      "log_odds_chosen": 0.47480469942092896,
      "log_odds_ratio": -0.599316418170929,
      "logits/chosen": -0.7767578363418579,
      "logits/rejected": -0.7171875238418579,
      "logps/chosen": -0.702343761920929,
      "logps/rejected": -1.0,
      "loss": 0.9829,
      "nll_loss": 0.9457031488418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07033691555261612,
      "rewards/margins": 0.0297698974609375,
      "rewards/rejected": -0.0999755859375,
      "step": 5930
    },
    {
      "epoch": 0.22259279383935096,
      "grad_norm": 1.4878688054010998,
      "learning_rate": 1.037998592215364e-06,
      "log_odds_chosen": 0.37822264432907104,
      "log_odds_ratio": -0.632128894329071,
      "logits/chosen": -0.7587890625,
      "logits/rejected": -0.699414074420929,
      "logps/chosen": -0.7349609136581421,
      "logps/rejected": -0.9898437261581421,
      "loss": 0.9925,
      "nll_loss": 0.9462890625,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07349853217601776,
      "rewards/margins": 0.02553253248333931,
      "rewards/rejected": -0.09896240383386612,
      "step": 5940
    },
    {
      "epoch": 0.22296752918251486,
      "grad_norm": 1.447810279363478,
      "learning_rate": 1.037125957683463e-06,
      "log_odds_chosen": 0.45111083984375,
      "log_odds_ratio": -0.6064453125,
      "logits/chosen": -0.777148425579071,
      "logits/rejected": -0.6903320550918579,
      "logps/chosen": -0.67431640625,
      "logps/rejected": -0.972460925579071,
      "loss": 1.0135,
      "nll_loss": 0.9095703363418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06749267876148224,
      "rewards/margins": 0.02981414832174778,
      "rewards/rejected": -0.0972900390625,
      "step": 5950
    },
    {
      "epoch": 0.22334226452567874,
      "grad_norm": 1.4735301962824203,
      "learning_rate": 1.0362555203016794e-06,
      "log_odds_chosen": 0.4572997987270355,
      "log_odds_ratio": -0.611523449420929,
      "logits/chosen": -0.7230468988418579,
      "logits/rejected": -0.666699230670929,
      "logps/chosen": -0.7724609375,
      "logps/rejected": -1.0769531726837158,
      "loss": 0.9896,
      "nll_loss": 0.9740234613418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07719726860523224,
      "rewards/margins": 0.03046112135052681,
      "rewards/rejected": -0.10769043117761612,
      "step": 5960
    },
    {
      "epoch": 0.22371699986884264,
      "grad_norm": 1.5011567218073574,
      "learning_rate": 1.035387270865359e-06,
      "log_odds_chosen": 0.4573608338832855,
      "log_odds_ratio": -0.6021484136581421,
      "logits/chosen": -0.780078113079071,
      "logits/rejected": -0.701953113079071,
      "logps/chosen": -0.7255859375,
      "logps/rejected": -1.024804711341858,
      "loss": 0.9765,
      "nll_loss": 0.896679699420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07258300483226776,
      "rewards/margins": 0.029903411865234375,
      "rewards/rejected": -0.10247802734375,
      "step": 5970
    },
    {
      "epoch": 0.2240917352120065,
      "grad_norm": 1.4618519815177857,
      "learning_rate": 1.0345212002237434e-06,
      "log_odds_chosen": 0.4058471620082855,
      "log_odds_ratio": -0.6241210699081421,
      "logits/chosen": -0.828906238079071,
      "logits/rejected": -0.7435547113418579,
      "logps/chosen": -0.6917968988418579,
      "logps/rejected": -0.971875011920929,
      "loss": 0.9688,
      "nll_loss": 0.9498046636581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06914062798023224,
      "rewards/margins": 0.02805175818502903,
      "rewards/rejected": -0.09716796875,
      "step": 5980
    },
    {
      "epoch": 0.2244664705551704,
      "grad_norm": 1.5497497269076637,
      "learning_rate": 1.0336572992795644e-06,
      "log_odds_chosen": 0.3843750059604645,
      "log_odds_ratio": -0.6102539300918579,
      "logits/chosen": -0.771484375,
      "logits/rejected": -0.7115234136581421,
      "logps/chosen": -0.6654297113418579,
      "logps/rejected": -0.9105468988418579,
      "loss": 0.9734,
      "nll_loss": 0.912890613079071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06656493991613388,
      "rewards/margins": 0.024443816393613815,
      "rewards/rejected": -0.09099121391773224,
      "step": 5990
    },
    {
      "epoch": 0.2248412058983343,
      "grad_norm": 1.4633157430488146,
      "learning_rate": 1.0327955589886444e-06,
      "log_odds_chosen": 0.41999512910842896,
      "log_odds_ratio": -0.600878894329071,
      "logits/chosen": -0.7919921875,
      "logits/rejected": -0.7247070074081421,
      "logps/chosen": -0.653613269329071,
      "logps/rejected": -0.920117199420929,
      "loss": 0.9745,
      "nll_loss": 0.8814452886581421,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06538085639476776,
      "rewards/margins": 0.02668151818215847,
      "rewards/rejected": -0.092041015625,
      "step": 6000
    },
    {
      "epoch": 0.22521594124149819,
      "grad_norm": 1.4440165317435083,
      "learning_rate": 1.0319359703594971e-06,
      "log_odds_chosen": 0.41473388671875,
      "log_odds_ratio": -0.610156238079071,
      "logits/chosen": -0.7486327886581421,
      "logits/rejected": -0.690234363079071,
      "logps/chosen": -0.7066406011581421,
      "logps/rejected": -0.9652343988418579,
      "loss": 0.9613,
      "nll_loss": 0.869921863079071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0706787109375,
      "rewards/margins": 0.025829315185546875,
      "rewards/rejected": -0.0965576171875,
      "step": 6010
    },
    {
      "epoch": 0.2255906765846621,
      "grad_norm": 1.5018037601560863,
      "learning_rate": 1.0310785244529341e-06,
      "log_odds_chosen": 0.35792237520217896,
      "log_odds_ratio": -0.668261706829071,
      "logits/chosen": -0.7828124761581421,
      "logits/rejected": -0.7236328125,
      "logps/chosen": -0.729296863079071,
      "logps/rejected": -0.9886718988418579,
      "loss": 1.0052,
      "nll_loss": 0.9671875238418579,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07290039211511612,
      "rewards/margins": 0.02596740797162056,
      "rewards/rejected": -0.09885253757238388,
      "step": 6020
    },
    {
      "epoch": 0.22596541192782596,
      "grad_norm": 1.4561555951420322,
      "learning_rate": 1.0302232123816746e-06,
      "log_odds_chosen": 0.5208740234375,
      "log_odds_ratio": -0.6107422113418579,
      "logits/chosen": -0.83642578125,
      "logits/rejected": -0.735546886920929,
      "logps/chosen": -0.721484363079071,
      "logps/rejected": -1.073339819908142,
      "loss": 0.9847,
      "nll_loss": 0.9253906011581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07210693508386612,
      "rewards/margins": 0.03523407131433487,
      "rewards/rejected": -0.1072998046875,
      "step": 6030
    },
    {
      "epoch": 0.22634014727098986,
      "grad_norm": 1.3716301008656524,
      "learning_rate": 1.0293700253099576e-06,
      "log_odds_chosen": 0.4629272520542145,
      "log_odds_ratio": -0.5967773199081421,
      "logits/chosen": -0.798535168170929,
      "logits/rejected": -0.72314453125,
      "logps/chosen": -0.6947265863418579,
      "logps/rejected": -0.989453136920929,
      "loss": 0.9905,
      "nll_loss": 0.9281250238418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06944580376148224,
      "rewards/margins": 0.02951812744140625,
      "rewards/rejected": -0.0989990234375,
      "step": 6040
    },
    {
      "epoch": 0.22671488261415376,
      "grad_norm": 1.4981892676515998,
      "learning_rate": 1.02851895445316e-06,
      "log_odds_chosen": 0.3902343809604645,
      "log_odds_ratio": -0.606249988079071,
      "logits/chosen": -0.7972656488418579,
      "logits/rejected": -0.700488269329071,
      "logps/chosen": -0.672167956829071,
      "logps/rejected": -0.9125000238418579,
      "loss": 0.9565,
      "nll_loss": 0.8939453363418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06723632663488388,
      "rewards/margins": 0.02398376539349556,
      "rewards/rejected": -0.09123535454273224,
      "step": 6050
    },
    {
      "epoch": 0.22708961795731764,
      "grad_norm": 1.527604063132908,
      "learning_rate": 1.0276699910774159e-06,
      "log_odds_chosen": 0.39271241426467896,
      "log_odds_ratio": -0.623242199420929,
      "logits/chosen": -0.8216797113418579,
      "logits/rejected": -0.722949206829071,
      "logps/chosen": -0.6830078363418579,
      "logps/rejected": -0.9566406011581421,
      "loss": 0.9844,
      "nll_loss": 0.855273425579071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06832275539636612,
      "rewards/margins": 0.02736205980181694,
      "rewards/rejected": -0.09565429389476776,
      "step": 6060
    },
    {
      "epoch": 0.22746435330048154,
      "grad_norm": 1.4401925223618977,
      "learning_rate": 1.0268231264992398e-06,
      "log_odds_chosen": 0.430938720703125,
      "log_odds_ratio": -0.629589855670929,
      "logits/chosen": -0.7914062738418579,
      "logits/rejected": -0.7132812738418579,
      "logps/chosen": -0.7535156011581421,
      "logps/rejected": -1.0509765148162842,
      "loss": 0.9816,
      "nll_loss": 1.0246093273162842,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07539062201976776,
      "rewards/margins": 0.02971496619284153,
      "rewards/rejected": -0.1051025390625,
      "step": 6070
    },
    {
      "epoch": 0.22783908864364544,
      "grad_norm": 1.457538700773583,
      "learning_rate": 1.0259783520851542e-06,
      "log_odds_chosen": 0.37519532442092896,
      "log_odds_ratio": -0.638671875,
      "logits/chosen": -0.8169921636581421,
      "logits/rejected": -0.739062488079071,
      "logps/chosen": -0.666015625,
      "logps/rejected": -0.9193359613418579,
      "loss": 0.9738,
      "nll_loss": 0.917773425579071,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06666259467601776,
      "rewards/margins": 0.025281524285674095,
      "rewards/rejected": -0.09189452975988388,
      "step": 6080
    },
    {
      "epoch": 0.2282138239868093,
      "grad_norm": 1.4261268855800702,
      "learning_rate": 1.0251356592513193e-06,
      "log_odds_chosen": 0.24586181342601776,
      "log_odds_ratio": -0.6666015386581421,
      "logits/chosen": -0.796875,
      "logits/rejected": -0.73828125,
      "logps/chosen": -0.7476562261581421,
      "logps/rejected": -0.9037109613418579,
      "loss": 1.0181,
      "nll_loss": 0.9927734136581421,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07467041164636612,
      "rewards/margins": 0.015657806769013405,
      "rewards/rejected": -0.09033203125,
      "step": 6090
    },
    {
      "epoch": 0.2285885593299732,
      "grad_norm": 1.8869226485093937,
      "learning_rate": 1.0242950394631678e-06,
      "log_odds_chosen": 0.366455078125,
      "log_odds_ratio": -0.642382800579071,
      "logits/chosen": -0.7701171636581421,
      "logits/rejected": -0.7406250238418579,
      "logps/chosen": -0.7215820550918579,
      "logps/rejected": -0.974609375,
      "loss": 0.9831,
      "nll_loss": 0.8900390863418579,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07220458984375,
      "rewards/margins": 0.02535095252096653,
      "rewards/rejected": -0.09755859524011612,
      "step": 6100
    },
    {
      "epoch": 0.22896329467313709,
      "grad_norm": 1.5033989162938561,
      "learning_rate": 1.0234564842350404e-06,
      "log_odds_chosen": 0.606494128704071,
      "log_odds_ratio": -0.541308581829071,
      "logits/chosen": -0.8443359136581421,
      "logits/rejected": -0.722851574420929,
      "logps/chosen": -0.690722644329071,
      "logps/rejected": -1.070703148841858,
      "loss": 0.9979,
      "nll_loss": 0.8662109375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06905517727136612,
      "rewards/margins": 0.03804779052734375,
      "rewards/rejected": -0.10708007961511612,
      "step": 6110
    },
    {
      "epoch": 0.229338030016301,
      "grad_norm": 1.501010098873475,
      "learning_rate": 1.0226199851298272e-06,
      "log_odds_chosen": 0.48424071073532104,
      "log_odds_ratio": -0.58837890625,
      "logits/chosen": -0.7958984375,
      "logits/rejected": -0.68603515625,
      "logps/chosen": -0.681835949420929,
      "logps/rejected": -1.0144531726837158,
      "loss": 0.9859,
      "nll_loss": 0.903124988079071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06820068508386612,
      "rewards/margins": 0.033184051513671875,
      "rewards/rejected": -0.10136719048023224,
      "step": 6120
    },
    {
      "epoch": 0.2297127653594649,
      "grad_norm": 1.4860632859969247,
      "learning_rate": 1.0217855337586106e-06,
      "log_odds_chosen": 0.3866027891635895,
      "log_odds_ratio": -0.6439453363418579,
      "logits/chosen": -0.758593738079071,
      "logits/rejected": -0.713085949420929,
      "logps/chosen": -0.7396484613418579,
      "logps/rejected": -0.9986327886581421,
      "loss": 0.9747,
      "nll_loss": 0.8851562738418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07399902492761612,
      "rewards/margins": 0.02590179443359375,
      "rewards/rejected": -0.099853515625,
      "step": 6130
    },
    {
      "epoch": 0.23008750070262876,
      "grad_norm": 1.56367624333533,
      "learning_rate": 1.0209531217803119e-06,
      "log_odds_chosen": 0.4864501953125,
      "log_odds_ratio": -0.5814453363418579,
      "logits/chosen": -0.796679675579071,
      "logits/rejected": -0.7412109375,
      "logps/chosen": -0.750195324420929,
      "logps/rejected": -1.0798828601837158,
      "loss": 0.9781,
      "nll_loss": 0.8529297113418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07498779147863388,
      "rewards/margins": 0.03291168063879013,
      "rewards/rejected": -0.10791015625,
      "step": 6140
    },
    {
      "epoch": 0.23046223604579266,
      "grad_norm": 1.5792168067536576,
      "learning_rate": 1.0201227409013412e-06,
      "log_odds_chosen": 0.2669921815395355,
      "log_odds_ratio": -0.6636718511581421,
      "logits/chosen": -0.806640625,
      "logits/rejected": -0.7294921875,
      "logps/chosen": -0.7007812261581421,
      "logps/rejected": -0.8853515386581421,
      "loss": 0.9968,
      "nll_loss": 0.965039074420929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07011719048023224,
      "rewards/margins": 0.01841430738568306,
      "rewards/rejected": -0.08851318061351776,
      "step": 6150
    },
    {
      "epoch": 0.23083697138895656,
      "grad_norm": 1.5848836460135842,
      "learning_rate": 1.0192943828752509e-06,
      "log_odds_chosen": 0.4742187559604645,
      "log_odds_ratio": -0.5914062261581421,
      "logits/chosen": -0.7757812738418579,
      "logits/rejected": -0.712109386920929,
      "logps/chosen": -0.7059570550918579,
      "logps/rejected": -1.00390625,
      "loss": 0.9845,
      "nll_loss": 0.899609386920929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07056884467601776,
      "rewards/margins": 0.02982482872903347,
      "rewards/rejected": -0.10039062798023224,
      "step": 6160
    },
    {
      "epoch": 0.23121170673212044,
      "grad_norm": 1.3910049416696846,
      "learning_rate": 1.0184680395023912e-06,
      "log_odds_chosen": 0.610668957233429,
      "log_odds_ratio": -0.536914050579071,
      "logits/chosen": -0.7939453125,
      "logits/rejected": -0.7193359136581421,
      "logps/chosen": -0.6787109375,
      "logps/rejected": -1.077539086341858,
      "loss": 0.9519,
      "nll_loss": 0.8974609375,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06785888969898224,
      "rewards/margins": 0.03986511379480362,
      "rewards/rejected": -0.107666015625,
      "step": 6170
    },
    {
      "epoch": 0.23158644207528434,
      "grad_norm": 1.4451236817053672,
      "learning_rate": 1.0176437026295688e-06,
      "log_odds_chosen": 0.674029529094696,
      "log_odds_ratio": -0.53759765625,
      "logits/chosen": -0.859570324420929,
      "logits/rejected": -0.733203113079071,
      "logps/chosen": -0.720703125,
      "logps/rejected": -1.174218773841858,
      "loss": 0.9512,
      "nll_loss": 0.861328125,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07204589992761612,
      "rewards/margins": 0.04532776027917862,
      "rewards/rejected": -0.11735840141773224,
      "step": 6180
    },
    {
      "epoch": 0.2319611774184482,
      "grad_norm": 1.4430801722032678,
      "learning_rate": 1.0168213641497094e-06,
      "log_odds_chosen": 0.21989746391773224,
      "log_odds_ratio": -0.69140625,
      "logits/chosen": -0.8193359375,
      "logits/rejected": -0.765820324420929,
      "logps/chosen": -0.7466796636581421,
      "logps/rejected": -0.916796863079071,
      "loss": 0.9928,
      "nll_loss": 0.884960949420929,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07463379204273224,
      "rewards/margins": 0.016971588134765625,
      "rewards/rejected": -0.09166260063648224,
      "step": 6190
    },
    {
      "epoch": 0.2323359127616121,
      "grad_norm": 1.656658375117593,
      "learning_rate": 1.016001016001524e-06,
      "log_odds_chosen": 0.6141601800918579,
      "log_odds_ratio": -0.572949230670929,
      "logits/chosen": -0.7943359613418579,
      "logits/rejected": -0.71875,
      "logps/chosen": -0.7154296636581421,
      "logps/rejected": -1.133203148841858,
      "loss": 0.9718,
      "nll_loss": 0.879101574420929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07154540717601776,
      "rewards/margins": 0.04184722900390625,
      "rewards/rejected": -0.11335448920726776,
      "step": 6200
    },
    {
      "epoch": 0.232710648104776,
      "grad_norm": 1.614906193445056,
      "learning_rate": 1.0151826501691747e-06,
      "log_odds_chosen": 0.4793701171875,
      "log_odds_ratio": -0.5908203125,
      "logits/chosen": -0.763476550579071,
      "logits/rejected": -0.640820324420929,
      "logps/chosen": -0.70703125,
      "logps/rejected": -0.9984375238418579,
      "loss": 0.9965,
      "nll_loss": 0.922070324420929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07069091498851776,
      "rewards/margins": 0.029170989990234375,
      "rewards/rejected": -0.09987793117761612,
      "step": 6210
    },
    {
      "epoch": 0.2330853834479399,
      "grad_norm": 1.4974963124233074,
      "learning_rate": 1.0143662586819475e-06,
      "log_odds_chosen": 0.49848634004592896,
      "log_odds_ratio": -0.6229492425918579,
      "logits/chosen": -0.7916015386581421,
      "logits/rejected": -0.6878906488418579,
      "logps/chosen": -0.731738269329071,
      "logps/rejected": -1.0693359375,
      "loss": 0.963,
      "nll_loss": 0.954296886920929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07321777194738388,
      "rewards/margins": 0.03369445726275444,
      "rewards/rejected": -0.10686035454273224,
      "step": 6220
    },
    {
      "epoch": 0.2334601187911038,
      "grad_norm": 1.4249597029092618,
      "learning_rate": 1.0135518336139257e-06,
      "log_odds_chosen": 0.40483397245407104,
      "log_odds_ratio": -0.640429675579071,
      "logits/chosen": -0.748730480670929,
      "logits/rejected": -0.6541992425918579,
      "logps/chosen": -0.716503918170929,
      "logps/rejected": -0.9732421636581421,
      "loss": 0.967,
      "nll_loss": 0.9312499761581421,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07166747748851776,
      "rewards/margins": 0.02560119703412056,
      "rewards/rejected": -0.09725341945886612,
      "step": 6230
    },
    {
      "epoch": 0.23383485413426766,
      "grad_norm": 1.5612454630457147,
      "learning_rate": 1.0127393670836666e-06,
      "log_odds_chosen": 0.44261473417282104,
      "log_odds_ratio": -0.613476574420929,
      "logits/chosen": -0.7392578125,
      "logits/rejected": -0.6744140386581421,
      "logps/chosen": -0.685253918170929,
      "logps/rejected": -0.9615234136581421,
      "loss": 0.9738,
      "nll_loss": 0.93359375,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06844482570886612,
      "rewards/margins": 0.02758941613137722,
      "rewards/rejected": -0.09611816704273224,
      "step": 6240
    },
    {
      "epoch": 0.23420958947743156,
      "grad_norm": 1.5653433774625873,
      "learning_rate": 1.0119288512538813e-06,
      "log_odds_chosen": 0.33447265625,
      "log_odds_ratio": -0.624707043170929,
      "logits/chosen": -0.7431640625,
      "logits/rejected": -0.6678711175918579,
      "logps/chosen": -0.753710925579071,
      "logps/rejected": -0.962890625,
      "loss": 0.9734,
      "nll_loss": 0.892382800579071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07539062201976776,
      "rewards/margins": 0.02093505859375,
      "rewards/rejected": -0.09628906100988388,
      "step": 6250
    },
    {
      "epoch": 0.23458432482059546,
      "grad_norm": 1.4904500649348906,
      "learning_rate": 1.0111202783311173e-06,
      "log_odds_chosen": 0.3804565370082855,
      "log_odds_ratio": -0.638476550579071,
      "logits/chosen": -0.7894531488418579,
      "logits/rejected": -0.7139648199081421,
      "logps/chosen": -0.662890613079071,
      "logps/rejected": -0.908984363079071,
      "loss": 0.9671,
      "nll_loss": 0.9234374761581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06625976413488388,
      "rewards/margins": 0.02453460730612278,
      "rewards/rejected": -0.09080810844898224,
      "step": 6260
    },
    {
      "epoch": 0.23495906016375934,
      "grad_norm": 1.402042149401425,
      "learning_rate": 1.010313640565443e-06,
      "log_odds_chosen": 0.3362060487270355,
      "log_odds_ratio": -0.646289050579071,
      "logits/chosen": -0.803906261920929,
      "logits/rejected": -0.7251952886581421,
      "logps/chosen": -0.7337890863418579,
      "logps/rejected": -0.980664074420929,
      "loss": 0.9792,
      "nll_loss": 0.919140636920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07341308891773224,
      "rewards/margins": 0.02470245398581028,
      "rewards/rejected": -0.09814453125,
      "step": 6270
    },
    {
      "epoch": 0.23533379550692324,
      "grad_norm": 1.4829040374173414,
      "learning_rate": 1.0095089302501373e-06,
      "log_odds_chosen": 0.33381348848342896,
      "log_odds_ratio": -0.6666015386581421,
      "logits/chosen": -0.783886730670929,
      "logits/rejected": -0.705761730670929,
      "logps/chosen": -0.7412109375,
      "logps/rejected": -0.9814453125,
      "loss": 0.9792,
      "nll_loss": 0.929492175579071,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.07416991889476776,
      "rewards/margins": 0.024034881964325905,
      "rewards/rejected": -0.09816894680261612,
      "step": 6280
    },
    {
      "epoch": 0.23570853085008714,
      "grad_norm": 1.626604850743254,
      "learning_rate": 1.0087061397213787e-06,
      "log_odds_chosen": 0.531811535358429,
      "log_odds_ratio": -0.590624988079071,
      "logits/chosen": -0.780468761920929,
      "logits/rejected": -0.719531238079071,
      "logps/chosen": -0.6595703363418579,
      "logps/rejected": -1.0027344226837158,
      "loss": 0.9948,
      "nll_loss": 0.880078136920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06595458835363388,
      "rewards/margins": 0.03423309326171875,
      "rewards/rejected": -0.10020752251148224,
      "step": 6290
    },
    {
      "epoch": 0.236083266193251,
      "grad_norm": 1.5455589607892959,
      "learning_rate": 1.007905261357939e-06,
      "log_odds_chosen": 0.4056030213832855,
      "log_odds_ratio": -0.620898425579071,
      "logits/chosen": -0.7689453363418579,
      "logits/rejected": -0.7445312738418579,
      "logps/chosen": -0.654980480670929,
      "logps/rejected": -0.8812500238418579,
      "loss": 0.9933,
      "nll_loss": 0.900390625,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06551513820886612,
      "rewards/margins": 0.02268371544778347,
      "rewards/rejected": -0.08820800483226776,
      "step": 6300
    },
    {
      "epoch": 0.2364580015364149,
      "grad_norm": 1.567033596207803,
      "learning_rate": 1.0071062875808811e-06,
      "log_odds_chosen": 0.522387683391571,
      "log_odds_ratio": -0.569042980670929,
      "logits/chosen": -0.770800769329071,
      "logits/rejected": -0.671582043170929,
      "logps/chosen": -0.702343761920929,
      "logps/rejected": -1.046289086341858,
      "loss": 0.9991,
      "nll_loss": 0.9820312261581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07023926079273224,
      "rewards/margins": 0.034313201904296875,
      "rewards/rejected": -0.10456542670726776,
      "step": 6310
    },
    {
      "epoch": 0.2368327368795788,
      "grad_norm": 1.492549894494993,
      "learning_rate": 1.0063092108532552e-06,
      "log_odds_chosen": 0.5068359375,
      "log_odds_ratio": -0.5750976800918579,
      "logits/chosen": -0.8304687738418579,
      "logits/rejected": -0.743359386920929,
      "logps/chosen": -0.6591796875,
      "logps/rejected": -0.9697265625,
      "loss": 0.9759,
      "nll_loss": 0.8531249761581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06591796875,
      "rewards/margins": 0.030991364270448685,
      "rewards/rejected": -0.09694824367761612,
      "step": 6320
    },
    {
      "epoch": 0.2372074722227427,
      "grad_norm": 1.5438190067000273,
      "learning_rate": 1.005514023679802e-06,
      "log_odds_chosen": 0.3165283203125,
      "log_odds_ratio": -0.6288086175918579,
      "logits/chosen": -0.8072265386581421,
      "logits/rejected": -0.730664074420929,
      "logps/chosen": -0.705371081829071,
      "logps/rejected": -0.889453113079071,
      "loss": 0.9792,
      "nll_loss": 0.8896484375,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07056884467601776,
      "rewards/margins": 0.01845550537109375,
      "rewards/rejected": -0.08895263820886612,
      "step": 6330
    },
    {
      "epoch": 0.2375822075659066,
      "grad_norm": 1.3785475406386,
      "learning_rate": 1.0047207186066567e-06,
      "log_odds_chosen": 0.3197998106479645,
      "log_odds_ratio": -0.6719726324081421,
      "logits/chosen": -0.7779296636581421,
      "logits/rejected": -0.683398425579071,
      "logps/chosen": -0.691601574420929,
      "logps/rejected": -0.9263671636581421,
      "loss": 0.9705,
      "nll_loss": 0.859179675579071,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.06918945163488388,
      "rewards/margins": 0.02347717247903347,
      "rewards/rejected": -0.09266357123851776,
      "step": 6340
    },
    {
      "epoch": 0.23795694290907046,
      "grad_norm": 1.413930833012397,
      "learning_rate": 1.0039292882210538e-06,
      "log_odds_chosen": 0.44731444120407104,
      "log_odds_ratio": -0.5892578363418579,
      "logits/chosen": -0.810546875,
      "logits/rejected": -0.709765613079071,
      "logps/chosen": -0.699023425579071,
      "logps/rejected": -0.9828125238418579,
      "loss": 0.964,
      "nll_loss": 0.8695312738418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06986083835363388,
      "rewards/margins": 0.02839508093893528,
      "rewards/rejected": -0.09829101711511612,
      "step": 6350
    },
    {
      "epoch": 0.23833167825223436,
      "grad_norm": 1.6060692240767127,
      "learning_rate": 1.0031397251510382e-06,
      "log_odds_chosen": 0.5918945074081421,
      "log_odds_ratio": -0.5462890863418579,
      "logits/chosen": -0.7734375,
      "logits/rejected": -0.704882800579071,
      "logps/chosen": -0.654296875,
      "logps/rejected": -1.0158202648162842,
      "loss": 0.9655,
      "nll_loss": 0.857617199420929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0654296875,
      "rewards/margins": 0.03620300441980362,
      "rewards/rejected": -0.10158691555261612,
      "step": 6360
    },
    {
      "epoch": 0.23870641359539824,
      "grad_norm": 1.6613269054526134,
      "learning_rate": 1.0023520220651762e-06,
      "log_odds_chosen": 0.3979125916957855,
      "log_odds_ratio": -0.631591796875,
      "logits/chosen": -0.76220703125,
      "logits/rejected": -0.6728515625,
      "logps/chosen": -0.7279297113418579,
      "logps/rejected": -1.0048828125,
      "loss": 0.9776,
      "nll_loss": 0.8970702886581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07279052585363388,
      "rewards/margins": 0.0276641845703125,
      "rewards/rejected": -0.10048828274011612,
      "step": 6370
    },
    {
      "epoch": 0.23908114893856214,
      "grad_norm": 1.6011807577626245,
      "learning_rate": 1.0015661716722687e-06,
      "log_odds_chosen": 0.31114500761032104,
      "log_odds_ratio": -0.6783202886581421,
      "logits/chosen": -0.7642577886581421,
      "logits/rejected": -0.6776367425918579,
      "logps/chosen": -0.7486327886581421,
      "logps/rejected": -0.9662109613418579,
      "loss": 0.9767,
      "nll_loss": 0.9908202886581421,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07492675632238388,
      "rewards/margins": 0.021738434210419655,
      "rewards/rejected": -0.09664306789636612,
      "step": 6380
    },
    {
      "epoch": 0.23945588428172604,
      "grad_norm": 1.6184808194442237,
      "learning_rate": 1.0007821667210687e-06,
      "log_odds_chosen": 0.35771483182907104,
      "log_odds_ratio": -0.6371093988418579,
      "logits/chosen": -0.796875,
      "logits/rejected": -0.7109375,
      "logps/chosen": -0.702441394329071,
      "logps/rejected": -0.955273449420929,
      "loss": 0.9808,
      "nll_loss": 0.8763672113418579,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07021484524011612,
      "rewards/margins": 0.025247955694794655,
      "rewards/rejected": -0.09552001953125,
      "step": 6390
    },
    {
      "epoch": 0.2398306196248899,
      "grad_norm": 1.529902385285489,
      "learning_rate": 1e-06,
      "log_odds_chosen": 0.5230957269668579,
      "log_odds_ratio": -0.5933593511581421,
      "logits/chosen": -0.810742199420929,
      "logits/rejected": -0.6919921636581421,
      "logps/chosen": -0.7284179925918579,
      "logps/rejected": -1.0771484375,
      "loss": 0.9952,
      "nll_loss": 0.948437511920929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07275390625,
      "rewards/margins": 0.03495941311120987,
      "rewards/rejected": -0.10771484673023224,
      "step": 6400
    },
    {
      "epoch": 0.2402053549680538,
      "grad_norm": 1.6396930718501554,
      "learning_rate": 9.992196643368784e-07,
      "log_odds_chosen": 0.4798828065395355,
      "log_odds_ratio": -0.6126953363418579,
      "logits/chosen": -0.7816406488418579,
      "logits/rejected": -0.6874023675918579,
      "logps/chosen": -0.7203124761581421,
      "logps/rejected": -1.0625,
      "loss": 0.9786,
      "nll_loss": 0.919140636920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07203368842601776,
      "rewards/margins": 0.03420410305261612,
      "rewards/rejected": -0.10625000298023224,
      "step": 6410
    },
    {
      "epoch": 0.24058009031121771,
      "grad_norm": 1.396587869783899,
      "learning_rate": 9.984411525986355e-07,
      "log_odds_chosen": 0.5888427495956421,
      "log_odds_ratio": -0.55908203125,
      "logits/chosen": -0.810742199420929,
      "logits/rejected": -0.711718738079071,
      "logps/chosen": -0.651562511920929,
      "logps/rejected": -1.0031249523162842,
      "loss": 0.9702,
      "nll_loss": 0.8726562261581421,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06512451171875,
      "rewards/margins": 0.03525390475988388,
      "rewards/rejected": -0.10036621242761612,
      "step": 6420
    },
    {
      "epoch": 0.2409548256543816,
      "grad_norm": 1.5919478372750848,
      "learning_rate": 9.97664457691046e-07,
      "log_odds_chosen": 0.38939207792282104,
      "log_odds_ratio": -0.62841796875,
      "logits/chosen": -0.7396484613418579,
      "logits/rejected": -0.69482421875,
      "logps/chosen": -0.686328113079071,
      "logps/rejected": -0.896289050579071,
      "loss": 0.9866,
      "nll_loss": 0.893359363079071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06867675483226776,
      "rewards/margins": 0.02102966234087944,
      "rewards/rejected": -0.08964844048023224,
      "step": 6430
    },
    {
      "epoch": 0.2413295609975455,
      "grad_norm": 1.5629909719396693,
      "learning_rate": 9.968895725584535e-07,
      "log_odds_chosen": 0.6307617425918579,
      "log_odds_ratio": -0.5204101800918579,
      "logits/chosen": -0.801953136920929,
      "logits/rejected": -0.7142578363418579,
      "logps/chosen": -0.641406238079071,
      "logps/rejected": -1.017968773841858,
      "loss": 0.9763,
      "nll_loss": 0.8583984375,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06417236477136612,
      "rewards/margins": 0.03763733059167862,
      "rewards/rejected": -0.10172118991613388,
      "step": 6440
    },
    {
      "epoch": 0.24170429634070936,
      "grad_norm": 1.5074804033332598,
      "learning_rate": 9.961164901835046e-07,
      "log_odds_chosen": 0.55938720703125,
      "log_odds_ratio": -0.5697265863418579,
      "logits/chosen": -0.8158203363418579,
      "logits/rejected": -0.70458984375,
      "logps/chosen": -0.6878906488418579,
      "logps/rejected": -1.0486328601837158,
      "loss": 0.9625,
      "nll_loss": 0.900390625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06875000149011612,
      "rewards/margins": 0.03614502027630806,
      "rewards/rejected": -0.10490722954273224,
      "step": 6450
    },
    {
      "epoch": 0.24207903168387326,
      "grad_norm": 1.50178449253303,
      "learning_rate": 9.95345203586879e-07,
      "log_odds_chosen": 0.44450682401657104,
      "log_odds_ratio": -0.59619140625,
      "logits/chosen": -0.8248046636581421,
      "logits/rejected": -0.735156238079071,
      "logps/chosen": -0.681640625,
      "logps/rejected": -0.9613281488418579,
      "loss": 0.9768,
      "nll_loss": 0.8812500238418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06816406548023224,
      "rewards/margins": 0.02793121337890625,
      "rewards/rejected": -0.09614257514476776,
      "step": 6460
    },
    {
      "epoch": 0.24245376702703716,
      "grad_norm": 1.5822088214189036,
      "learning_rate": 9.94575705827027e-07,
      "log_odds_chosen": 0.33454591035842896,
      "log_odds_ratio": -0.6693359613418579,
      "logits/chosen": -0.73193359375,
      "logits/rejected": -0.6800781488418579,
      "logps/chosen": -0.7450195550918579,
      "logps/rejected": -0.9921875,
      "loss": 0.9745,
      "nll_loss": 0.952343761920929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07449951022863388,
      "rewards/margins": 0.02465972863137722,
      "rewards/rejected": -0.09909667819738388,
      "step": 6470
    },
    {
      "epoch": 0.24282850237020104,
      "grad_norm": 1.4760771410299982,
      "learning_rate": 9.938079899999065e-07,
      "log_odds_chosen": 0.4336181581020355,
      "log_odds_ratio": -0.611328125,
      "logits/chosen": -0.7728515863418579,
      "logits/rejected": -0.667773425579071,
      "logps/chosen": -0.6908203363418579,
      "logps/rejected": -0.9775390625,
      "loss": 0.964,
      "nll_loss": 0.9380859136581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06907959282398224,
      "rewards/margins": 0.028623199090361595,
      "rewards/rejected": -0.09768066555261612,
      "step": 6480
    },
    {
      "epoch": 0.24320323771336494,
      "grad_norm": 1.6119545004988693,
      "learning_rate": 9.930420492387219e-07,
      "log_odds_chosen": 0.413330078125,
      "log_odds_ratio": -0.6207031011581421,
      "logits/chosen": -0.788281261920929,
      "logits/rejected": -0.70703125,
      "logps/chosen": -0.6830078363418579,
      "logps/rejected": -0.966015636920929,
      "loss": 0.9732,
      "nll_loss": 0.8759765625,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06827392429113388,
      "rewards/margins": 0.02830963209271431,
      "rewards/rejected": -0.09658203274011612,
      "step": 6490
    },
    {
      "epoch": 0.24357797305652884,
      "grad_norm": 1.4236696823032444,
      "learning_rate": 9.922778767136676e-07,
      "log_odds_chosen": 0.48613280057907104,
      "log_odds_ratio": -0.593945324420929,
      "logits/chosen": -0.7212890386581421,
      "logits/rejected": -0.661816418170929,
      "logps/chosen": -0.687792956829071,
      "logps/rejected": -0.987500011920929,
      "loss": 0.9748,
      "nll_loss": 0.872265636920929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06885986030101776,
      "rewards/margins": 0.0299072265625,
      "rewards/rejected": -0.09873046725988388,
      "step": 6500
    },
    {
      "epoch": 0.2439527083996927,
      "grad_norm": 1.5420577084818434,
      "learning_rate": 9.915154656316713e-07,
      "log_odds_chosen": 0.5885254144668579,
      "log_odds_ratio": -0.579296886920929,
      "logits/chosen": -0.779101550579071,
      "logits/rejected": -0.660937488079071,
      "logps/chosen": -0.7568359375,
      "logps/rejected": -1.150781273841858,
      "loss": 0.9934,
      "nll_loss": 0.987109363079071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07569579780101776,
      "rewards/margins": 0.03939361497759819,
      "rewards/rejected": -0.11513672024011612,
      "step": 6510
    },
    {
      "epoch": 0.24432744374285661,
      "grad_norm": 1.4417930399890375,
      "learning_rate": 9.907548092361398e-07,
      "log_odds_chosen": 0.3492065370082855,
      "log_odds_ratio": -0.626269519329071,
      "logits/chosen": -0.765429675579071,
      "logits/rejected": -0.684374988079071,
      "logps/chosen": -0.6924804449081421,
      "logps/rejected": -0.916796863079071,
      "loss": 0.9819,
      "nll_loss": 0.9224609136581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06918945163488388,
      "rewards/margins": 0.02243804931640625,
      "rewards/rejected": -0.09163818508386612,
      "step": 6520
    },
    {
      "epoch": 0.2447021790860205,
      "grad_norm": 1.5941294536014405,
      "learning_rate": 9.899959008067097e-07,
      "log_odds_chosen": 0.39927977323532104,
      "log_odds_ratio": -0.610156238079071,
      "logits/chosen": -0.7431640625,
      "logits/rejected": -0.679492175579071,
      "logps/chosen": -0.69140625,
      "logps/rejected": -0.931835949420929,
      "loss": 0.9884,
      "nll_loss": 0.9154297113418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06915283203125,
      "rewards/margins": 0.024030303582549095,
      "rewards/rejected": -0.09316406399011612,
      "step": 6530
    },
    {
      "epoch": 0.2450769144291844,
      "grad_norm": 1.4350232983902598,
      "learning_rate": 9.892387336589959e-07,
      "log_odds_chosen": 0.4305175840854645,
      "log_odds_ratio": -0.6000000238418579,
      "logits/chosen": -0.760058581829071,
      "logits/rejected": -0.666796863079071,
      "logps/chosen": -0.701367199420929,
      "logps/rejected": -0.985546886920929,
      "loss": 0.9616,
      "nll_loss": 0.870312511920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07009277492761612,
      "rewards/margins": 0.02841339074075222,
      "rewards/rejected": -0.0985107421875,
      "step": 6540
    },
    {
      "epoch": 0.2454516497723483,
      "grad_norm": 1.4596955054055776,
      "learning_rate": 9.884833011443446e-07,
      "log_odds_chosen": 0.5626220703125,
      "log_odds_ratio": -0.5703125,
      "logits/chosen": -0.746289074420929,
      "logits/rejected": -0.66748046875,
      "logps/chosen": -0.6240234375,
      "logps/rejected": -0.9671875238418579,
      "loss": 0.9386,
      "nll_loss": 0.830859363079071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06242675706744194,
      "rewards/margins": 0.03430290147662163,
      "rewards/rejected": -0.0968017578125,
      "step": 6550
    },
    {
      "epoch": 0.24582638511551216,
      "grad_norm": 1.4550223933240356,
      "learning_rate": 9.877295966495897e-07,
      "log_odds_chosen": 0.4349121153354645,
      "log_odds_ratio": -0.6318359375,
      "logits/chosen": -0.761523425579071,
      "logits/rejected": -0.6626952886581421,
      "logps/chosen": -0.7330077886581421,
      "logps/rejected": -1.017578125,
      "loss": 0.9922,
      "nll_loss": 0.951953113079071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07331542670726776,
      "rewards/margins": 0.02845306321978569,
      "rewards/rejected": -0.10170898586511612,
      "step": 6560
    },
    {
      "epoch": 0.24620112045867606,
      "grad_norm": 1.504750326768278,
      "learning_rate": 9.86977613596807e-07,
      "log_odds_chosen": 0.4014038145542145,
      "log_odds_ratio": -0.621289074420929,
      "logits/chosen": -0.7147461175918579,
      "logits/rejected": -0.6597656011581421,
      "logps/chosen": -0.708984375,
      "logps/rejected": -0.9677734375,
      "loss": 0.9868,
      "nll_loss": 0.99609375,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07088623195886612,
      "rewards/margins": 0.02590484544634819,
      "rewards/rejected": -0.09675292670726776,
      "step": 6570
    },
    {
      "epoch": 0.24657585580183994,
      "grad_norm": 1.4892657492765178,
      "learning_rate": 9.862273454430757e-07,
      "log_odds_chosen": 0.41569823026657104,
      "log_odds_ratio": -0.607226550579071,
      "logits/chosen": -0.737109363079071,
      "logits/rejected": -0.634960949420929,
      "logps/chosen": -0.7041015625,
      "logps/rejected": -0.960742175579071,
      "loss": 1.0016,
      "nll_loss": 0.920117199420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07049560546875,
      "rewards/margins": 0.025674819946289062,
      "rewards/rejected": -0.0960693359375,
      "step": 6580
    },
    {
      "epoch": 0.24695059114500384,
      "grad_norm": 1.4784969684222007,
      "learning_rate": 9.85478785680238e-07,
      "log_odds_chosen": 0.36018067598342896,
      "log_odds_ratio": -0.630566418170929,
      "logits/chosen": -0.7398437261581421,
      "logits/rejected": -0.667187511920929,
      "logps/chosen": -0.7593749761581421,
      "logps/rejected": -0.9986327886581421,
      "loss": 0.9752,
      "nll_loss": 0.9947265386581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07590331882238388,
      "rewards/margins": 0.024004364386200905,
      "rewards/rejected": -0.09991455078125,
      "step": 6590
    },
    {
      "epoch": 0.24732532648816774,
      "grad_norm": 1.5070542359023396,
      "learning_rate": 9.847319278346618e-07,
      "log_odds_chosen": 0.36021727323532104,
      "log_odds_ratio": -0.64404296875,
      "logits/chosen": -0.738085925579071,
      "logits/rejected": -0.6390625238418579,
      "logps/chosen": -0.7109375,
      "logps/rejected": -0.9580078125,
      "loss": 0.9806,
      "nll_loss": 0.950976550579071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07109375298023224,
      "rewards/margins": 0.024773025885224342,
      "rewards/rejected": -0.09580077975988388,
      "step": 6600
    },
    {
      "epoch": 0.2477000618313316,
      "grad_norm": 1.469033816914007,
      "learning_rate": 9.839867654670063e-07,
      "log_odds_chosen": 0.624755859375,
      "log_odds_ratio": -0.5523437261581421,
      "logits/chosen": -0.7333984375,
      "logits/rejected": -0.66796875,
      "logps/chosen": -0.654980480670929,
      "logps/rejected": -1.0548827648162842,
      "loss": 0.9937,
      "nll_loss": 0.8705078363418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06547851860523224,
      "rewards/margins": 0.0399932861328125,
      "rewards/rejected": -0.10544433444738388,
      "step": 6610
    },
    {
      "epoch": 0.24807479717449551,
      "grad_norm": 1.579167368367675,
      "learning_rate": 9.832432921719876e-07,
      "log_odds_chosen": 0.522021472454071,
      "log_odds_ratio": -0.5887695550918579,
      "logits/chosen": -0.7431640625,
      "logits/rejected": -0.643359363079071,
      "logps/chosen": -0.7093750238418579,
      "logps/rejected": -1.0392577648162842,
      "loss": 0.9815,
      "nll_loss": 0.9136718511581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07086181640625,
      "rewards/margins": 0.03303069993853569,
      "rewards/rejected": -0.10399170219898224,
      "step": 6620
    },
    {
      "epoch": 0.24844953251765942,
      "grad_norm": 1.6094053196969995,
      "learning_rate": 9.825015015781493e-07,
      "log_odds_chosen": 0.5596923828125,
      "log_odds_ratio": -0.579785168170929,
      "logits/chosen": -0.707226574420929,
      "logits/rejected": -0.648632824420929,
      "logps/chosen": -0.720898449420929,
      "logps/rejected": -1.1111328601837158,
      "loss": 0.9556,
      "nll_loss": 0.917187511920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07211913913488388,
      "rewards/margins": 0.03898010402917862,
      "rewards/rejected": -0.11110839992761612,
      "step": 6630
    },
    {
      "epoch": 0.2488242678608233,
      "grad_norm": 1.501355934011017,
      "learning_rate": 9.81761387347632e-07,
      "log_odds_chosen": 0.4422363340854645,
      "log_odds_ratio": -0.599609375,
      "logits/chosen": -0.75048828125,
      "logits/rejected": -0.6787109375,
      "logps/chosen": -0.741992175579071,
      "logps/rejected": -1.052343726158142,
      "loss": 0.9855,
      "nll_loss": 0.9458984136581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07420654594898224,
      "rewards/margins": 0.031005859375,
      "rewards/rejected": -0.10506591945886612,
      "step": 6640
    },
    {
      "epoch": 0.2491990032039872,
      "grad_norm": 1.4781982197111554,
      "learning_rate": 9.810229431759452e-07,
      "log_odds_chosen": 0.49492186307907104,
      "log_odds_ratio": -0.6190429925918579,
      "logits/chosen": -0.748046875,
      "logits/rejected": -0.6499999761581421,
      "logps/chosen": -0.71484375,
      "logps/rejected": -1.075292944908142,
      "loss": 0.9786,
      "nll_loss": 0.9859374761581421,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07150878757238388,
      "rewards/margins": 0.035984039306640625,
      "rewards/rejected": -0.10750732570886612,
      "step": 6650
    },
    {
      "epoch": 0.24957373854715106,
      "grad_norm": 1.6063552660998417,
      "learning_rate": 9.802861627917437e-07,
      "log_odds_chosen": 0.3333496153354645,
      "log_odds_ratio": -0.6478515863418579,
      "logits/chosen": -0.7857421636581421,
      "logits/rejected": -0.7378906011581421,
      "logps/chosen": -0.6787109375,
      "logps/rejected": -0.919921875,
      "loss": 0.9826,
      "nll_loss": 0.9175781011581421,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.06785888969898224,
      "rewards/margins": 0.02417144738137722,
      "rewards/rejected": -0.09199218451976776,
      "step": 6660
    },
    {
      "epoch": 0.24994847389031496,
      "grad_norm": 1.7260010765585432,
      "learning_rate": 9.795510399566016e-07,
      "log_odds_chosen": 0.4793701171875,
      "log_odds_ratio": -0.600781261920929,
      "logits/chosen": -0.7474609613418579,
      "logits/rejected": -0.659375011920929,
      "logps/chosen": -0.677734375,
      "logps/rejected": -1.005468726158142,
      "loss": 0.9725,
      "nll_loss": 0.884765625,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06776122748851776,
      "rewards/margins": 0.03278961032629013,
      "rewards/rejected": -0.10050048679113388,
      "step": 6670
    },
    {
      "epoch": 0.25032320923347884,
      "grad_norm": 1.4643060466190791,
      "learning_rate": 9.788175684647926e-07,
      "log_odds_chosen": 0.47770994901657104,
      "log_odds_ratio": -0.6001952886581421,
      "logits/chosen": -0.741015613079071,
      "logits/rejected": -0.6712890863418579,
      "logps/chosen": -0.694531261920929,
      "logps/rejected": -0.987500011920929,
      "loss": 0.9885,
      "nll_loss": 0.9300781488418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06949462741613388,
      "rewards/margins": 0.02930297888815403,
      "rewards/rejected": -0.09873046725988388,
      "step": 6680
    },
    {
      "epoch": 0.25069794457664274,
      "grad_norm": 1.5988365506272402,
      "learning_rate": 9.780857421430687e-07,
      "log_odds_chosen": 0.40544432401657104,
      "log_odds_ratio": -0.640332043170929,
      "logits/chosen": -0.769726574420929,
      "logits/rejected": -0.682421863079071,
      "logps/chosen": -0.7470703125,
      "logps/rejected": -1.025781273841858,
      "loss": 0.9637,
      "nll_loss": 0.93359375,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07471923530101776,
      "rewards/margins": 0.02789001539349556,
      "rewards/rejected": -0.1025390625,
      "step": 6690
    },
    {
      "epoch": 0.25107267991980664,
      "grad_norm": 1.530667180080285,
      "learning_rate": 9.773555548504417e-07,
      "log_odds_chosen": 0.47343748807907104,
      "log_odds_ratio": -0.5849609375,
      "logits/chosen": -0.7529296875,
      "logits/rejected": -0.644335925579071,
      "logps/chosen": -0.729296863079071,
      "logps/rejected": -1.032617211341858,
      "loss": 0.957,
      "nll_loss": 0.903124988079071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07294921576976776,
      "rewards/margins": 0.03028106689453125,
      "rewards/rejected": -0.10317382961511612,
      "step": 6700
    },
    {
      "epoch": 0.25144741526297054,
      "grad_norm": 1.5728338283615109,
      "learning_rate": 9.76627000477968e-07,
      "log_odds_chosen": 0.3163818418979645,
      "log_odds_ratio": -0.6517578363418579,
      "logits/chosen": -0.727832019329071,
      "logits/rejected": -0.660937488079071,
      "logps/chosen": -0.7183593511581421,
      "logps/rejected": -0.9130859375,
      "loss": 1.0053,
      "nll_loss": 0.974609375,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07181396335363388,
      "rewards/margins": 0.01949157752096653,
      "rewards/rejected": -0.09133300930261612,
      "step": 6710
    },
    {
      "epoch": 0.25182215060613444,
      "grad_norm": 1.4521920048013506,
      "learning_rate": 9.75900072948533e-07,
      "log_odds_chosen": 0.4195312559604645,
      "log_odds_ratio": -0.624707043170929,
      "logits/chosen": -0.7906249761581421,
      "logits/rejected": -0.65869140625,
      "logps/chosen": -0.736523449420929,
      "logps/rejected": -1.0070312023162842,
      "loss": 0.9745,
      "nll_loss": 0.908203125,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07362060248851776,
      "rewards/margins": 0.02708129957318306,
      "rewards/rejected": -0.1007080078125,
      "step": 6720
    },
    {
      "epoch": 0.2521968859492983,
      "grad_norm": 1.3955229998984244,
      "learning_rate": 9.751747662166388e-07,
      "log_odds_chosen": 0.46223145723342896,
      "log_odds_ratio": -0.5752929449081421,
      "logits/chosen": -0.7935546636581421,
      "logits/rejected": -0.6910156011581421,
      "logps/chosen": -0.6644531488418579,
      "logps/rejected": -0.965039074420929,
      "loss": 0.9893,
      "nll_loss": 0.882031261920929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06645508110523224,
      "rewards/margins": 0.03003234788775444,
      "rewards/rejected": -0.09641113132238388,
      "step": 6730
    },
    {
      "epoch": 0.2525716212924622,
      "grad_norm": 1.499481064813078,
      "learning_rate": 9.744510742681917e-07,
      "log_odds_chosen": 0.46668702363967896,
      "log_odds_ratio": -0.591113269329071,
      "logits/chosen": -0.8080078363418579,
      "logits/rejected": -0.7134765386581421,
      "logps/chosen": -0.723437488079071,
      "logps/rejected": -1.0310547351837158,
      "loss": 0.9666,
      "nll_loss": 0.874804675579071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07230224460363388,
      "rewards/margins": 0.03075561486184597,
      "rewards/rejected": -0.10311279445886612,
      "step": 6740
    },
    {
      "epoch": 0.2529463566356261,
      "grad_norm": 1.6387188161290414,
      "learning_rate": 9.737289911202953e-07,
      "log_odds_chosen": 0.3207153379917145,
      "log_odds_ratio": -0.6270507574081421,
      "logits/chosen": -0.785351574420929,
      "logits/rejected": -0.6922851800918579,
      "logps/chosen": -0.676953136920929,
      "logps/rejected": -0.873046875,
      "loss": 0.9566,
      "nll_loss": 0.912304699420929,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06770019233226776,
      "rewards/margins": 0.019625091925263405,
      "rewards/rejected": -0.08724365383386612,
      "step": 6750
    },
    {
      "epoch": 0.25332109197879,
      "grad_norm": 1.4823655106982432,
      "learning_rate": 9.730085108210398e-07,
      "log_odds_chosen": 0.4058593809604645,
      "log_odds_ratio": -0.611523449420929,
      "logits/chosen": -0.748046875,
      "logits/rejected": -0.681835949420929,
      "logps/chosen": -0.6644531488418579,
      "logps/rejected": -0.9095703363418579,
      "loss": 0.9766,
      "nll_loss": 0.9945312738418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06646728515625,
      "rewards/margins": 0.024590302258729935,
      "rewards/rejected": -0.09102783352136612,
      "step": 6760
    },
    {
      "epoch": 0.2536958273219539,
      "grad_norm": 1.5722780863886419,
      "learning_rate": 9.72289627449298e-07,
      "log_odds_chosen": 0.5234375,
      "log_odds_ratio": -0.5850585699081421,
      "logits/chosen": -0.7818359136581421,
      "logits/rejected": -0.700390636920929,
      "logps/chosen": -0.685742199420929,
      "logps/rejected": -1.0087890625,
      "loss": 0.9738,
      "nll_loss": 0.9478515386581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06854248046875,
      "rewards/margins": 0.03227538987994194,
      "rewards/rejected": -0.100830078125,
      "step": 6770
    },
    {
      "epoch": 0.25407056266511774,
      "grad_norm": 1.5655470251748247,
      "learning_rate": 9.715723351145206e-07,
      "log_odds_chosen": 0.43671876192092896,
      "log_odds_ratio": -0.60009765625,
      "logits/chosen": -0.7943359613418579,
      "logits/rejected": -0.6864258050918579,
      "logps/chosen": -0.715624988079071,
      "logps/rejected": -0.998046875,
      "loss": 0.9393,
      "nll_loss": 0.878125011920929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07150878757238388,
      "rewards/margins": 0.02825775183737278,
      "rewards/rejected": -0.09978027641773224,
      "step": 6780
    },
    {
      "epoch": 0.25444529800828164,
      "grad_norm": 1.5780045475820823,
      "learning_rate": 9.70856627956532e-07,
      "log_odds_chosen": 0.44731444120407104,
      "log_odds_ratio": -0.605273425579071,
      "logits/chosen": -0.7890625,
      "logits/rejected": -0.715039074420929,
      "logps/chosen": -0.6898437738418579,
      "logps/rejected": -0.94921875,
      "loss": 0.962,
      "nll_loss": 0.908007800579071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06898193061351776,
      "rewards/margins": 0.025948334485292435,
      "rewards/rejected": -0.09492187201976776,
      "step": 6790
    },
    {
      "epoch": 0.25482003335144554,
      "grad_norm": 1.5132879535902415,
      "learning_rate": 9.701425001453318e-07,
      "log_odds_chosen": 0.42091065645217896,
      "log_odds_ratio": -0.626660168170929,
      "logits/chosen": -0.8150390386581421,
      "logits/rejected": -0.69921875,
      "logps/chosen": -0.7342773675918579,
      "logps/rejected": -1.021093726158142,
      "loss": 0.9512,
      "nll_loss": 0.883593738079071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07342529296875,
      "rewards/margins": 0.02872619591653347,
      "rewards/rejected": -0.10213623195886612,
      "step": 6800
    },
    {
      "epoch": 0.25519476869460944,
      "grad_norm": 1.5115129819454907,
      "learning_rate": 9.694299458808932e-07,
      "log_odds_chosen": 0.4334716796875,
      "log_odds_ratio": -0.6181640625,
      "logits/chosen": -0.817578136920929,
      "logits/rejected": -0.7217773199081421,
      "logps/chosen": -0.6841796636581421,
      "logps/rejected": -0.9853515625,
      "loss": 0.97,
      "nll_loss": 0.9224609136581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06846924126148224,
      "rewards/margins": 0.03013458289206028,
      "rewards/rejected": -0.09855957329273224,
      "step": 6810
    },
    {
      "epoch": 0.25556950403777334,
      "grad_norm": 1.7280631359085803,
      "learning_rate": 9.687189593929655e-07,
      "log_odds_chosen": 0.46589356660842896,
      "log_odds_ratio": -0.61572265625,
      "logits/chosen": -0.7650390863418579,
      "logits/rejected": -0.69287109375,
      "logps/chosen": -0.701171875,
      "logps/rejected": -1.0070312023162842,
      "loss": 0.9784,
      "nll_loss": 0.968945324420929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07011719048023224,
      "rewards/margins": 0.03058471716940403,
      "rewards/rejected": -0.10068359225988388,
      "step": 6820
    },
    {
      "epoch": 0.2559442393809372,
      "grad_norm": 1.6060014545900623,
      "learning_rate": 9.680095349408789e-07,
      "log_odds_chosen": 0.4103149473667145,
      "log_odds_ratio": -0.6201171875,
      "logits/chosen": -0.7847656011581421,
      "logits/rejected": -0.7037109136581421,
      "logps/chosen": -0.6988281011581421,
      "logps/rejected": -0.9839843511581421,
      "loss": 0.9941,
      "nll_loss": 0.922070324420929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06993408501148224,
      "rewards/margins": 0.02854766882956028,
      "rewards/rejected": -0.09848632663488388,
      "step": 6830
    },
    {
      "epoch": 0.2563189747241011,
      "grad_norm": 1.4811893276396386,
      "learning_rate": 9.673016668133487e-07,
      "log_odds_chosen": 0.4452270567417145,
      "log_odds_ratio": -0.6097656488418579,
      "logits/chosen": -0.8275390863418579,
      "logits/rejected": -0.6971679925918579,
      "logps/chosen": -0.7201172113418579,
      "logps/rejected": -1.008398413658142,
      "loss": 0.9552,
      "nll_loss": 0.900195300579071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.072021484375,
      "rewards/margins": 0.028798675164580345,
      "rewards/rejected": -0.10085449367761612,
      "step": 6840
    },
    {
      "epoch": 0.256693710067265,
      "grad_norm": 1.5230257812411383,
      "learning_rate": 9.66595349328283e-07,
      "log_odds_chosen": 0.55224609375,
      "log_odds_ratio": -0.575976550579071,
      "logits/chosen": -0.7867187261581421,
      "logits/rejected": -0.6871093511581421,
      "logps/chosen": -0.667285144329071,
      "logps/rejected": -1.006250023841858,
      "loss": 0.9858,
      "nll_loss": 0.890820324420929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06673584133386612,
      "rewards/margins": 0.03391570970416069,
      "rewards/rejected": -0.10065917670726776,
      "step": 6850
    },
    {
      "epoch": 0.2570684454104289,
      "grad_norm": 1.5076926904735386,
      "learning_rate": 9.658905768325902e-07,
      "log_odds_chosen": 0.4066162109375,
      "log_odds_ratio": -0.60302734375,
      "logits/chosen": -0.7857421636581421,
      "logits/rejected": -0.741406261920929,
      "logps/chosen": -0.7353515625,
      "logps/rejected": -0.976367175579071,
      "loss": 0.9613,
      "nll_loss": 0.8988281488418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.073486328125,
      "rewards/margins": 0.02404174767434597,
      "rewards/rejected": -0.09750976413488388,
      "step": 6860
    },
    {
      "epoch": 0.2574431807535928,
      "grad_norm": 1.6121936851690626,
      "learning_rate": 9.651873437019902e-07,
      "log_odds_chosen": 0.3758911192417145,
      "log_odds_ratio": -0.61279296875,
      "logits/chosen": -0.8193359375,
      "logits/rejected": -0.731249988079071,
      "logps/chosen": -0.7099609375,
      "logps/rejected": -0.947265625,
      "loss": 0.9691,
      "nll_loss": 0.895703136920929,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07097168266773224,
      "rewards/margins": 0.02367248572409153,
      "rewards/rejected": -0.09470214694738388,
      "step": 6870
    },
    {
      "epoch": 0.25781791609675664,
      "grad_norm": 1.6413886748268882,
      "learning_rate": 9.644856443408243e-07,
      "log_odds_chosen": 0.4149169921875,
      "log_odds_ratio": -0.598925769329071,
      "logits/chosen": -0.7777343988418579,
      "logits/rejected": -0.6910156011581421,
      "logps/chosen": -0.703320324420929,
      "logps/rejected": -0.9990234375,
      "loss": 0.9731,
      "nll_loss": 0.889453113079071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07037353515625,
      "rewards/margins": 0.029615020379424095,
      "rewards/rejected": -0.09991455078125,
      "step": 6880
    },
    {
      "epoch": 0.25819265143992054,
      "grad_norm": 1.605914477501621,
      "learning_rate": 9.637854731818697e-07,
      "log_odds_chosen": 0.385986328125,
      "log_odds_ratio": -0.6424804925918579,
      "logits/chosen": -0.7583984136581421,
      "logits/rejected": -0.6668945550918579,
      "logps/chosen": -0.7115234136581421,
      "logps/rejected": -0.9525390863418579,
      "loss": 0.9911,
      "nll_loss": 0.951953113079071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07113037258386612,
      "rewards/margins": 0.024158477783203125,
      "rewards/rejected": -0.09528808295726776,
      "step": 6890
    },
    {
      "epoch": 0.25856738678308444,
      "grad_norm": 1.4698020075123048,
      "learning_rate": 9.630868246861536e-07,
      "log_odds_chosen": 0.4281249940395355,
      "log_odds_ratio": -0.6084960699081421,
      "logits/chosen": -0.7964843511581421,
      "logits/rejected": -0.7105468511581421,
      "logps/chosen": -0.695605456829071,
      "logps/rejected": -0.991406261920929,
      "loss": 0.9695,
      "nll_loss": 0.9019531011581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06959228217601776,
      "rewards/margins": 0.02954406663775444,
      "rewards/rejected": -0.09912109375,
      "step": 6900
    },
    {
      "epoch": 0.25894212212624834,
      "grad_norm": 1.5640523833321092,
      "learning_rate": 9.623896933427685e-07,
      "log_odds_chosen": 0.5010986328125,
      "log_odds_ratio": -0.5897461175918579,
      "logits/chosen": -0.7916015386581421,
      "logits/rejected": -0.703125,
      "logps/chosen": -0.751757800579071,
      "logps/rejected": -1.086328148841858,
      "loss": 0.9861,
      "nll_loss": 0.9068359136581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07510986179113388,
      "rewards/margins": 0.03350524976849556,
      "rewards/rejected": -0.10861816257238388,
      "step": 6910
    },
    {
      "epoch": 0.25931685746941224,
      "grad_norm": 1.4944135656930908,
      "learning_rate": 9.6169407366869e-07,
      "log_odds_chosen": 0.42974853515625,
      "log_odds_ratio": -0.59814453125,
      "logits/chosen": -0.804980456829071,
      "logits/rejected": -0.713671863079071,
      "logps/chosen": -0.7095702886581421,
      "logps/rejected": -0.998828113079071,
      "loss": 0.9572,
      "nll_loss": 0.8822265863418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07093505561351776,
      "rewards/margins": 0.02895050123333931,
      "rewards/rejected": -0.099853515625,
      "step": 6920
    },
    {
      "epoch": 0.25969159281257614,
      "grad_norm": 1.592247579263681,
      "learning_rate": 9.609999602085963e-07,
      "log_odds_chosen": 0.4052490293979645,
      "log_odds_ratio": -0.603710949420929,
      "logits/chosen": -0.79443359375,
      "logits/rejected": -0.7220703363418579,
      "logps/chosen": -0.6728515625,
      "logps/rejected": -0.941601574420929,
      "loss": 0.9678,
      "nll_loss": 0.839648425579071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06727294623851776,
      "rewards/margins": 0.026895904913544655,
      "rewards/rejected": -0.09420166164636612,
      "step": 6930
    },
    {
      "epoch": 0.26006632815574,
      "grad_norm": 1.5558952701415312,
      "learning_rate": 9.603073475346872e-07,
      "log_odds_chosen": 0.4245361387729645,
      "log_odds_ratio": -0.6436523199081421,
      "logits/chosen": -0.7466796636581421,
      "logits/rejected": -0.7021484375,
      "logps/chosen": -0.7420898675918579,
      "logps/rejected": -1.0244140625,
      "loss": 0.9905,
      "nll_loss": 0.9482421875,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07423095405101776,
      "rewards/margins": 0.02812499925494194,
      "rewards/rejected": -0.1024169921875,
      "step": 6940
    },
    {
      "epoch": 0.2604410634989039,
      "grad_norm": 1.7217315997798481,
      "learning_rate": 9.596162302465074e-07,
      "log_odds_chosen": 0.45942384004592896,
      "log_odds_ratio": -0.5869140625,
      "logits/chosen": -0.8173828125,
      "logits/rejected": -0.688671886920929,
      "logps/chosen": -0.712109386920929,
      "logps/rejected": -1.0126953125,
      "loss": 0.9843,
      "nll_loss": 0.9091796875,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07121582329273224,
      "rewards/margins": 0.03004913404583931,
      "rewards/rejected": -0.10118408501148224,
      "step": 6950
    },
    {
      "epoch": 0.2608157988420678,
      "grad_norm": 1.4660083579873824,
      "learning_rate": 9.589266029707683e-07,
      "log_odds_chosen": 0.49017333984375,
      "log_odds_ratio": -0.58203125,
      "logits/chosen": -0.7564452886581421,
      "logits/rejected": -0.6714843511581421,
      "logps/chosen": -0.656054675579071,
      "logps/rejected": -0.9605468511581421,
      "loss": 0.9808,
      "nll_loss": 0.9554687738418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06560058891773224,
      "rewards/margins": 0.03040618821978569,
      "rewards/rejected": -0.09597168117761612,
      "step": 6960
    },
    {
      "epoch": 0.2611905341852317,
      "grad_norm": 1.4499012562310267,
      "learning_rate": 9.582384603611731e-07,
      "log_odds_chosen": 0.33015137910842896,
      "log_odds_ratio": -0.6385742425918579,
      "logits/chosen": -0.7513672113418579,
      "logits/rejected": -0.69140625,
      "logps/chosen": -0.6509765386581421,
      "logps/rejected": -0.8521484136581421,
      "loss": 0.9786,
      "nll_loss": 0.9224609136581421,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06513671576976776,
      "rewards/margins": 0.02009429968893528,
      "rewards/rejected": -0.08519287407398224,
      "step": 6970
    },
    {
      "epoch": 0.2615652695283956,
      "grad_norm": 1.4673545749122612,
      "learning_rate": 9.575517970982428e-07,
      "log_odds_chosen": 0.27192384004592896,
      "log_odds_ratio": -0.667285144329071,
      "logits/chosen": -0.784375011920929,
      "logits/rejected": -0.718945324420929,
      "logps/chosen": -0.704882800579071,
      "logps/rejected": -0.87109375,
      "loss": 0.9748,
      "nll_loss": 0.902148425579071,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07050780951976776,
      "rewards/margins": 0.01650695875287056,
      "rewards/rejected": -0.08709716796875,
      "step": 6980
    },
    {
      "epoch": 0.26194000487155944,
      "grad_norm": 1.6173522409326742,
      "learning_rate": 9.568666078891436e-07,
      "log_odds_chosen": 0.4206176698207855,
      "log_odds_ratio": -0.620898425579071,
      "logits/chosen": -0.722851574420929,
      "logits/rejected": -0.650683581829071,
      "logps/chosen": -0.62255859375,
      "logps/rejected": -0.860156238079071,
      "loss": 0.9834,
      "nll_loss": 0.890820324420929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06223144382238388,
      "rewards/margins": 0.02379303053021431,
      "rewards/rejected": -0.08607177436351776,
      "step": 6990
    },
    {
      "epoch": 0.26231474021472334,
      "grad_norm": 1.5676871010467868,
      "learning_rate": 9.561828874675149e-07,
      "log_odds_chosen": 0.4712890684604645,
      "log_odds_ratio": -0.5840820074081421,
      "logits/chosen": -0.7513672113418579,
      "logits/rejected": -0.6666015386581421,
      "logps/chosen": -0.6591796875,
      "logps/rejected": -0.9525390863418579,
      "loss": 0.9712,
      "nll_loss": 0.8525390625,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06591796875,
      "rewards/margins": 0.02928466722369194,
      "rewards/rejected": -0.09514160454273224,
      "step": 7000
    },
    {
      "epoch": 0.26268947555788724,
      "grad_norm": 1.5010440051064584,
      "learning_rate": 9.555006305933e-07,
      "log_odds_chosen": 0.561572253704071,
      "log_odds_ratio": -0.5718749761581421,
      "logits/chosen": -0.7378906011581421,
      "logits/rejected": -0.669628918170929,
      "logps/chosen": -0.697558581829071,
      "logps/rejected": -1.066015601158142,
      "loss": 0.9748,
      "nll_loss": 0.9437500238418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06977538764476776,
      "rewards/margins": 0.03683166578412056,
      "rewards/rejected": -0.1065673828125,
      "step": 7010
    },
    {
      "epoch": 0.26306421090105114,
      "grad_norm": 1.7214335918936992,
      "learning_rate": 9.548198320525771e-07,
      "log_odds_chosen": 0.44049072265625,
      "log_odds_ratio": -0.613574206829071,
      "logits/chosen": -0.766796886920929,
      "logits/rejected": -0.7100585699081421,
      "logps/chosen": -0.73046875,
      "logps/rejected": -0.9906250238418579,
      "loss": 0.9637,
      "nll_loss": 0.9046875238418579,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07304687798023224,
      "rewards/margins": 0.02611846849322319,
      "rewards/rejected": -0.09912109375,
      "step": 7020
    },
    {
      "epoch": 0.26343894624421504,
      "grad_norm": 1.516077394663285,
      "learning_rate": 9.54140486657392e-07,
      "log_odds_chosen": 0.5549682378768921,
      "log_odds_ratio": -0.57275390625,
      "logits/chosen": -0.7738281488418579,
      "logits/rejected": -0.6913086175918579,
      "logps/chosen": -0.699414074420929,
      "logps/rejected": -1.074804663658142,
      "loss": 0.985,
      "nll_loss": 0.960156261920929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06993408501148224,
      "rewards/margins": 0.03765564039349556,
      "rewards/rejected": -0.10759277641773224,
      "step": 7030
    },
    {
      "epoch": 0.2638136815873789,
      "grad_norm": 1.5934200495997868,
      "learning_rate": 9.534625892455922e-07,
      "log_odds_chosen": 0.42747801542282104,
      "log_odds_ratio": -0.60400390625,
      "logits/chosen": -0.8037109375,
      "logits/rejected": -0.696093738079071,
      "logps/chosen": -0.691601574420929,
      "logps/rejected": -0.963085949420929,
      "loss": 0.9719,
      "nll_loss": 0.9134765863418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06914062798023224,
      "rewards/margins": 0.02715301513671875,
      "rewards/rejected": -0.09626464545726776,
      "step": 7040
    },
    {
      "epoch": 0.2641884169305428,
      "grad_norm": 1.5598907308883083,
      "learning_rate": 9.527861346806618e-07,
      "log_odds_chosen": 0.40864259004592896,
      "log_odds_ratio": -0.6122070550918579,
      "logits/chosen": -0.7642577886581421,
      "logits/rejected": -0.682421863079071,
      "logps/chosen": -0.6888672113418579,
      "logps/rejected": -0.958203136920929,
      "loss": 0.9611,
      "nll_loss": 0.8433593511581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06887207180261612,
      "rewards/margins": 0.02698821946978569,
      "rewards/rejected": -0.09586181491613388,
      "step": 7050
    },
    {
      "epoch": 0.2645631522737067,
      "grad_norm": 1.5631771594227564,
      "learning_rate": 9.521111178515582e-07,
      "log_odds_chosen": 0.4984985291957855,
      "log_odds_ratio": -0.557910144329071,
      "logits/chosen": -0.7718750238418579,
      "logits/rejected": -0.6689453125,
      "logps/chosen": -0.650585949420929,
      "logps/rejected": -0.97265625,
      "loss": 0.9727,
      "nll_loss": 0.8931640386581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06503906100988388,
      "rewards/margins": 0.03220672532916069,
      "rewards/rejected": -0.0972900390625,
      "step": 7060
    },
    {
      "epoch": 0.2649378876168706,
      "grad_norm": 1.4793317521596683,
      "learning_rate": 9.514375336725502e-07,
      "log_odds_chosen": 0.28106689453125,
      "log_odds_ratio": -0.6458984613418579,
      "logits/chosen": -0.7867187261581421,
      "logits/rejected": -0.7254883050918579,
      "logps/chosen": -0.7176758050918579,
      "logps/rejected": -0.8978515863418579,
      "loss": 0.9464,
      "nll_loss": 0.9300781488418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07176513969898224,
      "rewards/margins": 0.01803741417825222,
      "rewards/rejected": -0.08978271484375,
      "step": 7070
    },
    {
      "epoch": 0.2653126229600345,
      "grad_norm": 1.503949107167881,
      "learning_rate": 9.507653770830566e-07,
      "log_odds_chosen": 0.576062023639679,
      "log_odds_ratio": -0.583789050579071,
      "logits/chosen": -0.791015625,
      "logits/rejected": -0.679003894329071,
      "logps/chosen": -0.72998046875,
      "logps/rejected": -1.1183593273162842,
      "loss": 0.9851,
      "nll_loss": 0.939648449420929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.072998046875,
      "rewards/margins": 0.038793183863162994,
      "rewards/rejected": -0.1119384765625,
      "step": 7080
    },
    {
      "epoch": 0.26568735830319834,
      "grad_norm": 1.4554810184066165,
      "learning_rate": 9.500946430474869e-07,
      "log_odds_chosen": 0.35930174589157104,
      "log_odds_ratio": -0.643847644329071,
      "logits/chosen": -0.7705078125,
      "logits/rejected": -0.691210925579071,
      "logps/chosen": -0.7183593511581421,
      "logps/rejected": -0.9662109613418579,
      "loss": 0.9873,
      "nll_loss": 0.8759765625,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07176513969898224,
      "rewards/margins": 0.02487030066549778,
      "rewards/rejected": -0.09663085639476776,
      "step": 7090
    },
    {
      "epoch": 0.26606209364636224,
      "grad_norm": 1.452840707095692,
      "learning_rate": 9.494253265550825e-07,
      "log_odds_chosen": 0.31159669160842896,
      "log_odds_ratio": -0.6468750238418579,
      "logits/chosen": -0.7568359375,
      "logits/rejected": -0.6949218511581421,
      "logps/chosen": -0.7105468511581421,
      "logps/rejected": -0.9214843511581421,
      "loss": 0.9802,
      "nll_loss": 0.9535156488418579,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07099609076976776,
      "rewards/margins": 0.021152496337890625,
      "rewards/rejected": -0.09210205078125,
      "step": 7100
    },
    {
      "epoch": 0.26643682898952614,
      "grad_norm": 1.5755248358336287,
      "learning_rate": 9.4875742261976e-07,
      "log_odds_chosen": 0.4615722596645355,
      "log_odds_ratio": -0.6039062738418579,
      "logits/chosen": -0.797656238079071,
      "logits/rejected": -0.6973632574081421,
      "logps/chosen": -0.7124999761581421,
      "logps/rejected": -1.032617211341858,
      "loss": 0.9661,
      "nll_loss": 0.9150390625,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07126464694738388,
      "rewards/margins": 0.03201141208410263,
      "rewards/rejected": -0.10319824516773224,
      "step": 7110
    },
    {
      "epoch": 0.26681156433269004,
      "grad_norm": 1.5876342876718472,
      "learning_rate": 9.480909262799544e-07,
      "log_odds_chosen": 0.4933105409145355,
      "log_odds_ratio": -0.603320300579071,
      "logits/chosen": -0.7998046875,
      "logits/rejected": -0.7060546875,
      "logps/chosen": -0.6978515386581421,
      "logps/rejected": -1.0177733898162842,
      "loss": 0.9741,
      "nll_loss": 0.8931640386581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06978759914636612,
      "rewards/margins": 0.03207092359662056,
      "rewards/rejected": -0.10178222507238388,
      "step": 7120
    },
    {
      "epoch": 0.26718629967585394,
      "grad_norm": 1.7517745515361949,
      "learning_rate": 9.47425832598465e-07,
      "log_odds_chosen": 0.4377075135707855,
      "log_odds_ratio": -0.6363281011581421,
      "logits/chosen": -0.785351574420929,
      "logits/rejected": -0.6571289300918579,
      "logps/chosen": -0.693652331829071,
      "logps/rejected": -1.002343773841858,
      "loss": 0.9884,
      "nll_loss": 0.937695324420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0693359375,
      "rewards/margins": 0.03089141845703125,
      "rewards/rejected": -0.10025634616613388,
      "step": 7130
    },
    {
      "epoch": 0.26756103501901785,
      "grad_norm": 1.532688688166292,
      "learning_rate": 9.467621366623017e-07,
      "log_odds_chosen": 0.511914074420929,
      "log_odds_ratio": -0.592236340045929,
      "logits/chosen": -0.7476562261581421,
      "logits/rejected": -0.679492175579071,
      "logps/chosen": -0.716015636920929,
      "logps/rejected": -1.0339844226837158,
      "loss": 0.9834,
      "nll_loss": 0.926953136920929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07164306938648224,
      "rewards/margins": 0.03180236741900444,
      "rewards/rejected": -0.10345458984375,
      "step": 7140
    },
    {
      "epoch": 0.2679357703621817,
      "grad_norm": 1.6008687072881953,
      "learning_rate": 9.46099833582532e-07,
      "log_odds_chosen": 0.44874268770217896,
      "log_odds_ratio": -0.6148437261581421,
      "logits/chosen": -0.755664050579071,
      "logits/rejected": -0.669628918170929,
      "logps/chosen": -0.689648449420929,
      "logps/rejected": -0.970703125,
      "loss": 0.9664,
      "nll_loss": 0.834765613079071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06892089545726776,
      "rewards/margins": 0.02803802490234375,
      "rewards/rejected": -0.0970458984375,
      "step": 7150
    },
    {
      "epoch": 0.2683105057053456,
      "grad_norm": 1.4875129292031999,
      "learning_rate": 9.45438918494131e-07,
      "log_odds_chosen": 0.4830078184604645,
      "log_odds_ratio": -0.592480480670929,
      "logits/chosen": -0.768750011920929,
      "logits/rejected": -0.6880859136581421,
      "logps/chosen": -0.736328125,
      "logps/rejected": -1.0427734851837158,
      "loss": 0.9719,
      "nll_loss": 0.991015613079071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07363281399011612,
      "rewards/margins": 0.03067932091653347,
      "rewards/rejected": -0.10434570163488388,
      "step": 7160
    },
    {
      "epoch": 0.2686852410485095,
      "grad_norm": 1.5786161240637737,
      "learning_rate": 9.447793865558291e-07,
      "log_odds_chosen": 0.5413818359375,
      "log_odds_ratio": -0.5482422113418579,
      "logits/chosen": -0.8173828125,
      "logits/rejected": -0.716992199420929,
      "logps/chosen": -0.6830078363418579,
      "logps/rejected": -1.0255858898162842,
      "loss": 0.9615,
      "nll_loss": 0.859179675579071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06829833984375,
      "rewards/margins": 0.03420715406537056,
      "rewards/rejected": -0.10249023139476776,
      "step": 7170
    },
    {
      "epoch": 0.2690599763916734,
      "grad_norm": 1.5572594174554624,
      "learning_rate": 9.441212329499659e-07,
      "log_odds_chosen": 0.37492674589157104,
      "log_odds_ratio": -0.6241210699081421,
      "logits/chosen": -0.797656238079071,
      "logits/rejected": -0.701953113079071,
      "logps/chosen": -0.724609375,
      "logps/rejected": -0.991992175579071,
      "loss": 0.9858,
      "nll_loss": 0.9580078125,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07249756157398224,
      "rewards/margins": 0.026728058233857155,
      "rewards/rejected": -0.09923096001148224,
      "step": 7180
    },
    {
      "epoch": 0.2694347117348373,
      "grad_norm": 1.4866285983697742,
      "learning_rate": 9.434644528823399e-07,
      "log_odds_chosen": 0.6072753667831421,
      "log_odds_ratio": -0.548144519329071,
      "logits/chosen": -0.7435547113418579,
      "logits/rejected": -0.6607421636581421,
      "logps/chosen": -0.6993163824081421,
      "logps/rejected": -1.0517578125,
      "loss": 0.9643,
      "nll_loss": 0.9380859136581421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06986083835363388,
      "rewards/margins": 0.03530578687787056,
      "rewards/rejected": -0.10520019382238388,
      "step": 7190
    },
    {
      "epoch": 0.26980944707800114,
      "grad_norm": 1.4529121058145826,
      "learning_rate": 9.428090415820634e-07,
      "log_odds_chosen": 0.4767700135707855,
      "log_odds_ratio": -0.594531238079071,
      "logits/chosen": -0.8197265863418579,
      "logits/rejected": -0.6708008050918579,
      "logps/chosen": -0.6708008050918579,
      "logps/rejected": -0.973828136920929,
      "loss": 0.9954,
      "nll_loss": 0.8949218988418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06708984076976776,
      "rewards/margins": 0.0303192138671875,
      "rewards/rejected": -0.09737548977136612,
      "step": 7200
    },
    {
      "epoch": 0.27018418242116504,
      "grad_norm": 1.4611716511282589,
      "learning_rate": 9.42154994301416e-07,
      "log_odds_chosen": 0.46992188692092896,
      "log_odds_ratio": -0.619335949420929,
      "logits/chosen": -0.7144531011581421,
      "logits/rejected": -0.686230480670929,
      "logps/chosen": -0.7337890863418579,
      "logps/rejected": -1.024023413658142,
      "loss": 0.9841,
      "nll_loss": 0.9037109613418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07335205376148224,
      "rewards/margins": 0.029038239270448685,
      "rewards/rejected": -0.10238037258386612,
      "step": 7210
    },
    {
      "epoch": 0.27055891776432894,
      "grad_norm": 1.5240797173939402,
      "learning_rate": 9.415023063157008e-07,
      "log_odds_chosen": 0.45362550020217896,
      "log_odds_ratio": -0.614453136920929,
      "logits/chosen": -0.802734375,
      "logits/rejected": -0.724609375,
      "logps/chosen": -0.7066406011581421,
      "logps/rejected": -1.034765601158142,
      "loss": 0.9792,
      "nll_loss": 0.9439452886581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07073974609375,
      "rewards/margins": 0.032778166234493256,
      "rewards/rejected": -0.10352782905101776,
      "step": 7220
    },
    {
      "epoch": 0.27093365310749284,
      "grad_norm": 1.505783310218545,
      "learning_rate": 9.408509729231009e-07,
      "log_odds_chosen": 0.44471436738967896,
      "log_odds_ratio": -0.632128894329071,
      "logits/chosen": -0.822070300579071,
      "logits/rejected": -0.71142578125,
      "logps/chosen": -0.69873046875,
      "logps/rejected": -1.0095703601837158,
      "loss": 0.983,
      "nll_loss": 0.905078113079071,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06989745795726776,
      "rewards/margins": 0.0309600830078125,
      "rewards/rejected": -0.10084228217601776,
      "step": 7230
    },
    {
      "epoch": 0.27130838845065675,
      "grad_norm": 1.457552596826891,
      "learning_rate": 9.402009894445369e-07,
      "log_odds_chosen": 0.4488891661167145,
      "log_odds_ratio": -0.6161133050918579,
      "logits/chosen": -0.8212890625,
      "logits/rejected": -0.726367175579071,
      "logps/chosen": -0.739453136920929,
      "logps/rejected": -1.056054711341858,
      "loss": 0.9842,
      "nll_loss": 0.9341796636581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07392577826976776,
      "rewards/margins": 0.03163757175207138,
      "rewards/rejected": -0.10554198920726776,
      "step": 7240
    },
    {
      "epoch": 0.2716831237938206,
      "grad_norm": 1.633176987495773,
      "learning_rate": 9.395523512235255e-07,
      "log_odds_chosen": 0.41496580839157104,
      "log_odds_ratio": -0.6153320074081421,
      "logits/chosen": -0.8167968988418579,
      "logits/rejected": -0.750781238079071,
      "logps/chosen": -0.7481445074081421,
      "logps/rejected": -1.0212891101837158,
      "loss": 0.9609,
      "nll_loss": 0.8447265625,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07473144680261612,
      "rewards/margins": 0.02733306959271431,
      "rewards/rejected": -0.10200195014476776,
      "step": 7250
    },
    {
      "epoch": 0.2720578591369845,
      "grad_norm": 1.5378575221495963,
      "learning_rate": 9.389050536260404e-07,
      "log_odds_chosen": 0.387451171875,
      "log_odds_ratio": -0.623046875,
      "logits/chosen": -0.767871081829071,
      "logits/rejected": -0.686718761920929,
      "logps/chosen": -0.6943359375,
      "logps/rejected": -0.9447265863418579,
      "loss": 0.976,
      "nll_loss": 0.9574218988418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06938476860523224,
      "rewards/margins": 0.02504730224609375,
      "rewards/rejected": -0.09447021782398224,
      "step": 7260
    },
    {
      "epoch": 0.2724325944801484,
      "grad_norm": 1.6029741394675554,
      "learning_rate": 9.382590920403722e-07,
      "log_odds_chosen": 0.560253918170929,
      "log_odds_ratio": -0.587695300579071,
      "logits/chosen": -0.799023449420929,
      "logits/rejected": -0.729296863079071,
      "logps/chosen": -0.6776367425918579,
      "logps/rejected": -1.0595703125,
      "loss": 0.9726,
      "nll_loss": 0.9164062738418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06782226264476776,
      "rewards/margins": 0.03818969801068306,
      "rewards/rejected": -0.10600586235523224,
      "step": 7270
    },
    {
      "epoch": 0.2728073298233123,
      "grad_norm": 1.54746899687055,
      "learning_rate": 9.376144618769908e-07,
      "log_odds_chosen": 0.5535033941268921,
      "log_odds_ratio": -0.568408191204071,
      "logits/chosen": -0.787890613079071,
      "logits/rejected": -0.680468738079071,
      "logps/chosen": -0.7216796875,
      "logps/rejected": -1.0890624523162842,
      "loss": 0.9682,
      "nll_loss": 0.9798828363418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.0721435546875,
      "rewards/margins": 0.036769866943359375,
      "rewards/rejected": -0.10893554985523224,
      "step": 7280
    },
    {
      "epoch": 0.2731820651664762,
      "grad_norm": 1.6883127777265976,
      "learning_rate": 9.369711585684086e-07,
      "log_odds_chosen": 0.523364245891571,
      "log_odds_ratio": -0.588183581829071,
      "logits/chosen": -0.819531261920929,
      "logits/rejected": -0.677929699420929,
      "logps/chosen": -0.7015625238418579,
      "logps/rejected": -1.0603516101837158,
      "loss": 0.9655,
      "nll_loss": 0.9232422113418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07017822563648224,
      "rewards/margins": 0.03593902662396431,
      "rewards/rejected": -0.10604248195886612,
      "step": 7290
    },
    {
      "epoch": 0.27355680050964004,
      "grad_norm": 1.5115002232388752,
      "learning_rate": 9.363291775690445e-07,
      "log_odds_chosen": 0.3437744081020355,
      "log_odds_ratio": -0.6597656011581421,
      "logits/chosen": -0.815234363079071,
      "logits/rejected": -0.7515624761581421,
      "logps/chosen": -0.7339843511581421,
      "logps/rejected": -0.955078125,
      "loss": 0.9769,
      "nll_loss": 0.956835925579071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07335205376148224,
      "rewards/margins": 0.02212982252240181,
      "rewards/rejected": -0.095458984375,
      "step": 7300
    },
    {
      "epoch": 0.27393153585280394,
      "grad_norm": 1.6297638060837232,
      "learning_rate": 9.356885143550886e-07,
      "log_odds_chosen": 0.42103272676467896,
      "log_odds_ratio": -0.6181640625,
      "logits/chosen": -0.7802734375,
      "logits/rejected": -0.7196289300918579,
      "logps/chosen": -0.7095702886581421,
      "logps/rejected": -0.9638671875,
      "loss": 0.96,
      "nll_loss": 0.87890625,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07108154147863388,
      "rewards/margins": 0.02535247802734375,
      "rewards/rejected": -0.09641113132238388,
      "step": 7310
    },
    {
      "epoch": 0.27430627119596784,
      "grad_norm": 1.5236130445000942,
      "learning_rate": 9.350491644243688e-07,
      "log_odds_chosen": 0.4283447265625,
      "log_odds_ratio": -0.6226562261581421,
      "logits/chosen": -0.7816406488418579,
      "logits/rejected": -0.7108398675918579,
      "logps/chosen": -0.731249988079071,
      "logps/rejected": -1.0193359851837158,
      "loss": 0.9809,
      "nll_loss": 0.923632800579071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07313232123851776,
      "rewards/margins": 0.028762053698301315,
      "rewards/rejected": -0.10183105617761612,
      "step": 7320
    },
    {
      "epoch": 0.27468100653913174,
      "grad_norm": 1.5149171940943396,
      "learning_rate": 9.344111232962179e-07,
      "log_odds_chosen": 0.4458251893520355,
      "log_odds_ratio": -0.60302734375,
      "logits/chosen": -0.8076171875,
      "logits/rejected": -0.7347656488418579,
      "logps/chosen": -0.734375,
      "logps/rejected": -1.0392577648162842,
      "loss": 0.9856,
      "nll_loss": 0.954882800579071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07340087741613388,
      "rewards/margins": 0.0305328369140625,
      "rewards/rejected": -0.10399170219898224,
      "step": 7330
    },
    {
      "epoch": 0.27505574188229565,
      "grad_norm": 1.6588837011583675,
      "learning_rate": 9.337743865113415e-07,
      "log_odds_chosen": 0.677783191204071,
      "log_odds_ratio": -0.506542980670929,
      "logits/chosen": -0.8150390386581421,
      "logits/rejected": -0.7066406011581421,
      "logps/chosen": -0.662792980670929,
      "logps/rejected": -1.0822265148162842,
      "loss": 0.9701,
      "nll_loss": 0.8837890625,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06624756008386612,
      "rewards/margins": 0.04205475002527237,
      "rewards/rejected": -0.10822753608226776,
      "step": 7340
    },
    {
      "epoch": 0.27543047722545955,
      "grad_norm": 1.421182764372469,
      "learning_rate": 9.331389496316868e-07,
      "log_odds_chosen": 0.39556884765625,
      "log_odds_ratio": -0.61669921875,
      "logits/chosen": -0.764453113079071,
      "logits/rejected": -0.721484363079071,
      "logps/chosen": -0.7025390863418579,
      "logps/rejected": -0.954296886920929,
      "loss": 0.9931,
      "nll_loss": 0.9156249761581421,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07020263373851776,
      "rewards/margins": 0.02517089806497097,
      "rewards/rejected": -0.09538574516773224,
      "step": 7350
    },
    {
      "epoch": 0.2758052125686234,
      "grad_norm": 1.5810541256892054,
      "learning_rate": 9.325048082403138e-07,
      "log_odds_chosen": 0.41041260957717896,
      "log_odds_ratio": -0.64404296875,
      "logits/chosen": -0.7886718511581421,
      "logits/rejected": -0.7281249761581421,
      "logps/chosen": -0.735058605670929,
      "logps/rejected": -0.982226550579071,
      "loss": 0.9511,
      "nll_loss": 0.955273449420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07349853217601776,
      "rewards/margins": 0.02475280687212944,
      "rewards/rejected": -0.09832763671875,
      "step": 7360
    },
    {
      "epoch": 0.2761799479117873,
      "grad_norm": 1.5557686597710934,
      "learning_rate": 9.318719579412648e-07,
      "log_odds_chosen": 0.4743103086948395,
      "log_odds_ratio": -0.600390613079071,
      "logits/chosen": -0.7974609136581421,
      "logits/rejected": -0.719531238079071,
      "logps/chosen": -0.6929687261581421,
      "logps/rejected": -1.008398413658142,
      "loss": 0.9401,
      "nll_loss": 0.906445324420929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06932373344898224,
      "rewards/margins": 0.03144683688879013,
      "rewards/rejected": -0.10080566257238388,
      "step": 7370
    },
    {
      "epoch": 0.2765546832549512,
      "grad_norm": 1.5614579068642553,
      "learning_rate": 9.312403943594374e-07,
      "log_odds_chosen": 0.47480469942092896,
      "log_odds_ratio": -0.616406261920929,
      "logits/chosen": -0.7623046636581421,
      "logits/rejected": -0.6412109136581421,
      "logps/chosen": -0.744824230670929,
      "logps/rejected": -1.0652344226837158,
      "loss": 0.9692,
      "nll_loss": 0.9322265386581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07449951022863388,
      "rewards/margins": 0.03194122388958931,
      "rewards/rejected": -0.10649414360523224,
      "step": 7380
    },
    {
      "epoch": 0.2769294185981151,
      "grad_norm": 1.581837022356933,
      "learning_rate": 9.306101131404582e-07,
      "log_odds_chosen": 0.4783691465854645,
      "log_odds_ratio": -0.6103515625,
      "logits/chosen": -0.731640636920929,
      "logits/rejected": -0.6519531011581421,
      "logps/chosen": -0.6982421875,
      "logps/rejected": -1.032617211341858,
      "loss": 0.9584,
      "nll_loss": 0.9496093988418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06981201469898224,
      "rewards/margins": 0.033431243151426315,
      "rewards/rejected": -0.10324706882238388,
      "step": 7390
    },
    {
      "epoch": 0.277304153941279,
      "grad_norm": 1.5727136916016904,
      "learning_rate": 9.299811099505542e-07,
      "log_odds_chosen": 0.4085449278354645,
      "log_odds_ratio": -0.6297851800918579,
      "logits/chosen": -0.787109375,
      "logits/rejected": -0.697558581829071,
      "logps/chosen": -0.708984375,
      "logps/rejected": -0.9974609613418579,
      "loss": 0.9739,
      "nll_loss": 0.949023425579071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07087402045726776,
      "rewards/margins": 0.02892303466796875,
      "rewards/rejected": -0.0997314453125,
      "step": 7400
    },
    {
      "epoch": 0.27767888928444284,
      "grad_norm": 1.634654985418646,
      "learning_rate": 9.293533804764305e-07,
      "log_odds_chosen": 0.5394287109375,
      "log_odds_ratio": -0.5718749761581421,
      "logits/chosen": -0.817187488079071,
      "logits/rejected": -0.6947265863418579,
      "logps/chosen": -0.675976574420929,
      "logps/rejected": -1.0009765625,
      "loss": 0.9594,
      "nll_loss": 0.9097656011581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.067626953125,
      "rewards/margins": 0.032456208020448685,
      "rewards/rejected": -0.10009765625,
      "step": 7410
    },
    {
      "epoch": 0.27805362462760674,
      "grad_norm": 1.532299885323135,
      "learning_rate": 9.28726920425144e-07,
      "log_odds_chosen": 0.41975098848342896,
      "log_odds_ratio": -0.6275390386581421,
      "logits/chosen": -0.8199218511581421,
      "logits/rejected": -0.7437499761581421,
      "logps/chosen": -0.7090820074081421,
      "logps/rejected": -0.955859363079071,
      "loss": 0.9632,
      "nll_loss": 0.9078124761581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.0709228515625,
      "rewards/margins": 0.02467041090130806,
      "rewards/rejected": -0.09566650539636612,
      "step": 7420
    },
    {
      "epoch": 0.27842835997077064,
      "grad_norm": 1.6369602145052693,
      "learning_rate": 9.281017255239815e-07,
      "log_odds_chosen": 0.44340819120407104,
      "log_odds_ratio": -0.595996081829071,
      "logits/chosen": -0.783007800579071,
      "logits/rejected": -0.678515613079071,
      "logps/chosen": -0.780078113079071,
      "logps/rejected": -1.0830078125,
      "loss": 0.9813,
      "nll_loss": 0.9544922113418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07794189453125,
      "rewards/margins": 0.03030090406537056,
      "rewards/rejected": -0.10830078274011612,
      "step": 7430
    },
    {
      "epoch": 0.27880309531393455,
      "grad_norm": 1.5546199989852376,
      "learning_rate": 9.274777915203365e-07,
      "log_odds_chosen": 0.29937744140625,
      "log_odds_ratio": -0.66015625,
      "logits/chosen": -0.7779296636581421,
      "logits/rejected": -0.721875011920929,
      "logps/chosen": -0.7518554925918579,
      "logps/rejected": -0.939257800579071,
      "loss": 0.9793,
      "nll_loss": 1.0068359375,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07513427734375,
      "rewards/margins": 0.01873168908059597,
      "rewards/rejected": -0.09388427436351776,
      "step": 7440
    },
    {
      "epoch": 0.27917783065709845,
      "grad_norm": 1.5248730631097553,
      "learning_rate": 9.268551141815875e-07,
      "log_odds_chosen": 0.3472045958042145,
      "log_odds_ratio": -0.6529296636581421,
      "logits/chosen": -0.743359386920929,
      "logits/rejected": -0.6664062738418579,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -0.938671886920929,
      "loss": 0.9763,
      "nll_loss": 0.946484386920929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07066650688648224,
      "rewards/margins": 0.02325286902487278,
      "rewards/rejected": -0.09379883110523224,
      "step": 7450
    },
    {
      "epoch": 0.2795525660002623,
      "grad_norm": 1.4753628035306687,
      "learning_rate": 9.262336892949784e-07,
      "log_odds_chosen": 0.615063488483429,
      "log_odds_ratio": -0.55517578125,
      "logits/chosen": -0.740234375,
      "logits/rejected": -0.6201171875,
      "logps/chosen": -0.671679675579071,
      "logps/rejected": -1.074609398841858,
      "loss": 0.9641,
      "nll_loss": 0.9130859375,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06718750298023224,
      "rewards/margins": 0.04018554836511612,
      "rewards/rejected": -0.10739745944738388,
      "step": 7460
    },
    {
      "epoch": 0.2799273013434262,
      "grad_norm": 1.6105124830560853,
      "learning_rate": 9.256135126674977e-07,
      "log_odds_chosen": 0.4723754823207855,
      "log_odds_ratio": -0.5858398675918579,
      "logits/chosen": -0.8140624761581421,
      "logits/rejected": -0.733203113079071,
      "logps/chosen": -0.691601574420929,
      "logps/rejected": -0.9966796636581421,
      "loss": 0.9647,
      "nll_loss": 0.899218738079071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0692138671875,
      "rewards/margins": 0.03047332726418972,
      "rewards/rejected": -0.09959717094898224,
      "step": 7470
    },
    {
      "epoch": 0.2803020366865901,
      "grad_norm": 1.5355767665277569,
      "learning_rate": 9.249945801257605e-07,
      "log_odds_chosen": 0.42271727323532104,
      "log_odds_ratio": -0.6075195074081421,
      "logits/chosen": -0.798632800579071,
      "logits/rejected": -0.742968738079071,
      "logps/chosen": -0.724414050579071,
      "logps/rejected": -0.9609375,
      "loss": 0.9605,
      "nll_loss": 0.8681640625,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07242431491613388,
      "rewards/margins": 0.02370605431497097,
      "rewards/rejected": -0.0960693359375,
      "step": 7480
    },
    {
      "epoch": 0.280676772029754,
      "grad_norm": 1.4942493970863784,
      "learning_rate": 9.243768875158902e-07,
      "log_odds_chosen": 0.523120105266571,
      "log_odds_ratio": -0.57373046875,
      "logits/chosen": -0.7679687738418579,
      "logits/rejected": -0.7152343988418579,
      "logps/chosen": -0.6864258050918579,
      "logps/rejected": -1.026953101158142,
      "loss": 0.958,
      "nll_loss": 0.9078124761581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06864013522863388,
      "rewards/margins": 0.03409118577837944,
      "rewards/rejected": -0.10270996391773224,
      "step": 7490
    },
    {
      "epoch": 0.2810515073729179,
      "grad_norm": 1.5326933258566333,
      "learning_rate": 9.23760430703401e-07,
      "log_odds_chosen": 0.570849597454071,
      "log_odds_ratio": -0.544140636920929,
      "logits/chosen": -0.819140613079071,
      "logits/rejected": -0.711132824420929,
      "logps/chosen": -0.6649414300918579,
      "logps/rejected": -1.035546898841858,
      "loss": 0.9731,
      "nll_loss": 0.921679675579071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06645508110523224,
      "rewards/margins": 0.03702392429113388,
      "rewards/rejected": -0.10356445610523224,
      "step": 7500
    },
    {
      "epoch": 0.28142624271608174,
      "grad_norm": 1.5667066018267846,
      "learning_rate": 9.231452055730832e-07,
      "log_odds_chosen": 0.4181274473667145,
      "log_odds_ratio": -0.609179675579071,
      "logits/chosen": -0.814453125,
      "logits/rejected": -0.7046874761581421,
      "logps/chosen": -0.7074218988418579,
      "logps/rejected": -0.991015613079071,
      "loss": 0.9615,
      "nll_loss": 0.8951171636581421,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07087402045726776,
      "rewards/margins": 0.02832489088177681,
      "rewards/rejected": -0.09912109375,
      "step": 7510
    },
    {
      "epoch": 0.28180097805924564,
      "grad_norm": 1.6063107525437397,
      "learning_rate": 9.225312080288851e-07,
      "log_odds_chosen": 0.47216796875,
      "log_odds_ratio": -0.6039062738418579,
      "logits/chosen": -0.788281261920929,
      "logits/rejected": -0.7142578363418579,
      "logps/chosen": -0.731249988079071,
      "logps/rejected": -1.0476562976837158,
      "loss": 0.9631,
      "nll_loss": 0.8304687738418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.0731201171875,
      "rewards/margins": 0.031589508056640625,
      "rewards/rejected": -0.10468749701976776,
      "step": 7520
    },
    {
      "epoch": 0.28217571340240954,
      "grad_norm": 1.5585690620321229,
      "learning_rate": 9.219184339938013e-07,
      "log_odds_chosen": 0.4083496034145355,
      "log_odds_ratio": -0.6221679449081421,
      "logits/chosen": -0.7728515863418579,
      "logits/rejected": -0.707812488079071,
      "logps/chosen": -0.759960949420929,
      "logps/rejected": -1.0085937976837158,
      "loss": 0.9607,
      "nll_loss": 0.924023449420929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07609863579273224,
      "rewards/margins": 0.024852752685546875,
      "rewards/rejected": -0.10080566257238388,
      "step": 7530
    },
    {
      "epoch": 0.28255044874557345,
      "grad_norm": 1.5451927789805235,
      "learning_rate": 9.213068794097574e-07,
      "log_odds_chosen": 0.4285888671875,
      "log_odds_ratio": -0.6202148199081421,
      "logits/chosen": -0.7939453125,
      "logits/rejected": -0.7095702886581421,
      "logps/chosen": -0.691601574420929,
      "logps/rejected": -0.969921886920929,
      "loss": 0.9539,
      "nll_loss": 0.907421886920929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06916503608226776,
      "rewards/margins": 0.027883147820830345,
      "rewards/rejected": -0.09696044772863388,
      "step": 7540
    },
    {
      "epoch": 0.28292518408873735,
      "grad_norm": 1.424802838889132,
      "learning_rate": 9.206965402374975e-07,
      "log_odds_chosen": 0.41552734375,
      "log_odds_ratio": -0.631054699420929,
      "logits/chosen": -0.7416015863418579,
      "logits/rejected": -0.665234386920929,
      "logps/chosen": -0.726757824420929,
      "logps/rejected": -0.991992175579071,
      "loss": 0.9692,
      "nll_loss": 0.933398425579071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07270507514476776,
      "rewards/margins": 0.02663269080221653,
      "rewards/rejected": -0.09931640326976776,
      "step": 7550
    },
    {
      "epoch": 0.2832999194319012,
      "grad_norm": 1.5261867347711917,
      "learning_rate": 9.200874124564723e-07,
      "log_odds_chosen": 0.45061033964157104,
      "log_odds_ratio": -0.6138671636581421,
      "logits/chosen": -0.738476574420929,
      "logits/rejected": -0.648730456829071,
      "logps/chosen": -0.7564452886581421,
      "logps/rejected": -1.0470702648162842,
      "loss": 0.9738,
      "nll_loss": 0.9853515625,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07559814304113388,
      "rewards/margins": 0.02906188927590847,
      "rewards/rejected": -0.10468749701976776,
      "step": 7560
    },
    {
      "epoch": 0.2836746547750651,
      "grad_norm": 1.548413610752339,
      "learning_rate": 9.194794920647274e-07,
      "log_odds_chosen": 0.37373048067092896,
      "log_odds_ratio": -0.6456054449081421,
      "logits/chosen": -0.789843738079071,
      "logits/rejected": -0.6792968511581421,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -0.953320324420929,
      "loss": 0.9454,
      "nll_loss": 0.923046886920929,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0716552734375,
      "rewards/margins": 0.023613739758729935,
      "rewards/rejected": -0.09528808295726776,
      "step": 7570
    },
    {
      "epoch": 0.284049390118229,
      "grad_norm": 1.506023963227374,
      "learning_rate": 9.188727750787932e-07,
      "log_odds_chosen": 0.3864990174770355,
      "log_odds_ratio": -0.63916015625,
      "logits/chosen": -0.7837890386581421,
      "logits/rejected": -0.7005859613418579,
      "logps/chosen": -0.7032226324081421,
      "logps/rejected": -0.9613281488418579,
      "loss": 0.9499,
      "nll_loss": 0.9292968511581421,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07037353515625,
      "rewards/margins": 0.02573699876666069,
      "rewards/rejected": -0.09613037109375,
      "step": 7580
    },
    {
      "epoch": 0.2844241254613929,
      "grad_norm": 1.5511820171401234,
      "learning_rate": 9.182672575335757e-07,
      "log_odds_chosen": 0.48875731229782104,
      "log_odds_ratio": -0.5777343511581421,
      "logits/chosen": -0.75341796875,
      "logits/rejected": -0.659960925579071,
      "logps/chosen": -0.6805664300918579,
      "logps/rejected": -0.9837890863418579,
      "loss": 0.9709,
      "nll_loss": 0.8681640625,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06801757961511612,
      "rewards/margins": 0.03036346472799778,
      "rewards/rejected": -0.09841308742761612,
      "step": 7590
    },
    {
      "epoch": 0.2847988608045568,
      "grad_norm": 1.5975667330318324,
      "learning_rate": 9.176629354822469e-07,
      "log_odds_chosen": 0.474609375,
      "log_odds_ratio": -0.613964855670929,
      "logits/chosen": -0.751757800579071,
      "logits/rejected": -0.6558593511581421,
      "logps/chosen": -0.68994140625,
      "logps/rejected": -1.011328101158142,
      "loss": 0.9619,
      "nll_loss": 0.8388671875,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06892089545726776,
      "rewards/margins": 0.03216552734375,
      "rewards/rejected": -0.10112304985523224,
      "step": 7600
    },
    {
      "epoch": 0.2851735961477207,
      "grad_norm": 1.516552071842689,
      "learning_rate": 9.170598049961371e-07,
      "log_odds_chosen": 0.50189208984375,
      "log_odds_ratio": -0.59423828125,
      "logits/chosen": -0.714648425579071,
      "logits/rejected": -0.6312500238418579,
      "logps/chosen": -0.6919921636581421,
      "logps/rejected": -1.021484375,
      "loss": 0.9654,
      "nll_loss": 0.919726550579071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06922607123851776,
      "rewards/margins": 0.03291778638958931,
      "rewards/rejected": -0.10208740085363388,
      "step": 7610
    },
    {
      "epoch": 0.28554833149088454,
      "grad_norm": 1.5914279348751346,
      "learning_rate": 9.164578621646276e-07,
      "log_odds_chosen": 0.70770263671875,
      "log_odds_ratio": -0.529589831829071,
      "logits/chosen": -0.800976574420929,
      "logits/rejected": -0.6893554925918579,
      "logps/chosen": -0.7074218988418579,
      "logps/rejected": -1.1824219226837158,
      "loss": 0.9555,
      "nll_loss": 0.8873046636581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07076416164636612,
      "rewards/margins": 0.04743347316980362,
      "rewards/rejected": -0.1181640625,
      "step": 7620
    },
    {
      "epoch": 0.28592306683404844,
      "grad_norm": 1.5050209148260527,
      "learning_rate": 9.15857103095044e-07,
      "log_odds_chosen": 0.4359497129917145,
      "log_odds_ratio": -0.59619140625,
      "logits/chosen": -0.7767578363418579,
      "logits/rejected": -0.679492175579071,
      "logps/chosen": -0.7220703363418579,
      "logps/rejected": -0.994140625,
      "loss": 0.9706,
      "nll_loss": 0.8521484136581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07229004055261612,
      "rewards/margins": 0.02722320519387722,
      "rewards/rejected": -0.099365234375,
      "step": 7630
    },
    {
      "epoch": 0.28629780217721235,
      "grad_norm": 1.5277284612367827,
      "learning_rate": 9.15257523912551e-07,
      "log_odds_chosen": 0.5869140625,
      "log_odds_ratio": -0.568164050579071,
      "logits/chosen": -0.7835937738418579,
      "logits/rejected": -0.6859375238418579,
      "logps/chosen": -0.680859386920929,
      "logps/rejected": -1.068750023841858,
      "loss": 0.9496,
      "nll_loss": 0.8890625238418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06813964992761612,
      "rewards/margins": 0.03877716138958931,
      "rewards/rejected": -0.10688476264476776,
      "step": 7640
    },
    {
      "epoch": 0.28667253752037625,
      "grad_norm": 1.5473712178410244,
      "learning_rate": 9.146591207600472e-07,
      "log_odds_chosen": 0.3462280333042145,
      "log_odds_ratio": -0.6397460699081421,
      "logits/chosen": -0.781054675579071,
      "logits/rejected": -0.6708984375,
      "logps/chosen": -0.6968749761581421,
      "logps/rejected": -0.9066406488418579,
      "loss": 0.9474,
      "nll_loss": 0.861523449420929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06968994438648224,
      "rewards/margins": 0.02094879187643528,
      "rewards/rejected": -0.09062500298023224,
      "step": 7650
    },
    {
      "epoch": 0.28704727286354015,
      "grad_norm": 1.6533156127606754,
      "learning_rate": 9.140618897980601e-07,
      "log_odds_chosen": 0.4225097596645355,
      "log_odds_ratio": -0.5960937738418579,
      "logits/chosen": -0.764843761920929,
      "logits/rejected": -0.6923828125,
      "logps/chosen": -0.6709960699081421,
      "logps/rejected": -0.9312499761581421,
      "loss": 0.9738,
      "nll_loss": 0.8792968988418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06716308742761612,
      "rewards/margins": 0.025984954088926315,
      "rewards/rejected": -0.09311523288488388,
      "step": 7660
    },
    {
      "epoch": 0.287422008206704,
      "grad_norm": 1.6306500765043246,
      "learning_rate": 9.134658272046442e-07,
      "log_odds_chosen": 0.44072264432907104,
      "log_odds_ratio": -0.6167968511581421,
      "logits/chosen": -0.6996093988418579,
      "logits/rejected": -0.6250976324081421,
      "logps/chosen": -0.6844726800918579,
      "logps/rejected": -0.9818359613418579,
      "loss": 0.9591,
      "nll_loss": 0.874218761920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06849364936351776,
      "rewards/margins": 0.02971649169921875,
      "rewards/rejected": -0.09824218600988388,
      "step": 7670
    },
    {
      "epoch": 0.2877967435498679,
      "grad_norm": 1.5859542976025813,
      "learning_rate": 9.128709291752768e-07,
      "log_odds_chosen": 0.524517834186554,
      "log_odds_ratio": -0.5907226800918579,
      "logits/chosen": -0.7630859613418579,
      "logits/rejected": -0.6875,
      "logps/chosen": -0.6859375238418579,
      "logps/rejected": -1.008203148841858,
      "loss": 0.9548,
      "nll_loss": 0.920703113079071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06859131157398224,
      "rewards/margins": 0.03222961351275444,
      "rewards/rejected": -0.10085449367761612,
      "step": 7680
    },
    {
      "epoch": 0.2881714788930318,
      "grad_norm": 1.593327909306776,
      "learning_rate": 9.122771919227568e-07,
      "log_odds_chosen": 0.655566394329071,
      "log_odds_ratio": -0.5384765863418579,
      "logits/chosen": -0.8287109136581421,
      "logits/rejected": -0.6947265863418579,
      "logps/chosen": -0.703320324420929,
      "logps/rejected": -1.1492187976837158,
      "loss": 0.9571,
      "nll_loss": 0.864453136920929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07037353515625,
      "rewards/margins": 0.04452819749712944,
      "rewards/rejected": -0.1148681640625,
      "step": 7690
    },
    {
      "epoch": 0.2885462142361957,
      "grad_norm": 1.5736778569597565,
      "learning_rate": 9.116846116771035e-07,
      "log_odds_chosen": 0.3081420958042145,
      "log_odds_ratio": -0.646191418170929,
      "logits/chosen": -0.783398449420929,
      "logits/rejected": -0.6880859136581421,
      "logps/chosen": -0.703906238079071,
      "logps/rejected": -0.89453125,
      "loss": 0.9498,
      "nll_loss": 0.865039050579071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07036133110523224,
      "rewards/margins": 0.01913299597799778,
      "rewards/rejected": -0.0894775390625,
      "step": 7700
    },
    {
      "epoch": 0.2889209495793596,
      "grad_norm": 1.7176013338299334,
      "learning_rate": 9.110931846854553e-07,
      "log_odds_chosen": 0.287841796875,
      "log_odds_ratio": -0.6626952886581421,
      "logits/chosen": -0.75537109375,
      "logits/rejected": -0.6705077886581421,
      "logps/chosen": -0.728515625,
      "logps/rejected": -0.9263671636581421,
      "loss": 0.9733,
      "nll_loss": 0.9273437261581421,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07282714545726776,
      "rewards/margins": 0.019747162237763405,
      "rewards/rejected": -0.09256591647863388,
      "step": 7710
    },
    {
      "epoch": 0.28929568492252344,
      "grad_norm": 2.015916735545887,
      "learning_rate": 9.105029072119708e-07,
      "log_odds_chosen": 0.3920532166957855,
      "log_odds_ratio": -0.612988293170929,
      "logits/chosen": -0.7880859375,
      "logits/rejected": -0.7291015386581421,
      "logps/chosen": -0.6880859136581421,
      "logps/rejected": -0.9369140863418579,
      "loss": 0.9625,
      "nll_loss": 0.8853515386581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06878662109375,
      "rewards/margins": 0.024919509887695312,
      "rewards/rejected": -0.09371338039636612,
      "step": 7720
    },
    {
      "epoch": 0.28967042026568735,
      "grad_norm": 1.5176699201786736,
      "learning_rate": 9.099137755377291e-07,
      "log_odds_chosen": 0.47136229276657104,
      "log_odds_ratio": -0.6304687261581421,
      "logits/chosen": -0.758007824420929,
      "logits/rejected": -0.6825195550918579,
      "logps/chosen": -0.71630859375,
      "logps/rejected": -1.028222680091858,
      "loss": 0.962,
      "nll_loss": 0.875195324420929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07163085788488388,
      "rewards/margins": 0.03121643140912056,
      "rewards/rejected": -0.10280761867761612,
      "step": 7730
    },
    {
      "epoch": 0.29004515560885125,
      "grad_norm": 1.508809752032448,
      "learning_rate": 9.093257859606311e-07,
      "log_odds_chosen": 0.455810546875,
      "log_odds_ratio": -0.60986328125,
      "logits/chosen": -0.7646484375,
      "logits/rejected": -0.6548827886581421,
      "logps/chosen": -0.701367199420929,
      "logps/rejected": -0.986523449420929,
      "loss": 0.9655,
      "nll_loss": 0.8871093988418579,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07015381008386612,
      "rewards/margins": 0.028577422723174095,
      "rewards/rejected": -0.09865722805261612,
      "step": 7740
    },
    {
      "epoch": 0.29041989095201515,
      "grad_norm": 1.6882412658530184,
      "learning_rate": 9.087389347953037e-07,
      "log_odds_chosen": 0.4726806581020355,
      "log_odds_ratio": -0.60400390625,
      "logits/chosen": -0.791796863079071,
      "logits/rejected": -0.7021484375,
      "logps/chosen": -0.7012695074081421,
      "logps/rejected": -1.011328101158142,
      "loss": 0.9628,
      "nll_loss": 0.8832031488418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.070068359375,
      "rewards/margins": 0.03107757493853569,
      "rewards/rejected": -0.1011962890625,
      "step": 7750
    },
    {
      "epoch": 0.29079462629517905,
      "grad_norm": 1.6192267277860295,
      "learning_rate": 9.081532183729995e-07,
      "log_odds_chosen": 0.4541870057582855,
      "log_odds_ratio": -0.586132824420929,
      "logits/chosen": -0.7769531011581421,
      "logits/rejected": -0.6693359613418579,
      "logps/chosen": -0.6732422113418579,
      "logps/rejected": -0.972851574420929,
      "loss": 0.9651,
      "nll_loss": 0.943359375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06730957329273224,
      "rewards/margins": 0.029944609850645065,
      "rewards/rejected": -0.0972900390625,
      "step": 7760
    },
    {
      "epoch": 0.2911693616383429,
      "grad_norm": 1.5413860797750576,
      "learning_rate": 9.075686330415037e-07,
      "log_odds_chosen": 0.46568602323532104,
      "log_odds_ratio": -0.5873047113418579,
      "logits/chosen": -0.778124988079071,
      "logits/rejected": -0.6768554449081421,
      "logps/chosen": -0.6488281488418579,
      "logps/rejected": -0.9193359613418579,
      "loss": 0.9491,
      "nll_loss": 0.8841797113418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06489257514476776,
      "rewards/margins": 0.02700958214700222,
      "rewards/rejected": -0.09194336086511612,
      "step": 7770
    },
    {
      "epoch": 0.2915440969815068,
      "grad_norm": 1.5999511436403568,
      "learning_rate": 9.069851751650364e-07,
      "log_odds_chosen": 0.513378918170929,
      "log_odds_ratio": -0.5711914300918579,
      "logits/chosen": -0.772656261920929,
      "logits/rejected": -0.658398449420929,
      "logps/chosen": -0.6664062738418579,
      "logps/rejected": -0.996874988079071,
      "loss": 0.9548,
      "nll_loss": 0.927539050579071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06669922173023224,
      "rewards/margins": 0.03304290771484375,
      "rewards/rejected": -0.09978027641773224,
      "step": 7780
    },
    {
      "epoch": 0.2919188323246707,
      "grad_norm": 1.548328047768949,
      "learning_rate": 9.064028411241582e-07,
      "log_odds_chosen": 0.48859864473342896,
      "log_odds_ratio": -0.610058605670929,
      "logits/chosen": -0.785937488079071,
      "logits/rejected": -0.71826171875,
      "logps/chosen": -0.67333984375,
      "logps/rejected": -0.990429699420929,
      "loss": 0.9503,
      "nll_loss": 0.886914074420929,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.06728515774011612,
      "rewards/margins": 0.031710434705019,
      "rewards/rejected": -0.09902343899011612,
      "step": 7790
    },
    {
      "epoch": 0.2922935676678346,
      "grad_norm": 1.7178561144594153,
      "learning_rate": 9.058216273156764e-07,
      "log_odds_chosen": 0.3824706971645355,
      "log_odds_ratio": -0.613085925579071,
      "logits/chosen": -0.783398449420929,
      "logits/rejected": -0.682324230670929,
      "logps/chosen": -0.706250011920929,
      "logps/rejected": -0.951953113079071,
      "loss": 0.9594,
      "nll_loss": 0.928906261920929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07065429538488388,
      "rewards/margins": 0.02457275427877903,
      "rewards/rejected": -0.09526367485523224,
      "step": 7800
    },
    {
      "epoch": 0.2926683030109985,
      "grad_norm": 1.5192295033436887,
      "learning_rate": 9.052415301525511e-07,
      "log_odds_chosen": 0.35626220703125,
      "log_odds_ratio": -0.646777331829071,
      "logits/chosen": -0.7679687738418579,
      "logits/rejected": -0.6953125,
      "logps/chosen": -0.7457031011581421,
      "logps/rejected": -0.994140625,
      "loss": 0.9728,
      "nll_loss": 0.9429687261581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07457275688648224,
      "rewards/margins": 0.02483673021197319,
      "rewards/rejected": -0.09943847358226776,
      "step": 7810
    },
    {
      "epoch": 0.2930430383541624,
      "grad_norm": 1.5420340639617982,
      "learning_rate": 9.046625460638012e-07,
      "log_odds_chosen": 0.38066405057907104,
      "log_odds_ratio": -0.6480468511581421,
      "logits/chosen": -0.7640625238418579,
      "logits/rejected": -0.67431640625,
      "logps/chosen": -0.7171875238418579,
      "logps/rejected": -0.9742187261581421,
      "loss": 0.9654,
      "nll_loss": 0.9068359136581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07174072414636612,
      "rewards/margins": 0.02564849890768528,
      "rewards/rejected": -0.09733887016773224,
      "step": 7820
    },
    {
      "epoch": 0.29341777369732625,
      "grad_norm": 1.4843318251276756,
      "learning_rate": 9.040846714944138e-07,
      "log_odds_chosen": 0.4935058653354645,
      "log_odds_ratio": -0.5791991949081421,
      "logits/chosen": -0.780468761920929,
      "logits/rejected": -0.6998046636581421,
      "logps/chosen": -0.6664062738418579,
      "logps/rejected": -0.9576171636581421,
      "loss": 0.9568,
      "nll_loss": 0.867871105670929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06656493991613388,
      "rewards/margins": 0.02917327918112278,
      "rewards/rejected": -0.09580077975988388,
      "step": 7830
    },
    {
      "epoch": 0.29379250904049015,
      "grad_norm": 1.4927353502950682,
      "learning_rate": 9.035079029052513e-07,
      "log_odds_chosen": 0.45184326171875,
      "log_odds_ratio": -0.579296886920929,
      "logits/chosen": -0.8486328125,
      "logits/rejected": -0.727734386920929,
      "logps/chosen": -0.648144543170929,
      "logps/rejected": -0.9208984375,
      "loss": 0.964,
      "nll_loss": 0.786328136920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06483153998851776,
      "rewards/margins": 0.02729492262005806,
      "rewards/rejected": -0.092041015625,
      "step": 7840
    },
    {
      "epoch": 0.29416724438365405,
      "grad_norm": 1.4878170059846396,
      "learning_rate": 9.029322367729605e-07,
      "log_odds_chosen": 0.5196533203125,
      "log_odds_ratio": -0.5575195550918579,
      "logits/chosen": -0.800585925579071,
      "logits/rejected": -0.678906261920929,
      "logps/chosen": -0.7001953125,
      "logps/rejected": -1.0330078601837158,
      "loss": 0.9565,
      "nll_loss": 0.9283202886581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.070068359375,
      "rewards/margins": 0.0332489013671875,
      "rewards/rejected": -0.103271484375,
      "step": 7850
    },
    {
      "epoch": 0.29454197972681795,
      "grad_norm": 1.762797702642782,
      "learning_rate": 9.02357669589883e-07,
      "log_odds_chosen": 0.41943359375,
      "log_odds_ratio": -0.62890625,
      "logits/chosen": -0.8101562261581421,
      "logits/rejected": -0.7279297113418579,
      "logps/chosen": -0.763867199420929,
      "logps/rejected": -1.027734398841858,
      "loss": 0.9555,
      "nll_loss": 0.8837890625,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07642821967601776,
      "rewards/margins": 0.02635650709271431,
      "rewards/rejected": -0.10280761867761612,
      "step": 7860
    },
    {
      "epoch": 0.29491671506998185,
      "grad_norm": 1.5409570474181014,
      "learning_rate": 9.017841978639643e-07,
      "log_odds_chosen": 0.4751342833042145,
      "log_odds_ratio": -0.5860351324081421,
      "logits/chosen": -0.7549804449081421,
      "logits/rejected": -0.7127929925918579,
      "logps/chosen": -0.7216796875,
      "logps/rejected": -1.0339844226837158,
      "loss": 0.9388,
      "nll_loss": 0.8697265386581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07211913913488388,
      "rewards/margins": 0.03126373142004013,
      "rewards/rejected": -0.1033935546875,
      "step": 7870
    },
    {
      "epoch": 0.2952914504131457,
      "grad_norm": 1.72133126079081,
      "learning_rate": 9.012118181186658e-07,
      "log_odds_chosen": 0.4674438536167145,
      "log_odds_ratio": -0.588183581829071,
      "logits/chosen": -0.769335925579071,
      "logits/rejected": -0.661328136920929,
      "logps/chosen": -0.686328113079071,
      "logps/rejected": -0.9876953363418579,
      "loss": 0.964,
      "nll_loss": 0.9427734613418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06865234673023224,
      "rewards/margins": 0.03008880652487278,
      "rewards/rejected": -0.09876708686351776,
      "step": 7880
    },
    {
      "epoch": 0.2956661857563096,
      "grad_norm": 1.6439739192070477,
      "learning_rate": 9.00640526892875e-07,
      "log_odds_chosen": 0.4161132872104645,
      "log_odds_ratio": -0.61572265625,
      "logits/chosen": -0.7496093511581421,
      "logits/rejected": -0.643750011920929,
      "logps/chosen": -0.711718738079071,
      "logps/rejected": -0.993945300579071,
      "loss": 0.9678,
      "nll_loss": 0.9556640386581421,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07119140774011612,
      "rewards/margins": 0.02820587158203125,
      "rewards/rejected": -0.09935303032398224,
      "step": 7890
    },
    {
      "epoch": 0.2960409210994735,
      "grad_norm": 1.4717050704026102,
      "learning_rate": 9.000703207408191e-07,
      "log_odds_chosen": 0.5172119140625,
      "log_odds_ratio": -0.5743163824081421,
      "logits/chosen": -0.7564452886581421,
      "logits/rejected": -0.640429675579071,
      "logps/chosen": -0.711718738079071,
      "logps/rejected": -1.0314452648162842,
      "loss": 0.9709,
      "nll_loss": 0.952343761920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07108154147863388,
      "rewards/margins": 0.031931303441524506,
      "rewards/rejected": -0.10301513969898224,
      "step": 7900
    },
    {
      "epoch": 0.2964156564426374,
      "grad_norm": 1.7080110956066479,
      "learning_rate": 8.995011962319761e-07,
      "log_odds_chosen": 0.4346679747104645,
      "log_odds_ratio": -0.621777355670929,
      "logits/chosen": -0.787890613079071,
      "logits/rejected": -0.7021484375,
      "logps/chosen": -0.76171875,
      "logps/rejected": -1.070898413658142,
      "loss": 0.9504,
      "nll_loss": 0.8902343511581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07614745944738388,
      "rewards/margins": 0.03093719482421875,
      "rewards/rejected": -0.1070556640625,
      "step": 7910
    },
    {
      "epoch": 0.2967903917858013,
      "grad_norm": 1.5622227992046591,
      "learning_rate": 8.989331499509894e-07,
      "log_odds_chosen": 0.518444836139679,
      "log_odds_ratio": -0.588671863079071,
      "logits/chosen": -0.7466796636581421,
      "logits/rejected": -0.701171875,
      "logps/chosen": -0.719042956829071,
      "logps/rejected": -1.0310547351837158,
      "loss": 0.9674,
      "nll_loss": 0.8501952886581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07188721001148224,
      "rewards/margins": 0.031302642077207565,
      "rewards/rejected": -0.10317382961511612,
      "step": 7920
    },
    {
      "epoch": 0.29716512712896515,
      "grad_norm": 1.5450165290998417,
      "learning_rate": 8.983661784975812e-07,
      "log_odds_chosen": 0.4205322265625,
      "log_odds_ratio": -0.58154296875,
      "logits/chosen": -0.7865234613418579,
      "logits/rejected": -0.677929699420929,
      "logps/chosen": -0.659375011920929,
      "logps/rejected": -0.9146484136581421,
      "loss": 0.9418,
      "nll_loss": 0.871289074420929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06589355319738388,
      "rewards/margins": 0.02557067945599556,
      "rewards/rejected": -0.09150390326976776,
      "step": 7930
    },
    {
      "epoch": 0.29753986247212905,
      "grad_norm": 1.5476267864385838,
      "learning_rate": 8.97800278486467e-07,
      "log_odds_chosen": 0.34581297636032104,
      "log_odds_ratio": -0.654492199420929,
      "logits/chosen": -0.716503918170929,
      "logits/rejected": -0.636523425579071,
      "logps/chosen": -0.6913086175918579,
      "logps/rejected": -0.9267578125,
      "loss": 0.9689,
      "nll_loss": 0.896289050579071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.069091796875,
      "rewards/margins": 0.02349548414349556,
      "rewards/rejected": -0.09259033203125,
      "step": 7940
    },
    {
      "epoch": 0.29791459781529295,
      "grad_norm": 1.56447443757873,
      "learning_rate": 8.972354465472708e-07,
      "log_odds_chosen": 0.29923707246780396,
      "log_odds_ratio": -0.6751953363418579,
      "logits/chosen": -0.783398449420929,
      "logits/rejected": -0.6839843988418579,
      "logps/chosen": -0.677441418170929,
      "logps/rejected": -0.8794921636581421,
      "loss": 0.9489,
      "nll_loss": 0.8697265386581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06772460788488388,
      "rewards/margins": 0.020212555304169655,
      "rewards/rejected": -0.08796386420726776,
      "step": 7950
    },
    {
      "epoch": 0.29828933315845685,
      "grad_norm": 1.6295195288077693,
      "learning_rate": 8.966716793244405e-07,
      "log_odds_chosen": 0.410888671875,
      "log_odds_ratio": -0.638671875,
      "logits/chosen": -0.7464843988418579,
      "logits/rejected": -0.6470702886581421,
      "logps/chosen": -0.7269531488418579,
      "logps/rejected": -1.0026366710662842,
      "loss": 0.9492,
      "nll_loss": 0.868359386920929,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07271728664636612,
      "rewards/margins": 0.02755432203412056,
      "rewards/rejected": -0.10020752251148224,
      "step": 7960
    },
    {
      "epoch": 0.29866406850162075,
      "grad_norm": 1.5724835493953682,
      "learning_rate": 8.96108973477165e-07,
      "log_odds_chosen": 0.4115966856479645,
      "log_odds_ratio": -0.619140625,
      "logits/chosen": -0.779492199420929,
      "logits/rejected": -0.6680663824081421,
      "logps/chosen": -0.7144531011581421,
      "logps/rejected": -1.0041015148162842,
      "loss": 0.9319,
      "nll_loss": 0.9136718511581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07139892876148224,
      "rewards/margins": 0.02899017371237278,
      "rewards/rejected": -0.10041503608226776,
      "step": 7970
    },
    {
      "epoch": 0.2990388038447846,
      "grad_norm": 1.6262090510714253,
      "learning_rate": 8.955473256792899e-07,
      "log_odds_chosen": 0.4969848692417145,
      "log_odds_ratio": -0.592968761920929,
      "logits/chosen": -0.8115234375,
      "logits/rejected": -0.6712890863418579,
      "logps/chosen": -0.673828125,
      "logps/rejected": -0.977734386920929,
      "loss": 0.9562,
      "nll_loss": 0.8955078125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06735839694738388,
      "rewards/margins": 0.03030548058450222,
      "rewards/rejected": -0.09763183444738388,
      "step": 7980
    },
    {
      "epoch": 0.2994135391879485,
      "grad_norm": 1.5060903122124687,
      "learning_rate": 8.949867326192358e-07,
      "log_odds_chosen": 0.39783936738967896,
      "log_odds_ratio": -0.641894519329071,
      "logits/chosen": -0.750781238079071,
      "logits/rejected": -0.6597656011581421,
      "logps/chosen": -0.705078125,
      "logps/rejected": -0.9751952886581421,
      "loss": 0.9612,
      "nll_loss": 0.9583984613418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07056884467601776,
      "rewards/margins": 0.02703704871237278,
      "rewards/rejected": -0.09763183444738388,
      "step": 7990
    },
    {
      "epoch": 0.2997882745311124,
      "grad_norm": 1.6791620632860487,
      "learning_rate": 8.944271909999158e-07,
      "log_odds_chosen": 0.4085449278354645,
      "log_odds_ratio": -0.622363269329071,
      "logits/chosen": -0.8509765863418579,
      "logits/rejected": -0.741406261920929,
      "logps/chosen": -0.663281261920929,
      "logps/rejected": -0.930468738079071,
      "loss": 0.9588,
      "nll_loss": 0.9185546636581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06632079929113388,
      "rewards/margins": 0.02677154541015625,
      "rewards/rejected": -0.09311523288488388,
      "step": 8000
    },
    {
      "epoch": 0.3001630098742763,
      "grad_norm": 1.5874492101391828,
      "learning_rate": 8.938686975386545e-07,
      "log_odds_chosen": 0.3817993104457855,
      "log_odds_ratio": -0.664257824420929,
      "logits/chosen": -0.7378906011581421,
      "logits/rejected": -0.688281238079071,
      "logps/chosen": -0.7152343988418579,
      "logps/rejected": -0.9921875,
      "loss": 0.956,
      "nll_loss": 0.914257824420929,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07149658352136612,
      "rewards/margins": 0.02766265906393528,
      "rewards/rejected": -0.09916992485523224,
      "step": 8010
    },
    {
      "epoch": 0.3005377452174402,
      "grad_norm": 1.5216531044604162,
      "learning_rate": 8.933112489671067e-07,
      "log_odds_chosen": 0.34245604276657104,
      "log_odds_ratio": -0.6353515386581421,
      "logits/chosen": -0.853320300579071,
      "logits/rejected": -0.752734363079071,
      "logps/chosen": -0.69580078125,
      "logps/rejected": -0.92578125,
      "loss": 0.9534,
      "nll_loss": 0.8447265625,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.069580078125,
      "rewards/margins": 0.023015594109892845,
      "rewards/rejected": -0.0926513671875,
      "step": 8020
    },
    {
      "epoch": 0.3009124805606041,
      "grad_norm": 2.089017781702326,
      "learning_rate": 8.927548420311771e-07,
      "log_odds_chosen": 0.561572253704071,
      "log_odds_ratio": -0.576855480670929,
      "logits/chosen": -0.808398425579071,
      "logits/rejected": -0.694531261920929,
      "logps/chosen": -0.6810547113418579,
      "logps/rejected": -1.03125,
      "loss": 0.9401,
      "nll_loss": 0.895312488079071,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06815185397863388,
      "rewards/margins": 0.03495330736041069,
      "rewards/rejected": -0.10312499850988388,
      "step": 8030
    },
    {
      "epoch": 0.30128721590376795,
      "grad_norm": 1.6560072787130184,
      "learning_rate": 8.921994734909409e-07,
      "log_odds_chosen": 0.5307251214981079,
      "log_odds_ratio": -0.5868164300918579,
      "logits/chosen": -0.8216797113418579,
      "logits/rejected": -0.690625011920929,
      "logps/chosen": -0.741992175579071,
      "logps/rejected": -1.097070336341858,
      "loss": 0.9591,
      "nll_loss": 0.907421886920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07421875,
      "rewards/margins": 0.03555145114660263,
      "rewards/rejected": -0.10964355617761612,
      "step": 8040
    },
    {
      "epoch": 0.30166195124693185,
      "grad_norm": 1.5546558136606867,
      "learning_rate": 8.916451401205645e-07,
      "log_odds_chosen": 0.5962158441543579,
      "log_odds_ratio": -0.560546875,
      "logits/chosen": -0.7955077886581421,
      "logits/rejected": -0.698046863079071,
      "logps/chosen": -0.658203125,
      "logps/rejected": -1.0222656726837158,
      "loss": 0.9262,
      "nll_loss": 0.8353515863418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06582031399011612,
      "rewards/margins": 0.03647003322839737,
      "rewards/rejected": -0.102294921875,
      "step": 8050
    },
    {
      "epoch": 0.30203668659009575,
      "grad_norm": 1.5961623237062332,
      "learning_rate": 8.91091838708226e-07,
      "log_odds_chosen": 0.38713377714157104,
      "log_odds_ratio": -0.6182616949081421,
      "logits/chosen": -0.757617175579071,
      "logits/rejected": -0.675585925579071,
      "logps/chosen": -0.724609375,
      "logps/rejected": -0.981249988079071,
      "loss": 0.97,
      "nll_loss": 0.915820300579071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07242431491613388,
      "rewards/margins": 0.02567443810403347,
      "rewards/rejected": -0.09814453125,
      "step": 8060
    },
    {
      "epoch": 0.30241142193325965,
      "grad_norm": 1.6113055982650843,
      "learning_rate": 8.905395660560378e-07,
      "log_odds_chosen": 0.5679687261581421,
      "log_odds_ratio": -0.5516601800918579,
      "logits/chosen": -0.795703113079071,
      "logits/rejected": -0.654101550579071,
      "logps/chosen": -0.6396484375,
      "logps/rejected": -0.9876953363418579,
      "loss": 0.9525,
      "nll_loss": 0.858203113079071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06401367485523224,
      "rewards/margins": 0.03478393703699112,
      "rewards/rejected": -0.09880371391773224,
      "step": 8070
    },
    {
      "epoch": 0.30278615727642355,
      "grad_norm": 1.6564449815097604,
      "learning_rate": 8.899883189799695e-07,
      "log_odds_chosen": 0.29597169160842896,
      "log_odds_ratio": -0.713183581829071,
      "logits/chosen": -0.7466796636581421,
      "logits/rejected": -0.644824206829071,
      "logps/chosen": -0.7529296875,
      "logps/rejected": -0.9892578125,
      "loss": 0.9782,
      "nll_loss": 0.9525390863418579,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.0753173828125,
      "rewards/margins": 0.023659516125917435,
      "rewards/rejected": -0.09886474907398224,
      "step": 8080
    },
    {
      "epoch": 0.3031608926195874,
      "grad_norm": 1.5478930250804153,
      "learning_rate": 8.894380943097694e-07,
      "log_odds_chosen": 0.543261706829071,
      "log_odds_ratio": -0.567919909954071,
      "logits/chosen": -0.7220703363418579,
      "logits/rejected": -0.6138671636581421,
      "logps/chosen": -0.7210937738418579,
      "logps/rejected": -1.083984375,
      "loss": 0.9513,
      "nll_loss": 0.943554699420929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07209472358226776,
      "rewards/margins": 0.03625183179974556,
      "rewards/rejected": -0.10842285305261612,
      "step": 8090
    },
    {
      "epoch": 0.3035356279627513,
      "grad_norm": 1.47754036603636,
      "learning_rate": 8.888888888888888e-07,
      "log_odds_chosen": 0.46379393339157104,
      "log_odds_ratio": -0.589160144329071,
      "logits/chosen": -0.7490234375,
      "logits/rejected": -0.6512695550918579,
      "logps/chosen": -0.709667980670929,
      "logps/rejected": -0.976367175579071,
      "loss": 0.9566,
      "nll_loss": 0.9234374761581421,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07093505561351776,
      "rewards/margins": 0.02664947509765625,
      "rewards/rejected": -0.09758301079273224,
      "step": 8100
    },
    {
      "epoch": 0.3039103633059152,
      "grad_norm": 1.4593841454474032,
      "learning_rate": 8.883406995744061e-07,
      "log_odds_chosen": 0.3172363340854645,
      "log_odds_ratio": -0.6519531011581421,
      "logits/chosen": -0.789257824420929,
      "logits/rejected": -0.740234375,
      "logps/chosen": -0.705859363079071,
      "logps/rejected": -0.910351574420929,
      "loss": 0.959,
      "nll_loss": 0.876269519329071,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07064209133386612,
      "rewards/margins": 0.020429229363799095,
      "rewards/rejected": -0.09104003757238388,
      "step": 8110
    },
    {
      "epoch": 0.3042850986490791,
      "grad_norm": 1.4883800772230624,
      "learning_rate": 8.877935232369506e-07,
      "log_odds_chosen": 0.4490600526332855,
      "log_odds_ratio": -0.6026366949081421,
      "logits/chosen": -0.7738281488418579,
      "logits/rejected": -0.7056640386581421,
      "logps/chosen": -0.6998046636581421,
      "logps/rejected": -0.9896484613418579,
      "loss": 0.9425,
      "nll_loss": 0.915234386920929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.0699462890625,
      "rewards/margins": 0.02903137169778347,
      "rewards/rejected": -0.09902343899011612,
      "step": 8120
    },
    {
      "epoch": 0.304659833992243,
      "grad_norm": 1.6179250527343294,
      "learning_rate": 8.872473567606276e-07,
      "log_odds_chosen": 0.4605956971645355,
      "log_odds_ratio": -0.583789050579071,
      "logits/chosen": -0.7337890863418579,
      "logits/rejected": -0.64208984375,
      "logps/chosen": -0.70458984375,
      "logps/rejected": -0.9857422113418579,
      "loss": 0.9587,
      "nll_loss": 0.923632800579071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07049560546875,
      "rewards/margins": 0.02802429161965847,
      "rewards/rejected": -0.09859619289636612,
      "step": 8130
    },
    {
      "epoch": 0.30503456933540685,
      "grad_norm": 1.580448054524399,
      "learning_rate": 8.867021970429453e-07,
      "log_odds_chosen": 0.4758056700229645,
      "log_odds_ratio": -0.60986328125,
      "logits/chosen": -0.71044921875,
      "logits/rejected": -0.648632824420929,
      "logps/chosen": -0.7603515386581421,
      "logps/rejected": -1.0720703601837158,
      "loss": 0.9693,
      "nll_loss": 0.9359375238418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07603760063648224,
      "rewards/margins": 0.03119201585650444,
      "rewards/rejected": -0.10715331882238388,
      "step": 8140
    },
    {
      "epoch": 0.30540930467857075,
      "grad_norm": 1.4927318337982118,
      "learning_rate": 8.86158040994738e-07,
      "log_odds_chosen": 0.41737061738967896,
      "log_odds_ratio": -0.6327148675918579,
      "logits/chosen": -0.786328136920929,
      "logits/rejected": -0.6615234613418579,
      "logps/chosen": -0.705078125,
      "logps/rejected": -0.994921863079071,
      "loss": 0.9641,
      "nll_loss": 0.8716796636581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07049560546875,
      "rewards/margins": 0.02895050123333931,
      "rewards/rejected": -0.09941406548023224,
      "step": 8150
    },
    {
      "epoch": 0.30578404002173465,
      "grad_norm": 1.5544218205779021,
      "learning_rate": 8.856148855400954e-07,
      "log_odds_chosen": 0.573577880859375,
      "log_odds_ratio": -0.552441418170929,
      "logits/chosen": -0.7886718511581421,
      "logits/rejected": -0.6761718988418579,
      "logps/chosen": -0.701953113079071,
      "logps/rejected": -1.090234398841858,
      "loss": 0.9549,
      "nll_loss": 0.8414062261581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07017822563648224,
      "rewards/margins": 0.03879242017865181,
      "rewards/rejected": -0.10894775390625,
      "step": 8160
    },
    {
      "epoch": 0.30615877536489855,
      "grad_norm": 1.6774974970881278,
      "learning_rate": 8.850727276162873e-07,
      "log_odds_chosen": 0.3535400331020355,
      "log_odds_ratio": -0.6529296636581421,
      "logits/chosen": -0.7701171636581421,
      "logits/rejected": -0.6705077886581421,
      "logps/chosen": -0.7529296875,
      "logps/rejected": -0.986132800579071,
      "loss": 0.9514,
      "nll_loss": 0.913867175579071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07525634765625,
      "rewards/margins": 0.02334289625287056,
      "rewards/rejected": -0.09857177734375,
      "step": 8170
    },
    {
      "epoch": 0.30653351070806245,
      "grad_norm": 1.5794793187756075,
      "learning_rate": 8.845315641736929e-07,
      "log_odds_chosen": 0.44111329317092896,
      "log_odds_ratio": -0.6019531488418579,
      "logits/chosen": -0.7113281488418579,
      "logits/rejected": -0.6351562738418579,
      "logps/chosen": -0.701464831829071,
      "logps/rejected": -0.962890625,
      "loss": 0.9605,
      "nll_loss": 0.8929687738418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07011719048023224,
      "rewards/margins": 0.02620544470846653,
      "rewards/rejected": -0.09636230766773224,
      "step": 8180
    },
    {
      "epoch": 0.3069082460512263,
      "grad_norm": 1.717789826991417,
      "learning_rate": 8.839913921757278e-07,
      "log_odds_chosen": 0.548474133014679,
      "log_odds_ratio": -0.5673828125,
      "logits/chosen": -0.7574218511581421,
      "logits/rejected": -0.6548827886581421,
      "logps/chosen": -0.68359375,
      "logps/rejected": -1.036718726158142,
      "loss": 0.9708,
      "nll_loss": 0.9140625,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06839599460363388,
      "rewards/margins": 0.035288237035274506,
      "rewards/rejected": -0.10356445610523224,
      "step": 8190
    },
    {
      "epoch": 0.3072829813943902,
      "grad_norm": 1.493880126742016,
      "learning_rate": 8.834522085987722e-07,
      "log_odds_chosen": 0.5655761957168579,
      "log_odds_ratio": -0.5809570550918579,
      "logits/chosen": -0.7802734375,
      "logits/rejected": -0.6600586175918579,
      "logps/chosen": -0.676074206829071,
      "logps/rejected": -1.050390601158142,
      "loss": 0.9429,
      "nll_loss": 0.868945300579071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06765136867761612,
      "rewards/margins": 0.03737182542681694,
      "rewards/rejected": -0.10507812350988388,
      "step": 8200
    },
    {
      "epoch": 0.3076577167375541,
      "grad_norm": 1.4955737859246765,
      "learning_rate": 8.829140104321008e-07,
      "log_odds_chosen": 0.27360838651657104,
      "log_odds_ratio": -0.6455078125,
      "logits/chosen": -0.7603515386581421,
      "logits/rejected": -0.713183581829071,
      "logps/chosen": -0.696484386920929,
      "logps/rejected": -0.8726562261581421,
      "loss": 0.9461,
      "nll_loss": 0.896289050579071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06965331733226776,
      "rewards/margins": 0.017626190558075905,
      "rewards/rejected": -0.08723144233226776,
      "step": 8210
    },
    {
      "epoch": 0.308032452080718,
      "grad_norm": 1.4903472246918392,
      "learning_rate": 8.82376794677811e-07,
      "log_odds_chosen": 0.45759278535842896,
      "log_odds_ratio": -0.60595703125,
      "logits/chosen": -0.743945300579071,
      "logits/rejected": -0.6644531488418579,
      "logps/chosen": -0.668749988079071,
      "logps/rejected": -0.960156261920929,
      "loss": 0.9176,
      "nll_loss": 0.865039050579071,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06687011569738388,
      "rewards/margins": 0.02917327918112278,
      "rewards/rejected": -0.09597168117761612,
      "step": 8220
    },
    {
      "epoch": 0.3084071874238819,
      "grad_norm": 3.639372747041408,
      "learning_rate": 8.818405583507537e-07,
      "log_odds_chosen": 0.47016602754592896,
      "log_odds_ratio": -0.581738293170929,
      "logits/chosen": -0.7357422113418579,
      "logits/rejected": -0.632128894329071,
      "logps/chosen": -0.661816418170929,
      "logps/rejected": -0.940234363079071,
      "loss": 0.9764,
      "nll_loss": 0.9154297113418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06614990532398224,
      "rewards/margins": 0.02786102332174778,
      "rewards/rejected": -0.09404297173023224,
      "step": 8230
    },
    {
      "epoch": 0.30878192276704575,
      "grad_norm": 1.6471533377203718,
      "learning_rate": 8.813052984784634e-07,
      "log_odds_chosen": 0.4612060487270355,
      "log_odds_ratio": -0.603222668170929,
      "logits/chosen": -0.740917980670929,
      "logits/rejected": -0.6273437738418579,
      "logps/chosen": -0.6895507574081421,
      "logps/rejected": -0.9839843511581421,
      "loss": 0.953,
      "nll_loss": 0.868945300579071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06899414211511612,
      "rewards/margins": 0.02940521202981472,
      "rewards/rejected": -0.09829101711511612,
      "step": 8240
    },
    {
      "epoch": 0.30915665811020965,
      "grad_norm": 1.7390570981275164,
      "learning_rate": 8.807710121010885e-07,
      "log_odds_chosen": 0.5206543207168579,
      "log_odds_ratio": -0.556835949420929,
      "logits/chosen": -0.734667956829071,
      "logits/rejected": -0.6328125,
      "logps/chosen": -0.683789074420929,
      "logps/rejected": -1.0080077648162842,
      "loss": 0.9485,
      "nll_loss": 0.884570300579071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06840820610523224,
      "rewards/margins": 0.03242645412683487,
      "rewards/rejected": -0.10078124701976776,
      "step": 8250
    },
    {
      "epoch": 0.30953139345337355,
      "grad_norm": 1.697146961732769,
      "learning_rate": 8.802376962713231e-07,
      "log_odds_chosen": 0.5540405511856079,
      "log_odds_ratio": -0.593554675579071,
      "logits/chosen": -0.7464843988418579,
      "logits/rejected": -0.65673828125,
      "logps/chosen": -0.720507800579071,
      "logps/rejected": -1.0603516101837158,
      "loss": 0.9624,
      "nll_loss": 0.872851550579071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.072021484375,
      "rewards/margins": 0.033957671374082565,
      "rewards/rejected": -0.10600586235523224,
      "step": 8260
    },
    {
      "epoch": 0.30990612879653745,
      "grad_norm": 1.7460139506958787,
      "learning_rate": 8.797053480543386e-07,
      "log_odds_chosen": 0.5705932378768921,
      "log_odds_ratio": -0.5401366949081421,
      "logits/chosen": -0.7847656011581421,
      "logits/rejected": -0.675976574420929,
      "logps/chosen": -0.7105468511581421,
      "logps/rejected": -1.0583984851837158,
      "loss": 0.9578,
      "nll_loss": 0.8666015863418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07110595703125,
      "rewards/margins": 0.034825898706912994,
      "rewards/rejected": -0.10590820014476776,
      "step": 8270
    },
    {
      "epoch": 0.31028086413970135,
      "grad_norm": 1.6129175389385875,
      "learning_rate": 8.79173964527716e-07,
      "log_odds_chosen": 0.5891968011856079,
      "log_odds_ratio": -0.5645507574081421,
      "logits/chosen": -0.740429699420929,
      "logits/rejected": -0.637402355670929,
      "logps/chosen": -0.661816418170929,
      "logps/rejected": -1.0623047351837158,
      "loss": 0.9545,
      "nll_loss": 0.8935546875,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06617431342601776,
      "rewards/margins": 0.04001922532916069,
      "rewards/rejected": -0.10629882663488388,
      "step": 8280
    },
    {
      "epoch": 0.31065559948286525,
      "grad_norm": 1.7035101784785616,
      "learning_rate": 8.78643542781378e-07,
      "log_odds_chosen": 0.594311535358429,
      "log_odds_ratio": -0.560253918170929,
      "logits/chosen": -0.768750011920929,
      "logits/rejected": -0.690625011920929,
      "logps/chosen": -0.706835925579071,
      "logps/rejected": -1.108984351158142,
      "loss": 0.9409,
      "nll_loss": 0.8802734613418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07073974609375,
      "rewards/margins": 0.04014282301068306,
      "rewards/rejected": -0.11088867485523224,
      "step": 8290
    },
    {
      "epoch": 0.3110303348260291,
      "grad_norm": 1.624046492825026,
      "learning_rate": 8.781140799175228e-07,
      "log_odds_chosen": 0.4087158143520355,
      "log_odds_ratio": -0.64453125,
      "logits/chosen": -0.7896484136581421,
      "logits/rejected": -0.6884765625,
      "logps/chosen": -0.7193359136581421,
      "logps/rejected": -1.003320336341858,
      "loss": 0.9449,
      "nll_loss": 0.91015625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07196044921875,
      "rewards/margins": 0.02834320068359375,
      "rewards/rejected": -0.10036621242761612,
      "step": 8300
    },
    {
      "epoch": 0.311405070169193,
      "grad_norm": 1.5930965358645652,
      "learning_rate": 8.775855730505568e-07,
      "log_odds_chosen": 0.601574718952179,
      "log_odds_ratio": -0.583300769329071,
      "logits/chosen": -0.766796886920929,
      "logits/rejected": -0.6656249761581421,
      "logps/chosen": -0.6859375238418579,
      "logps/rejected": -1.0798828601837158,
      "loss": 0.9564,
      "nll_loss": 0.8916015625,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06856689602136612,
      "rewards/margins": 0.039412688463926315,
      "rewards/rejected": -0.10802002251148224,
      "step": 8310
    },
    {
      "epoch": 0.3117798055123569,
      "grad_norm": 1.5157708870763194,
      "learning_rate": 8.770580193070291e-07,
      "log_odds_chosen": 0.5656982660293579,
      "log_odds_ratio": -0.6221679449081421,
      "logits/chosen": -0.728710949420929,
      "logits/rejected": -0.600878894329071,
      "logps/chosen": -0.7021484375,
      "logps/rejected": -1.1173827648162842,
      "loss": 0.9726,
      "nll_loss": 0.917773425579071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07023926079273224,
      "rewards/margins": 0.04161987453699112,
      "rewards/rejected": -0.11184082180261612,
      "step": 8320
    },
    {
      "epoch": 0.3121545408555208,
      "grad_norm": 1.5505240479041356,
      "learning_rate": 8.765314158255661e-07,
      "log_odds_chosen": 0.4874511659145355,
      "log_odds_ratio": -0.598828136920929,
      "logits/chosen": -0.7607421875,
      "logits/rejected": -0.653027355670929,
      "logps/chosen": -0.696582019329071,
      "logps/rejected": -1.036718726158142,
      "loss": 0.9493,
      "nll_loss": 0.830859363079071,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.06968994438648224,
      "rewards/margins": 0.034101106226444244,
      "rewards/rejected": -0.10372314602136612,
      "step": 8330
    },
    {
      "epoch": 0.3125292761986847,
      "grad_norm": 1.57339928245169,
      "learning_rate": 8.760057597568057e-07,
      "log_odds_chosen": 0.22921141982078552,
      "log_odds_ratio": -0.68896484375,
      "logits/chosen": -0.7459961175918579,
      "logits/rejected": -0.673046886920929,
      "logps/chosen": -0.6949218511581421,
      "logps/rejected": -0.8421875238418579,
      "loss": 0.9494,
      "nll_loss": 0.8949218988418579,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.06944580376148224,
      "rewards/margins": 0.01471557654440403,
      "rewards/rejected": -0.08424071967601776,
      "step": 8340
    },
    {
      "epoch": 0.31290401154184855,
      "grad_norm": 1.5838727048014059,
      "learning_rate": 8.754810482633324e-07,
      "log_odds_chosen": 0.3254455626010895,
      "log_odds_ratio": -0.6532226800918579,
      "logits/chosen": -0.747851550579071,
      "logits/rejected": -0.639453113079071,
      "logps/chosen": -0.7275390625,
      "logps/rejected": -0.921093761920929,
      "loss": 0.9534,
      "nll_loss": 0.896484375,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07265625149011612,
      "rewards/margins": 0.019481658935546875,
      "rewards/rejected": -0.09215088188648224,
      "step": 8350
    },
    {
      "epoch": 0.31327874688501245,
      "grad_norm": 1.5724339908071654,
      "learning_rate": 8.749572785196142e-07,
      "log_odds_chosen": 0.555615246295929,
      "log_odds_ratio": -0.604687511920929,
      "logits/chosen": -0.751171886920929,
      "logits/rejected": -0.609570324420929,
      "logps/chosen": -0.707226574420929,
      "logps/rejected": -1.105859398841858,
      "loss": 0.9584,
      "nll_loss": 0.9193359613418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07075195014476776,
      "rewards/margins": 0.03986816480755806,
      "rewards/rejected": -0.11054687201976776,
      "step": 8360
    },
    {
      "epoch": 0.31365348222817635,
      "grad_norm": 1.539569929794516,
      "learning_rate": 8.744344477119373e-07,
      "log_odds_chosen": 0.534472644329071,
      "log_odds_ratio": -0.587109386920929,
      "logits/chosen": -0.7129882574081421,
      "logits/rejected": -0.654296875,
      "logps/chosen": -0.6776367425918579,
      "logps/rejected": -1.007226586341858,
      "loss": 0.9826,
      "nll_loss": 0.9613281488418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06777343899011612,
      "rewards/margins": 0.03302917629480362,
      "rewards/rejected": -0.10073242336511612,
      "step": 8370
    },
    {
      "epoch": 0.31402821757134025,
      "grad_norm": 1.7762938428156378,
      "learning_rate": 8.739125530383433e-07,
      "log_odds_chosen": 0.5322021245956421,
      "log_odds_ratio": -0.594921886920929,
      "logits/chosen": -0.783203125,
      "logits/rejected": -0.675976574420929,
      "logps/chosen": -0.695117175579071,
      "logps/rejected": -1.0548827648162842,
      "loss": 0.9432,
      "nll_loss": 0.8707031011581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06956787407398224,
      "rewards/margins": 0.03595580905675888,
      "rewards/rejected": -0.10546875,
      "step": 8380
    },
    {
      "epoch": 0.31440295291450415,
      "grad_norm": 1.5085864925686532,
      "learning_rate": 8.733915917085661e-07,
      "log_odds_chosen": 0.3684326112270355,
      "log_odds_ratio": -0.6241210699081421,
      "logits/chosen": -0.7708984613418579,
      "logits/rejected": -0.6796875,
      "logps/chosen": -0.689648449420929,
      "logps/rejected": -0.9349609613418579,
      "loss": 0.9514,
      "nll_loss": 0.8818359375,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.0689697265625,
      "rewards/margins": 0.02456970140337944,
      "rewards/rejected": -0.09357909858226776,
      "step": 8390
    },
    {
      "epoch": 0.314777688257668,
      "grad_norm": 1.6703687967425307,
      "learning_rate": 8.728715609439695e-07,
      "log_odds_chosen": 0.5283447504043579,
      "log_odds_ratio": -0.602343738079071,
      "logits/chosen": -0.7232421636581421,
      "logits/rejected": -0.635449230670929,
      "logps/chosen": -0.708789050579071,
      "logps/rejected": -1.068359375,
      "loss": 0.9728,
      "nll_loss": 0.9222656488418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07086181640625,
      "rewards/margins": 0.03585357591509819,
      "rewards/rejected": -0.10671386867761612,
      "step": 8400
    },
    {
      "epoch": 0.3151524236008319,
      "grad_norm": 1.6728044894451861,
      "learning_rate": 8.72352457977484e-07,
      "log_odds_chosen": 0.3649536073207855,
      "log_odds_ratio": -0.617382824420929,
      "logits/chosen": -0.763671875,
      "logits/rejected": -0.6826171875,
      "logps/chosen": -0.6787109375,
      "logps/rejected": -0.884960949420929,
      "loss": 0.9326,
      "nll_loss": 0.865429699420929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06784667819738388,
      "rewards/margins": 0.0206298828125,
      "rewards/rejected": -0.08847656100988388,
      "step": 8410
    },
    {
      "epoch": 0.3155271589439958,
      "grad_norm": 1.6239942389684707,
      "learning_rate": 8.718342800535456e-07,
      "log_odds_chosen": 0.3980468809604645,
      "log_odds_ratio": -0.6337890625,
      "logits/chosen": -0.699414074420929,
      "logits/rejected": -0.6220703125,
      "logps/chosen": -0.700390636920929,
      "logps/rejected": -0.9395507574081421,
      "loss": 0.9728,
      "nll_loss": 0.9013671875,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07008056342601776,
      "rewards/margins": 0.02390136756002903,
      "rewards/rejected": -0.09403076022863388,
      "step": 8420
    },
    {
      "epoch": 0.3159018942871597,
      "grad_norm": 1.5902427452440255,
      "learning_rate": 8.713170244280353e-07,
      "log_odds_chosen": 0.4078613221645355,
      "log_odds_ratio": -0.628222644329071,
      "logits/chosen": -0.747265636920929,
      "logits/rejected": -0.639843761920929,
      "logps/chosen": -0.730664074420929,
      "logps/rejected": -1.000585913658142,
      "loss": 0.9644,
      "nll_loss": 0.9917968511581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07310791313648224,
      "rewards/margins": 0.02696533128619194,
      "rewards/rejected": -0.10009765625,
      "step": 8430
    },
    {
      "epoch": 0.3162766296303236,
      "grad_norm": 1.558572997350103,
      "learning_rate": 8.708006883682162e-07,
      "log_odds_chosen": 0.4173583984375,
      "log_odds_ratio": -0.6025390625,
      "logits/chosen": -0.7505859136581421,
      "logits/rejected": -0.6732422113418579,
      "logps/chosen": -0.70751953125,
      "logps/rejected": -0.9595702886581421,
      "loss": 0.9606,
      "nll_loss": 0.852343738079071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07078857719898224,
      "rewards/margins": 0.025249481201171875,
      "rewards/rejected": -0.09598388522863388,
      "step": 8440
    },
    {
      "epoch": 0.31665136497348745,
      "grad_norm": 1.6947655049296908,
      "learning_rate": 8.702852691526739e-07,
      "log_odds_chosen": 0.4844970703125,
      "log_odds_ratio": -0.5870116949081421,
      "logits/chosen": -0.765429675579071,
      "logits/rejected": -0.6875,
      "logps/chosen": -0.761523425579071,
      "logps/rejected": -1.091406226158142,
      "loss": 0.9546,
      "nll_loss": 0.970507800579071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.076171875,
      "rewards/margins": 0.03295745700597763,
      "rewards/rejected": -0.10911865532398224,
      "step": 8450
    },
    {
      "epoch": 0.31702610031665135,
      "grad_norm": 1.631506455066722,
      "learning_rate": 8.697707640712562e-07,
      "log_odds_chosen": 0.48609620332717896,
      "log_odds_ratio": -0.574414074420929,
      "logits/chosen": -0.70263671875,
      "logits/rejected": -0.6346679925918579,
      "logps/chosen": -0.671191394329071,
      "logps/rejected": -0.9712890386581421,
      "loss": 0.956,
      "nll_loss": 0.885058581829071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06712646782398224,
      "rewards/margins": 0.02999572828412056,
      "rewards/rejected": -0.09719238430261612,
      "step": 8460
    },
    {
      "epoch": 0.31740083565981525,
      "grad_norm": 1.4901604521721996,
      "learning_rate": 8.692571704250135e-07,
      "log_odds_chosen": 0.635302722454071,
      "log_odds_ratio": -0.544726550579071,
      "logits/chosen": -0.76611328125,
      "logits/rejected": -0.690625011920929,
      "logps/chosen": -0.701171875,
      "logps/rejected": -1.1181640625,
      "loss": 0.9435,
      "nll_loss": 0.878125011920929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.070068359375,
      "rewards/margins": 0.04179535061120987,
      "rewards/rejected": -0.11186523735523224,
      "step": 8470
    },
    {
      "epoch": 0.31777557100297915,
      "grad_norm": 1.6016350345934758,
      "learning_rate": 8.687444855261388e-07,
      "log_odds_chosen": 0.3653564453125,
      "log_odds_ratio": -0.6260741949081421,
      "logits/chosen": -0.7298828363418579,
      "logits/rejected": -0.68212890625,
      "logps/chosen": -0.72265625,
      "logps/rejected": -0.9556640386581421,
      "loss": 0.9353,
      "nll_loss": 0.9537109136581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.072265625,
      "rewards/margins": 0.02324829064309597,
      "rewards/rejected": -0.09560547024011612,
      "step": 8480
    },
    {
      "epoch": 0.31815030634614305,
      "grad_norm": 1.60262754470117,
      "learning_rate": 8.682327066979084e-07,
      "log_odds_chosen": 0.4605956971645355,
      "log_odds_ratio": -0.606640636920929,
      "logits/chosen": -0.783398449420929,
      "logits/rejected": -0.7021484375,
      "logps/chosen": -0.7113281488418579,
      "logps/rejected": -0.98828125,
      "loss": 0.9472,
      "nll_loss": 0.906054675579071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07119140774011612,
      "rewards/margins": 0.02765655517578125,
      "rewards/rejected": -0.09882812201976776,
      "step": 8490
    },
    {
      "epoch": 0.31852504168930695,
      "grad_norm": 1.5515119207781052,
      "learning_rate": 8.677218312746247e-07,
      "log_odds_chosen": 0.30128175020217896,
      "log_odds_ratio": -0.6744140386581421,
      "logits/chosen": -0.776171863079071,
      "logits/rejected": -0.677539050579071,
      "logps/chosen": -0.6924804449081421,
      "logps/rejected": -0.888476550579071,
      "loss": 0.9402,
      "nll_loss": 0.874218761920929,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06925048679113388,
      "rewards/margins": 0.019588470458984375,
      "rewards/rejected": -0.08878173679113388,
      "step": 8500
    },
    {
      "epoch": 0.3188997770324708,
      "grad_norm": 1.8222508937568287,
      "learning_rate": 8.672118566015558e-07,
      "log_odds_chosen": 0.43065184354782104,
      "log_odds_ratio": -0.597851574420929,
      "logits/chosen": -0.767773449420929,
      "logits/rejected": -0.6854492425918579,
      "logps/chosen": -0.7095702886581421,
      "logps/rejected": -0.9888671636581421,
      "loss": 0.9542,
      "nll_loss": 0.929492175579071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07098388671875,
      "rewards/margins": 0.02798614464700222,
      "rewards/rejected": -0.09893798828125,
      "step": 8510
    },
    {
      "epoch": 0.3192745123756347,
      "grad_norm": 1.535416040850429,
      "learning_rate": 8.667027800348789e-07,
      "log_odds_chosen": 0.5799316167831421,
      "log_odds_ratio": -0.54931640625,
      "logits/chosen": -0.7914062738418579,
      "logits/rejected": -0.6737304925918579,
      "logps/chosen": -0.683398425579071,
      "logps/rejected": -1.0402343273162842,
      "loss": 0.9294,
      "nll_loss": 0.8828125,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.068359375,
      "rewards/margins": 0.03578529506921768,
      "rewards/rejected": -0.10416259616613388,
      "step": 8520
    },
    {
      "epoch": 0.3196492477187986,
      "grad_norm": 1.5015431765145562,
      "learning_rate": 8.661945989416229e-07,
      "log_odds_chosen": 0.620288074016571,
      "log_odds_ratio": -0.540820300579071,
      "logits/chosen": -0.8271484375,
      "logits/rejected": -0.652148425579071,
      "logps/chosen": -0.684765636920929,
      "logps/rejected": -1.066796898841858,
      "loss": 0.9545,
      "nll_loss": 0.854296863079071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06850586086511612,
      "rewards/margins": 0.03824157640337944,
      "rewards/rejected": -0.10664062201976776,
      "step": 8530
    },
    {
      "epoch": 0.3200239830619625,
      "grad_norm": 1.4853717330928933,
      "learning_rate": 8.6568731069961e-07,
      "log_odds_chosen": 0.5149902105331421,
      "log_odds_ratio": -0.5787109136581421,
      "logits/chosen": -0.7544921636581421,
      "logits/rejected": -0.647656261920929,
      "logps/chosen": -0.7212890386581421,
      "logps/rejected": -1.065820336341858,
      "loss": 0.9569,
      "nll_loss": 0.9341796636581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07213135063648224,
      "rewards/margins": 0.03448333591222763,
      "rewards/rejected": -0.106689453125,
      "step": 8540
    },
    {
      "epoch": 0.3203987184051264,
      "grad_norm": 1.8457821689742049,
      "learning_rate": 8.651809126974002e-07,
      "log_odds_chosen": 0.5739990472793579,
      "log_odds_ratio": -0.568652331829071,
      "logits/chosen": -0.73388671875,
      "logits/rejected": -0.645703136920929,
      "logps/chosen": -0.681640625,
      "logps/rejected": -1.056054711341858,
      "loss": 0.9406,
      "nll_loss": 0.8902343511581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06815185397863388,
      "rewards/margins": 0.037508390843868256,
      "rewards/rejected": -0.10567627102136612,
      "step": 8550
    },
    {
      "epoch": 0.32077345374829025,
      "grad_norm": 1.6548159366304271,
      "learning_rate": 8.646754023342339e-07,
      "log_odds_chosen": 0.43293458223342896,
      "log_odds_ratio": -0.619433581829071,
      "logits/chosen": -0.71484375,
      "logits/rejected": -0.6357421875,
      "logps/chosen": -0.6728515625,
      "logps/rejected": -0.952343761920929,
      "loss": 0.9349,
      "nll_loss": 0.881640613079071,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0672607421875,
      "rewards/margins": 0.027959441766142845,
      "rewards/rejected": -0.09520263969898224,
      "step": 8560
    },
    {
      "epoch": 0.32114818909145415,
      "grad_norm": 1.4526596427035807,
      "learning_rate": 8.64170777019976e-07,
      "log_odds_chosen": 0.5521484613418579,
      "log_odds_ratio": -0.5869140625,
      "logits/chosen": -0.7367187738418579,
      "logits/rejected": -0.653613269329071,
      "logps/chosen": -0.7103515863418579,
      "logps/rejected": -1.0832030773162842,
      "loss": 0.9704,
      "nll_loss": 0.8548828363418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07094726711511612,
      "rewards/margins": 0.03731079027056694,
      "rewards/rejected": -0.10825195163488388,
      "step": 8570
    },
    {
      "epoch": 0.32152292443461805,
      "grad_norm": 1.5174860400132406,
      "learning_rate": 8.636670341750609e-07,
      "log_odds_chosen": 0.49260252714157104,
      "log_odds_ratio": -0.583691418170929,
      "logits/chosen": -0.7378906011581421,
      "logits/rejected": -0.643847644329071,
      "logps/chosen": -0.67578125,
      "logps/rejected": -1.0009765625,
      "loss": 0.9425,
      "nll_loss": 0.8681640625,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06761474907398224,
      "rewards/margins": 0.0325496681034565,
      "rewards/rejected": -0.10019531100988388,
      "step": 8580
    },
    {
      "epoch": 0.32189765977778195,
      "grad_norm": 1.5836584746145028,
      "learning_rate": 8.631641712304359e-07,
      "log_odds_chosen": 0.47252196073532104,
      "log_odds_ratio": -0.600878894329071,
      "logits/chosen": -0.740429699420929,
      "logits/rejected": -0.639453113079071,
      "logps/chosen": -0.725781261920929,
      "logps/rejected": -1.037695288658142,
      "loss": 0.9566,
      "nll_loss": 0.9195312261581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07260742038488388,
      "rewards/margins": 0.031119536608457565,
      "rewards/rejected": -0.10371093451976776,
      "step": 8590
    },
    {
      "epoch": 0.32227239512094585,
      "grad_norm": 1.7509305107872384,
      "learning_rate": 8.626621856275073e-07,
      "log_odds_chosen": 0.4427490234375,
      "log_odds_ratio": -0.66015625,
      "logits/chosen": -0.714160144329071,
      "logits/rejected": -0.6552734375,
      "logps/chosen": -0.7152343988418579,
      "logps/rejected": -1.019140601158142,
      "loss": 0.9517,
      "nll_loss": 0.854296863079071,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07154540717601776,
      "rewards/margins": 0.03029327467083931,
      "rewards/rejected": -0.10185547173023224,
      "step": 8600
    },
    {
      "epoch": 0.3226471304641097,
      "grad_norm": 1.5882210196861724,
      "learning_rate": 8.621610748180847e-07,
      "log_odds_chosen": 0.6162353754043579,
      "log_odds_ratio": -0.5497070550918579,
      "logits/chosen": -0.73046875,
      "logits/rejected": -0.6143554449081421,
      "logps/chosen": -0.67529296875,
      "logps/rejected": -1.060937523841858,
      "loss": 0.9208,
      "nll_loss": 0.8994140625,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06759033352136612,
      "rewards/margins": 0.038495637476444244,
      "rewards/rejected": -0.1060791015625,
      "step": 8610
    },
    {
      "epoch": 0.3230218658072736,
      "grad_norm": 1.6027465829784917,
      "learning_rate": 8.616608362643274e-07,
      "log_odds_chosen": 0.38164061307907104,
      "log_odds_ratio": -0.6273437738418579,
      "logits/chosen": -0.7440429925918579,
      "logits/rejected": -0.62353515625,
      "logps/chosen": -0.731249988079071,
      "logps/rejected": -0.9750000238418579,
      "loss": 0.9793,
      "nll_loss": 0.897656261920929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07310791313648224,
      "rewards/margins": 0.024353791028261185,
      "rewards/rejected": -0.09746094048023224,
      "step": 8620
    },
    {
      "epoch": 0.3233966011504375,
      "grad_norm": 1.7630695091664563,
      "learning_rate": 8.611614674386904e-07,
      "log_odds_chosen": 0.45649415254592896,
      "log_odds_ratio": -0.595703125,
      "logits/chosen": -0.732226550579071,
      "logits/rejected": -0.662304699420929,
      "logps/chosen": -0.667187511920929,
      "logps/rejected": -0.949023425579071,
      "loss": 0.9432,
      "nll_loss": 0.8531249761581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06673584133386612,
      "rewards/margins": 0.02812499925494194,
      "rewards/rejected": -0.0948486328125,
      "step": 8630
    },
    {
      "epoch": 0.3237713364936014,
      "grad_norm": 1.5297633841539717,
      "learning_rate": 8.606629658238703e-07,
      "log_odds_chosen": 0.506121814250946,
      "log_odds_ratio": -0.5902343988418579,
      "logits/chosen": -0.7298828363418579,
      "logits/rejected": -0.62451171875,
      "logps/chosen": -0.716503918170929,
      "logps/rejected": -1.044335961341858,
      "loss": 0.9463,
      "nll_loss": 0.903124988079071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0716552734375,
      "rewards/margins": 0.032837677747011185,
      "rewards/rejected": -0.10446777194738388,
      "step": 8640
    },
    {
      "epoch": 0.3241460718367653,
      "grad_norm": 1.6156031846213315,
      "learning_rate": 8.601653289127525e-07,
      "log_odds_chosen": 0.4281372129917145,
      "log_odds_ratio": -0.60693359375,
      "logits/chosen": -0.747851550579071,
      "logits/rejected": -0.635937511920929,
      "logps/chosen": -0.641406238079071,
      "logps/rejected": -0.9017578363418579,
      "loss": 0.9548,
      "nll_loss": 0.885546863079071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06418456882238388,
      "rewards/margins": 0.026036834344267845,
      "rewards/rejected": -0.0902099609375,
      "step": 8650
    },
    {
      "epoch": 0.32452080717992915,
      "grad_norm": 1.5589010391176978,
      "learning_rate": 8.596685542083577e-07,
      "log_odds_chosen": 0.423828125,
      "log_odds_ratio": -0.6419922113418579,
      "logits/chosen": -0.703906238079071,
      "logits/rejected": -0.62646484375,
      "logps/chosen": -0.713671863079071,
      "logps/rejected": -1.007226586341858,
      "loss": 0.9608,
      "nll_loss": 0.8628906011581421,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07136230170726776,
      "rewards/margins": 0.029270172119140625,
      "rewards/rejected": -0.10063476860523224,
      "step": 8660
    },
    {
      "epoch": 0.32489554252309305,
      "grad_norm": 2.0950370525305733,
      "learning_rate": 8.591726392237899e-07,
      "log_odds_chosen": 0.56085205078125,
      "log_odds_ratio": -0.5630859136581421,
      "logits/chosen": -0.746289074420929,
      "logits/rejected": -0.6416991949081421,
      "logps/chosen": -0.6650390625,
      "logps/rejected": -1.027734398841858,
      "loss": 0.9423,
      "nll_loss": 0.8648437261581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06647948920726776,
      "rewards/margins": 0.03626556321978569,
      "rewards/rejected": -0.10280761867761612,
      "step": 8670
    },
    {
      "epoch": 0.32527027786625695,
      "grad_norm": 1.526192576566616,
      "learning_rate": 8.586775814821837e-07,
      "log_odds_chosen": 0.37957763671875,
      "log_odds_ratio": -0.62890625,
      "logits/chosen": -0.7520507574081421,
      "logits/rejected": -0.6717773675918579,
      "logps/chosen": -0.7139648199081421,
      "logps/rejected": -0.9732421636581421,
      "loss": 0.9571,
      "nll_loss": 0.871874988079071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07137451320886612,
      "rewards/margins": 0.025922393426299095,
      "rewards/rejected": -0.0972900390625,
      "step": 8680
    },
    {
      "epoch": 0.32564501320942085,
      "grad_norm": 1.5742942038968164,
      "learning_rate": 8.58183378516652e-07,
      "log_odds_chosen": 0.560864269733429,
      "log_odds_ratio": -0.5721679925918579,
      "logits/chosen": -0.7225586175918579,
      "logits/rejected": -0.625,
      "logps/chosen": -0.665234386920929,
      "logps/rejected": -1.05078125,
      "loss": 0.9483,
      "nll_loss": 0.8560546636581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06650390475988388,
      "rewards/margins": 0.03864288330078125,
      "rewards/rejected": -0.10507812350988388,
      "step": 8690
    },
    {
      "epoch": 0.32601974855258475,
      "grad_norm": 1.575082021629467,
      "learning_rate": 8.576900278702358e-07,
      "log_odds_chosen": 0.5042968988418579,
      "log_odds_ratio": -0.6034179925918579,
      "logits/chosen": -0.7826172113418579,
      "logits/rejected": -0.698437511920929,
      "logps/chosen": -0.6885741949081421,
      "logps/rejected": -1.0271484851837158,
      "loss": 0.9364,
      "nll_loss": 0.8662109375,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06881103664636612,
      "rewards/margins": 0.03384704515337944,
      "rewards/rejected": -0.10270996391773224,
      "step": 8700
    },
    {
      "epoch": 0.32639448389574865,
      "grad_norm": 1.6904525939565653,
      "learning_rate": 8.57197527095851e-07,
      "log_odds_chosen": 0.47954100370407104,
      "log_odds_ratio": -0.58935546875,
      "logits/chosen": -0.771679699420929,
      "logits/rejected": -0.689746081829071,
      "logps/chosen": -0.7093750238418579,
      "logps/rejected": -1.019140601158142,
      "loss": 0.9407,
      "nll_loss": 0.882031261920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07089843600988388,
      "rewards/margins": 0.03100738488137722,
      "rewards/rejected": -0.10190429538488388,
      "step": 8710
    },
    {
      "epoch": 0.3267692192389125,
      "grad_norm": 1.7520852339543467,
      "learning_rate": 8.567058737562385e-07,
      "log_odds_chosen": 0.6585937738418579,
      "log_odds_ratio": -0.55810546875,
      "logits/chosen": -0.7783203125,
      "logits/rejected": -0.669921875,
      "logps/chosen": -0.707226574420929,
      "logps/rejected": -1.1408202648162842,
      "loss": 0.9736,
      "nll_loss": 0.9263671636581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07069091498851776,
      "rewards/margins": 0.04331665113568306,
      "rewards/rejected": -0.11406250298023224,
      "step": 8720
    },
    {
      "epoch": 0.3271439545820764,
      "grad_norm": 1.601338535973298,
      "learning_rate": 8.562150654239141e-07,
      "log_odds_chosen": 0.37592774629592896,
      "log_odds_ratio": -0.6610351800918579,
      "logits/chosen": -0.7158203125,
      "logits/rejected": -0.6346679925918579,
      "logps/chosen": -0.6996093988418579,
      "logps/rejected": -0.959765613079071,
      "loss": 0.9581,
      "nll_loss": 0.956835925579071,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.06993408501148224,
      "rewards/margins": 0.02601318433880806,
      "rewards/rejected": -0.09592285007238388,
      "step": 8730
    },
    {
      "epoch": 0.3275186899252403,
      "grad_norm": 1.582282507699851,
      "learning_rate": 8.55725099681116e-07,
      "log_odds_chosen": 0.47779542207717896,
      "log_odds_ratio": -0.612597644329071,
      "logits/chosen": -0.736328125,
      "logits/rejected": -0.640332043170929,
      "logps/chosen": -0.6922851800918579,
      "logps/rejected": -1.0087890625,
      "loss": 0.9547,
      "nll_loss": 0.8822265863418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06916503608226776,
      "rewards/margins": 0.031737517565488815,
      "rewards/rejected": -0.10092773288488388,
      "step": 8740
    },
    {
      "epoch": 0.3278934252684042,
      "grad_norm": 1.4998735456459156,
      "learning_rate": 8.552359741197579e-07,
      "log_odds_chosen": 0.49995118379592896,
      "log_odds_ratio": -0.590136706829071,
      "logits/chosen": -0.73974609375,
      "logits/rejected": -0.677539050579071,
      "logps/chosen": -0.7181640863418579,
      "logps/rejected": -1.024999976158142,
      "loss": 0.9582,
      "nll_loss": 0.897265613079071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07180175930261612,
      "rewards/margins": 0.03067779541015625,
      "rewards/rejected": -0.1025390625,
      "step": 8750
    },
    {
      "epoch": 0.3282681606115681,
      "grad_norm": 1.7246434729500917,
      "learning_rate": 8.547476863413765e-07,
      "log_odds_chosen": 0.48420411348342896,
      "log_odds_ratio": -0.5874999761581421,
      "logits/chosen": -0.7359374761581421,
      "logits/rejected": -0.6558593511581421,
      "logps/chosen": -0.683398425579071,
      "logps/rejected": -0.9886718988418579,
      "loss": 0.9476,
      "nll_loss": 0.900195300579071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06839599460363388,
      "rewards/margins": 0.03057098388671875,
      "rewards/rejected": -0.09901122748851776,
      "step": 8760
    },
    {
      "epoch": 0.32864289595473195,
      "grad_norm": 1.5353051963824629,
      "learning_rate": 8.54260233957083e-07,
      "log_odds_chosen": 0.5293334722518921,
      "log_odds_ratio": -0.568554699420929,
      "logits/chosen": -0.782421886920929,
      "logits/rejected": -0.6439453363418579,
      "logps/chosen": -0.7105468511581421,
      "logps/rejected": -1.043359398841858,
      "loss": 0.943,
      "nll_loss": 0.8792968988418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07103271782398224,
      "rewards/margins": 0.033294677734375,
      "rewards/rejected": -0.10432128608226776,
      "step": 8770
    },
    {
      "epoch": 0.32901763129789585,
      "grad_norm": 1.790003699585134,
      "learning_rate": 8.537736145875154e-07,
      "log_odds_chosen": 0.4715332090854645,
      "log_odds_ratio": -0.6045898199081421,
      "logits/chosen": -0.6880859136581421,
      "logits/rejected": -0.632519543170929,
      "logps/chosen": -0.685742199420929,
      "logps/rejected": -0.9794921875,
      "loss": 0.9558,
      "nll_loss": 0.903124988079071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06851806491613388,
      "rewards/margins": 0.02941436693072319,
      "rewards/rejected": -0.09801025688648224,
      "step": 8780
    },
    {
      "epoch": 0.32939236664105975,
      "grad_norm": 1.8099625878280516,
      "learning_rate": 8.532878258627874e-07,
      "log_odds_chosen": 0.4048217833042145,
      "log_odds_ratio": -0.608691394329071,
      "logits/chosen": -0.720507800579071,
      "logits/rejected": -0.6435546875,
      "logps/chosen": -0.7173827886581421,
      "logps/rejected": -0.9789062738418579,
      "loss": 0.9594,
      "nll_loss": 0.9400390386581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07175292819738388,
      "rewards/margins": 0.02607421949505806,
      "rewards/rejected": -0.09787597507238388,
      "step": 8790
    },
    {
      "epoch": 0.32976710198422365,
      "grad_norm": 1.6577307695840193,
      "learning_rate": 8.528028654224416e-07,
      "log_odds_chosen": 0.546191394329071,
      "log_odds_ratio": -0.5648437738418579,
      "logits/chosen": -0.7275390625,
      "logits/rejected": -0.6387695074081421,
      "logps/chosen": -0.7007812261581421,
      "logps/rejected": -1.044921875,
      "loss": 0.9442,
      "nll_loss": 0.8681640625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07005615532398224,
      "rewards/margins": 0.03441505506634712,
      "rewards/rejected": -0.1044921875,
      "step": 8800
    },
    {
      "epoch": 0.33014183732738756,
      "grad_norm": 1.6640421853547318,
      "learning_rate": 8.523187309154008e-07,
      "log_odds_chosen": 0.6134033203125,
      "log_odds_ratio": -0.575390636920929,
      "logits/chosen": -0.716113269329071,
      "logits/rejected": -0.619140625,
      "logps/chosen": -0.708203136920929,
      "logps/rejected": -1.119140625,
      "loss": 0.9603,
      "nll_loss": 0.9297851324081421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07078857719898224,
      "rewards/margins": 0.041141510009765625,
      "rewards/rejected": -0.11196289211511612,
      "step": 8810
    },
    {
      "epoch": 0.3305165726705514,
      "grad_norm": 1.6722094006195283,
      "learning_rate": 8.518354199999198e-07,
      "log_odds_chosen": 0.49956053495407104,
      "log_odds_ratio": -0.5948241949081421,
      "logits/chosen": -0.6898437738418579,
      "logits/rejected": -0.59326171875,
      "logps/chosen": -0.6900390386581421,
      "logps/rejected": -0.994921863079071,
      "loss": 0.9533,
      "nll_loss": 0.9576171636581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06900634616613388,
      "rewards/margins": 0.030405426397919655,
      "rewards/rejected": -0.09941406548023224,
      "step": 8820
    },
    {
      "epoch": 0.3308913080137153,
      "grad_norm": 1.6704854827420337,
      "learning_rate": 8.513529303435386e-07,
      "log_odds_chosen": 0.2464599609375,
      "log_odds_ratio": -0.68798828125,
      "logits/chosen": -0.7593749761581421,
      "logits/rejected": -0.6556640863418579,
      "logps/chosen": -0.69921875,
      "logps/rejected": -0.8578125238418579,
      "loss": 0.9355,
      "nll_loss": 0.8695312738418579,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.06989745795726776,
      "rewards/margins": 0.015938948839902878,
      "rewards/rejected": -0.08588866889476776,
      "step": 8830
    },
    {
      "epoch": 0.3312660433568792,
      "grad_norm": 1.639464724846932,
      "learning_rate": 8.50871259623034e-07,
      "log_odds_chosen": 0.360595703125,
      "log_odds_ratio": -0.6636718511581421,
      "logits/chosen": -0.7294921875,
      "logits/rejected": -0.637988269329071,
      "logps/chosen": -0.711718738079071,
      "logps/rejected": -0.969433605670929,
      "loss": 0.9484,
      "nll_loss": 0.9115234613418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07117919623851776,
      "rewards/margins": 0.025823211297392845,
      "rewards/rejected": -0.09705810248851776,
      "step": 8840
    },
    {
      "epoch": 0.3316407787000431,
      "grad_norm": 1.6367021208624744,
      "learning_rate": 8.503904055243742e-07,
      "log_odds_chosen": 0.482177734375,
      "log_odds_ratio": -0.600390613079071,
      "logits/chosen": -0.6957031488418579,
      "logits/rejected": -0.6279296875,
      "logps/chosen": -0.7066406011581421,
      "logps/rejected": -1.034570336341858,
      "loss": 0.9431,
      "nll_loss": 0.9078124761581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07056884467601776,
      "rewards/margins": 0.03287658840417862,
      "rewards/rejected": -0.10352782905101776,
      "step": 8850
    },
    {
      "epoch": 0.332015514043207,
      "grad_norm": 1.9533631940414908,
      "learning_rate": 8.499103657426704e-07,
      "log_odds_chosen": 0.3165649473667145,
      "log_odds_ratio": -0.6553710699081421,
      "logits/chosen": -0.706835925579071,
      "logits/rejected": -0.645703136920929,
      "logps/chosen": -0.751953125,
      "logps/rejected": -0.949414074420929,
      "loss": 0.9374,
      "nll_loss": 0.860546886920929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07521972805261612,
      "rewards/margins": 0.0197296142578125,
      "rewards/rejected": -0.09492187201976776,
      "step": 8860
    },
    {
      "epoch": 0.33239024938637085,
      "grad_norm": 1.540172543035465,
      "learning_rate": 8.494311379821314e-07,
      "log_odds_chosen": 0.585498034954071,
      "log_odds_ratio": -0.5870116949081421,
      "logits/chosen": -0.7007812261581421,
      "logits/rejected": -0.5953124761581421,
      "logps/chosen": -0.7181640863418579,
      "logps/rejected": -1.085351586341858,
      "loss": 0.9383,
      "nll_loss": 0.9189453125,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07175292819738388,
      "rewards/margins": 0.036693572998046875,
      "rewards/rejected": -0.10849609225988388,
      "step": 8870
    },
    {
      "epoch": 0.33276498472953475,
      "grad_norm": 1.6383846314115946,
      "learning_rate": 8.489527199560178e-07,
      "log_odds_chosen": 0.38896483182907104,
      "log_odds_ratio": -0.609667956829071,
      "logits/chosen": -0.6996093988418579,
      "logits/rejected": -0.642382800579071,
      "logps/chosen": -0.6630859375,
      "logps/rejected": -0.891406238079071,
      "loss": 0.9419,
      "nll_loss": 0.931640625,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06625976413488388,
      "rewards/margins": 0.02280426025390625,
      "rewards/rejected": -0.089111328125,
      "step": 8880
    },
    {
      "epoch": 0.33313972007269865,
      "grad_norm": 1.661637556228715,
      "learning_rate": 8.484751093865948e-07,
      "log_odds_chosen": 0.3582519590854645,
      "log_odds_ratio": -0.618847668170929,
      "logits/chosen": -0.7652343511581421,
      "logits/rejected": -0.6708984375,
      "logps/chosen": -0.7359374761581421,
      "logps/rejected": -0.959179699420929,
      "loss": 0.9396,
      "nll_loss": 0.9037109613418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07363281399011612,
      "rewards/margins": 0.02233734168112278,
      "rewards/rejected": -0.095947265625,
      "step": 8890
    },
    {
      "epoch": 0.33351445541586255,
      "grad_norm": 1.5638824089797465,
      "learning_rate": 8.47998304005088e-07,
      "log_odds_chosen": 0.2709594666957855,
      "log_odds_ratio": -0.673632800579071,
      "logits/chosen": -0.7767578363418579,
      "logits/rejected": -0.6871093511581421,
      "logps/chosen": -0.7025390863418579,
      "logps/rejected": -0.8788086175918579,
      "loss": 0.9468,
      "nll_loss": 0.8999999761581421,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07028808444738388,
      "rewards/margins": 0.017547607421875,
      "rewards/rejected": -0.08775635063648224,
      "step": 8900
    },
    {
      "epoch": 0.33388919075902646,
      "grad_norm": 1.6372458734434792,
      "learning_rate": 8.475223015516377e-07,
      "log_odds_chosen": 0.4284912049770355,
      "log_odds_ratio": -0.637011706829071,
      "logits/chosen": -0.755664050579071,
      "logits/rejected": -0.675488293170929,
      "logps/chosen": -0.703808605670929,
      "logps/rejected": -0.9750000238418579,
      "loss": 0.9709,
      "nll_loss": 0.874218761920929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07030029594898224,
      "rewards/margins": 0.02715911902487278,
      "rewards/rejected": -0.09746094048023224,
      "step": 8910
    },
    {
      "epoch": 0.3342639261021903,
      "grad_norm": 1.620964136983903,
      "learning_rate": 8.470470997752534e-07,
      "log_odds_chosen": 0.42662352323532104,
      "log_odds_ratio": -0.61767578125,
      "logits/chosen": -0.7373046875,
      "logits/rejected": -0.66357421875,
      "logps/chosen": -0.7603515386581421,
      "logps/rejected": -1.041406273841858,
      "loss": 0.9604,
      "nll_loss": 0.8828125,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07603760063648224,
      "rewards/margins": 0.02803649939596653,
      "rewards/rejected": -0.10406494140625,
      "step": 8920
    },
    {
      "epoch": 0.3346386614453542,
      "grad_norm": 1.6393231741714684,
      "learning_rate": 8.465726964337702e-07,
      "log_odds_chosen": 0.2630615234375,
      "log_odds_ratio": -0.6947265863418579,
      "logits/chosen": -0.766406238079071,
      "logits/rejected": -0.682421863079071,
      "logps/chosen": -0.683789074420929,
      "logps/rejected": -0.84423828125,
      "loss": 0.945,
      "nll_loss": 0.872265636920929,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.068359375,
      "rewards/margins": 0.016100693494081497,
      "rewards/rejected": -0.08448486030101776,
      "step": 8930
    },
    {
      "epoch": 0.3350133967885181,
      "grad_norm": 1.6480942094338582,
      "learning_rate": 8.460990892938031e-07,
      "log_odds_chosen": 0.561230480670929,
      "log_odds_ratio": -0.567187488079071,
      "logits/chosen": -0.731152355670929,
      "logits/rejected": -0.6201171875,
      "logps/chosen": -0.671191394329071,
      "logps/rejected": -1.009765625,
      "loss": 0.9294,
      "nll_loss": 0.8843749761581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06707763671875,
      "rewards/margins": 0.03375549241900444,
      "rewards/rejected": -0.10085449367761612,
      "step": 8940
    },
    {
      "epoch": 0.335388132131682,
      "grad_norm": 1.6211547548678185,
      "learning_rate": 8.456262761307038e-07,
      "log_odds_chosen": 0.4714111387729645,
      "log_odds_ratio": -0.6083008050918579,
      "logits/chosen": -0.770703136920929,
      "logits/rejected": -0.666796863079071,
      "logps/chosen": -0.712695300579071,
      "logps/rejected": -1.033789038658142,
      "loss": 0.9645,
      "nll_loss": 0.9144531488418579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.0712890625,
      "rewards/margins": 0.032141875475645065,
      "rewards/rejected": -0.10334472358226776,
      "step": 8950
    },
    {
      "epoch": 0.3357628674748459,
      "grad_norm": 1.5703200353361595,
      "learning_rate": 8.451542547285166e-07,
      "log_odds_chosen": 0.39281004667282104,
      "log_odds_ratio": -0.620410144329071,
      "logits/chosen": -0.779003918170929,
      "logits/rejected": -0.6636718511581421,
      "logps/chosen": -0.7261718511581421,
      "logps/rejected": -0.966992199420929,
      "loss": 0.9267,
      "nll_loss": 0.840624988079071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07269287109375,
      "rewards/margins": 0.02403717115521431,
      "rewards/rejected": -0.0966796875,
      "step": 8960
    },
    {
      "epoch": 0.3361376028180098,
      "grad_norm": 1.8055703586836112,
      "learning_rate": 8.44683022879934e-07,
      "log_odds_chosen": 0.4660888612270355,
      "log_odds_ratio": -0.605273425579071,
      "logits/chosen": -0.737500011920929,
      "logits/rejected": -0.64404296875,
      "logps/chosen": -0.704882800579071,
      "logps/rejected": -0.997753918170929,
      "loss": 0.9466,
      "nll_loss": 0.8871093988418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07052002102136612,
      "rewards/margins": 0.02928772009909153,
      "rewards/rejected": -0.09970702975988388,
      "step": 8970
    },
    {
      "epoch": 0.33651233816117365,
      "grad_norm": 1.7578644560677177,
      "learning_rate": 8.442125783862544e-07,
      "log_odds_chosen": 0.49244385957717896,
      "log_odds_ratio": -0.58837890625,
      "logits/chosen": -0.768750011920929,
      "logits/rejected": -0.633496105670929,
      "logps/chosen": -0.6859375238418579,
      "logps/rejected": -1.024999976158142,
      "loss": 0.9474,
      "nll_loss": 0.873828113079071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06859131157398224,
      "rewards/margins": 0.0338592529296875,
      "rewards/rejected": -0.10244140774011612,
      "step": 8980
    },
    {
      "epoch": 0.33688707350433755,
      "grad_norm": 1.5758851285731201,
      "learning_rate": 8.437429190573388e-07,
      "log_odds_chosen": 0.25816649198532104,
      "log_odds_ratio": -0.700390636920929,
      "logits/chosen": -0.744140625,
      "logits/rejected": -0.671679675579071,
      "logps/chosen": -0.741015613079071,
      "logps/rejected": -0.9244140386581421,
      "loss": 0.9484,
      "nll_loss": 0.883105456829071,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.07406006008386612,
      "rewards/margins": 0.01836700364947319,
      "rewards/rejected": -0.09245605766773224,
      "step": 8990
    },
    {
      "epoch": 0.33726180884750145,
      "grad_norm": 1.4475782542304396,
      "learning_rate": 8.432740427115678e-07,
      "log_odds_chosen": 0.595166027545929,
      "log_odds_ratio": -0.550000011920929,
      "logits/chosen": -0.7572265863418579,
      "logits/rejected": -0.6732422113418579,
      "logps/chosen": -0.6441406011581421,
      "logps/rejected": -1.026953101158142,
      "loss": 0.9435,
      "nll_loss": 0.8583984375,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06442870944738388,
      "rewards/margins": 0.038237761706113815,
      "rewards/rejected": -0.1026611328125,
      "step": 9000
    },
    {
      "epoch": 0.33763654419066536,
      "grad_norm": 1.7396422652725385,
      "learning_rate": 8.428059471757984e-07,
      "log_odds_chosen": 0.39842528104782104,
      "log_odds_ratio": -0.6309570074081421,
      "logits/chosen": -0.791796863079071,
      "logits/rejected": -0.692578136920929,
      "logps/chosen": -0.701171875,
      "logps/rejected": -0.953320324420929,
      "loss": 0.9679,
      "nll_loss": 0.900585949420929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07015381008386612,
      "rewards/margins": 0.02515563927590847,
      "rewards/rejected": -0.09523925930261612,
      "step": 9010
    },
    {
      "epoch": 0.33801127953382926,
      "grad_norm": 1.6960071385144535,
      "learning_rate": 8.423386302853226e-07,
      "log_odds_chosen": 0.609790027141571,
      "log_odds_ratio": -0.573925793170929,
      "logits/chosen": -0.775585949420929,
      "logits/rejected": -0.672656238079071,
      "logps/chosen": -0.6958984136581421,
      "logps/rejected": -1.079687476158142,
      "loss": 0.9495,
      "nll_loss": 0.8623046875,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06959228217601776,
      "rewards/margins": 0.038393400609493256,
      "rewards/rejected": -0.10799560695886612,
      "step": 9020
    },
    {
      "epoch": 0.3383860148769931,
      "grad_norm": 1.628044087098894,
      "learning_rate": 8.418720898838254e-07,
      "log_odds_chosen": 0.3836608827114105,
      "log_odds_ratio": -0.6507812738418579,
      "logits/chosen": -0.745898425579071,
      "logits/rejected": -0.6751953363418579,
      "logps/chosen": -0.764453113079071,
      "logps/rejected": -1.0478515625,
      "loss": 0.9626,
      "nll_loss": 0.9427734613418579,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07644043117761612,
      "rewards/margins": 0.02835540845990181,
      "rewards/rejected": -0.10480956733226776,
      "step": 9030
    },
    {
      "epoch": 0.338760750220157,
      "grad_norm": 1.681682883883179,
      "learning_rate": 8.414063238233425e-07,
      "log_odds_chosen": 0.51507568359375,
      "log_odds_ratio": -0.600781261920929,
      "logits/chosen": -0.6924804449081421,
      "logits/rejected": -0.6146484613418579,
      "logps/chosen": -0.7406250238418579,
      "logps/rejected": -1.084570288658142,
      "loss": 0.9808,
      "nll_loss": 0.9527343511581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07404784858226776,
      "rewards/margins": 0.03440246731042862,
      "rewards/rejected": -0.10847167670726776,
      "step": 9040
    },
    {
      "epoch": 0.3391354855633209,
      "grad_norm": 1.5611528221264726,
      "learning_rate": 8.409413299642188e-07,
      "log_odds_chosen": 0.49604493379592896,
      "log_odds_ratio": -0.5956054925918579,
      "logits/chosen": -0.775390625,
      "logits/rejected": -0.6885741949081421,
      "logps/chosen": -0.7142578363418579,
      "logps/rejected": -1.055273413658142,
      "loss": 0.9141,
      "nll_loss": 0.9134765863418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07143554836511612,
      "rewards/margins": 0.03415069729089737,
      "rewards/rejected": -0.10549316555261612,
      "step": 9050
    },
    {
      "epoch": 0.3395102209064848,
      "grad_norm": 1.6739302278133978,
      "learning_rate": 8.404771061750672e-07,
      "log_odds_chosen": 0.3790283203125,
      "log_odds_ratio": -0.6207031011581421,
      "logits/chosen": -0.7383788824081421,
      "logits/rejected": -0.6297851800918579,
      "logps/chosen": -0.6957031488418579,
      "logps/rejected": -0.9466797113418579,
      "loss": 0.9352,
      "nll_loss": 0.8330078125,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06955566257238388,
      "rewards/margins": 0.02512512169778347,
      "rewards/rejected": -0.0946044921875,
      "step": 9060
    },
    {
      "epoch": 0.3398849562496487,
      "grad_norm": 1.5919213287325162,
      "learning_rate": 8.400136503327277e-07,
      "log_odds_chosen": 0.3276123106479645,
      "log_odds_ratio": -0.667285144329071,
      "logits/chosen": -0.73681640625,
      "logits/rejected": -0.691113293170929,
      "logps/chosen": -0.689453125,
      "logps/rejected": -0.9078124761581421,
      "loss": 0.97,
      "nll_loss": 0.8294922113418579,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06893310695886612,
      "rewards/margins": 0.021866608411073685,
      "rewards/rejected": -0.09077148139476776,
      "step": 9070
    },
    {
      "epoch": 0.34025969159281255,
      "grad_norm": 1.6310046510776355,
      "learning_rate": 8.395509603222271e-07,
      "log_odds_chosen": 0.43388670682907104,
      "log_odds_ratio": -0.610546886920929,
      "logits/chosen": -0.7271484136581421,
      "logits/rejected": -0.6402343511581421,
      "logps/chosen": -0.6890624761581421,
      "logps/rejected": -0.974804699420929,
      "loss": 0.9346,
      "nll_loss": 0.927734375,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06898193061351776,
      "rewards/margins": 0.02856292761862278,
      "rewards/rejected": -0.09746094048023224,
      "step": 9080
    },
    {
      "epoch": 0.34063442693597645,
      "grad_norm": 1.5443780076414233,
      "learning_rate": 8.390890340367368e-07,
      "log_odds_chosen": 0.43572998046875,
      "log_odds_ratio": -0.6319335699081421,
      "logits/chosen": -0.7216796875,
      "logits/rejected": -0.615429699420929,
      "logps/chosen": -0.7144531011581421,
      "logps/rejected": -0.9791015386581421,
      "loss": 0.9303,
      "nll_loss": 0.8599609136581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07147216796875,
      "rewards/margins": 0.02641754224896431,
      "rewards/rejected": -0.09787597507238388,
      "step": 9090
    },
    {
      "epoch": 0.34100916227914035,
      "grad_norm": 1.652198148588578,
      "learning_rate": 8.386278693775346e-07,
      "log_odds_chosen": 0.49750977754592896,
      "log_odds_ratio": -0.583300769329071,
      "logits/chosen": -0.773242175579071,
      "logits/rejected": -0.6732422113418579,
      "logps/chosen": -0.7162109613418579,
      "logps/rejected": -1.041406273841858,
      "loss": 0.9538,
      "nll_loss": 0.8958984613418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0716552734375,
      "rewards/margins": 0.03244323655962944,
      "rewards/rejected": -0.10410156100988388,
      "step": 9100
    },
    {
      "epoch": 0.34138389762230426,
      "grad_norm": 1.713660781032561,
      "learning_rate": 8.381674642539632e-07,
      "log_odds_chosen": 0.37403565645217896,
      "log_odds_ratio": -0.6253906488418579,
      "logits/chosen": -0.727832019329071,
      "logits/rejected": -0.6617187261581421,
      "logps/chosen": -0.7080078125,
      "logps/rejected": -0.958203136920929,
      "loss": 0.9478,
      "nll_loss": 0.9058593511581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07075195014476776,
      "rewards/margins": 0.02493896521627903,
      "rewards/rejected": -0.09573974460363388,
      "step": 9110
    },
    {
      "epoch": 0.34175863296546816,
      "grad_norm": 1.7350237405449216,
      "learning_rate": 8.37707816583391e-07,
      "log_odds_chosen": 0.514080822467804,
      "log_odds_ratio": -0.571972668170929,
      "logits/chosen": -0.7364257574081421,
      "logits/rejected": -0.6524413824081421,
      "logps/chosen": -0.6666015386581421,
      "logps/rejected": -0.9867187738418579,
      "loss": 0.9361,
      "nll_loss": 0.870312511920929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06674804538488388,
      "rewards/margins": 0.032019805163145065,
      "rewards/rejected": -0.09874267876148224,
      "step": 9120
    },
    {
      "epoch": 0.342133368308632,
      "grad_norm": 1.5934333245461996,
      "learning_rate": 8.372489242911724e-07,
      "log_odds_chosen": 0.3865722715854645,
      "log_odds_ratio": -0.652294933795929,
      "logits/chosen": -0.697460949420929,
      "logits/rejected": -0.621777355670929,
      "logps/chosen": -0.683398425579071,
      "logps/rejected": -0.9603515863418579,
      "loss": 0.9323,
      "nll_loss": 0.8736327886581421,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.068359375,
      "rewards/margins": 0.027666473761200905,
      "rewards/rejected": -0.09597168117761612,
      "step": 9130
    },
    {
      "epoch": 0.3425081036517959,
      "grad_norm": 1.5007470887058836,
      "learning_rate": 8.367907853106078e-07,
      "log_odds_chosen": 0.4310974180698395,
      "log_odds_ratio": -0.603222668170929,
      "logits/chosen": -0.823437511920929,
      "logits/rejected": -0.6982421875,
      "logps/chosen": -0.667773425579071,
      "logps/rejected": -0.927734375,
      "loss": 0.9231,
      "nll_loss": 0.826367199420929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06674804538488388,
      "rewards/margins": 0.02595214918255806,
      "rewards/rejected": -0.09278564155101776,
      "step": 9140
    },
    {
      "epoch": 0.3428828389949598,
      "grad_norm": 1.6927018235100078,
      "learning_rate": 8.363333975829066e-07,
      "log_odds_chosen": 0.4173217713832855,
      "log_odds_ratio": -0.65283203125,
      "logits/chosen": -0.731249988079071,
      "logits/rejected": -0.645312488079071,
      "logps/chosen": -0.712695300579071,
      "logps/rejected": -1.007226586341858,
      "loss": 0.9654,
      "nll_loss": 0.89990234375,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07125244289636612,
      "rewards/margins": 0.02940673753619194,
      "rewards/rejected": -0.10065917670726776,
      "step": 9150
    },
    {
      "epoch": 0.3432575743381237,
      "grad_norm": 1.6090148588099225,
      "learning_rate": 8.358767590571457e-07,
      "log_odds_chosen": 0.2704834043979645,
      "log_odds_ratio": -0.689453125,
      "logits/chosen": -0.7388671636581421,
      "logits/rejected": -0.66552734375,
      "logps/chosen": -0.7017577886581421,
      "logps/rejected": -0.8802734613418579,
      "loss": 0.9525,
      "nll_loss": 0.9087890386581421,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07027588039636612,
      "rewards/margins": 0.017781829461455345,
      "rewards/rejected": -0.08804931491613388,
      "step": 9160
    },
    {
      "epoch": 0.3436323096812876,
      "grad_norm": 1.680134170017807,
      "learning_rate": 8.354208676902326e-07,
      "log_odds_chosen": 0.534106433391571,
      "log_odds_ratio": -0.580371081829071,
      "logits/chosen": -0.740527331829071,
      "logits/rejected": -0.6629883050918579,
      "logps/chosen": -0.650195300579071,
      "logps/rejected": -0.966601550579071,
      "loss": 0.9297,
      "nll_loss": 0.9306640625,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06500244140625,
      "rewards/margins": 0.031653594225645065,
      "rewards/rejected": -0.09663085639476776,
      "step": 9170
    },
    {
      "epoch": 0.3440070450244515,
      "grad_norm": 1.7166956752545983,
      "learning_rate": 8.349657214468659e-07,
      "log_odds_chosen": 0.4844970703125,
      "log_odds_ratio": -0.587597668170929,
      "logits/chosen": -0.750781238079071,
      "logits/rejected": -0.654101550579071,
      "logps/chosen": -0.685253918170929,
      "logps/rejected": -0.9800781011581421,
      "loss": 0.9463,
      "nll_loss": 0.897265613079071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0684814453125,
      "rewards/margins": 0.02944488450884819,
      "rewards/rejected": -0.09794922173023224,
      "step": 9180
    },
    {
      "epoch": 0.34438178036761535,
      "grad_norm": 1.6918139833490462,
      "learning_rate": 8.345113182994988e-07,
      "log_odds_chosen": 0.3943237364292145,
      "log_odds_ratio": -0.638476550579071,
      "logits/chosen": -0.75146484375,
      "logits/rejected": -0.673828125,
      "logps/chosen": -0.736523449420929,
      "logps/rejected": -0.9710937738418579,
      "loss": 0.9417,
      "nll_loss": 0.8892577886581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0736083984375,
      "rewards/margins": 0.02344665490090847,
      "rewards/rejected": -0.09707031399011612,
      "step": 9190
    },
    {
      "epoch": 0.34475651571077925,
      "grad_norm": 2.9693099307637003,
      "learning_rate": 8.34057656228299e-07,
      "log_odds_chosen": 0.503662109375,
      "log_odds_ratio": -0.581738293170929,
      "logits/chosen": -0.768359363079071,
      "logits/rejected": -0.669726550579071,
      "logps/chosen": -0.70703125,
      "logps/rejected": -1.056054711341858,
      "loss": 0.9516,
      "nll_loss": 0.8794921636581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07061767578125,
      "rewards/margins": 0.03487548977136612,
      "rewards/rejected": -0.10546875,
      "step": 9200
    },
    {
      "epoch": 0.34513125105394316,
      "grad_norm": 1.472286965697829,
      "learning_rate": 8.336047332211128e-07,
      "log_odds_chosen": 0.47453612089157104,
      "log_odds_ratio": -0.5997070074081421,
      "logits/chosen": -0.7646484375,
      "logits/rejected": -0.6747070550918579,
      "logps/chosen": -0.670117199420929,
      "logps/rejected": -0.98046875,
      "loss": 0.9474,
      "nll_loss": 0.8919922113418579,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06691894680261612,
      "rewards/margins": 0.03113708458840847,
      "rewards/rejected": -0.0980224609375,
      "step": 9210
    },
    {
      "epoch": 0.34550598639710706,
      "grad_norm": 1.679441808271671,
      "learning_rate": 8.331525472734267e-07,
      "log_odds_chosen": 0.47413331270217896,
      "log_odds_ratio": -0.5966796875,
      "logits/chosen": -0.8056640625,
      "logits/rejected": -0.695605456829071,
      "logps/chosen": -0.650585949420929,
      "logps/rejected": -0.9457031488418579,
      "loss": 0.9101,
      "nll_loss": 0.859570324420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06507568061351776,
      "rewards/margins": 0.02945861779153347,
      "rewards/rejected": -0.09455566108226776,
      "step": 9220
    },
    {
      "epoch": 0.34588072174027096,
      "grad_norm": 1.6660416522299646,
      "learning_rate": 8.327010963883302e-07,
      "log_odds_chosen": 0.4417480528354645,
      "log_odds_ratio": -0.6036132574081421,
      "logits/chosen": -0.758984386920929,
      "logits/rejected": -0.665332019329071,
      "logps/chosen": -0.698437511920929,
      "logps/rejected": -1.0027344226837158,
      "loss": 0.951,
      "nll_loss": 0.9722656011581421,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06988525390625,
      "rewards/margins": 0.03045501746237278,
      "rewards/rejected": -0.10032959282398224,
      "step": 9230
    },
    {
      "epoch": 0.3462554570834348,
      "grad_norm": 1.8040536013745105,
      "learning_rate": 8.322503785764789e-07,
      "log_odds_chosen": 0.364990234375,
      "log_odds_ratio": -0.662792980670929,
      "logits/chosen": -0.7298828363418579,
      "logits/rejected": -0.6581054925918579,
      "logps/chosen": -0.7154296636581421,
      "logps/rejected": -0.9791015386581421,
      "loss": 0.9353,
      "nll_loss": 0.898242175579071,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07158203423023224,
      "rewards/margins": 0.02631225623190403,
      "rewards/rejected": -0.097900390625,
      "step": 9240
    },
    {
      "epoch": 0.3466301924265987,
      "grad_norm": 1.6028127989403882,
      "learning_rate": 8.318003918560583e-07,
      "log_odds_chosen": 0.5550781488418579,
      "log_odds_ratio": -0.583691418170929,
      "logits/chosen": -0.7738281488418579,
      "logits/rejected": -0.671679675579071,
      "logps/chosen": -0.689160168170929,
      "logps/rejected": -1.0402343273162842,
      "loss": 0.953,
      "nll_loss": 0.892382800579071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06890869140625,
      "rewards/margins": 0.035158537328243256,
      "rewards/rejected": -0.10402832180261612,
      "step": 9250
    },
    {
      "epoch": 0.3470049277697626,
      "grad_norm": 1.6180189586147704,
      "learning_rate": 8.313511342527453e-07,
      "log_odds_chosen": 0.3587646484375,
      "log_odds_ratio": -0.650195300579071,
      "logits/chosen": -0.7583984136581421,
      "logits/rejected": -0.651562511920929,
      "logps/chosen": -0.747265636920929,
      "logps/rejected": -0.997851550579071,
      "loss": 0.9294,
      "nll_loss": 0.908203125,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07469482719898224,
      "rewards/margins": 0.02515563927590847,
      "rewards/rejected": -0.099853515625,
      "step": 9260
    },
    {
      "epoch": 0.3473796631129265,
      "grad_norm": 1.7435849442539912,
      "learning_rate": 8.309026037996745e-07,
      "log_odds_chosen": 0.29182130098342896,
      "log_odds_ratio": -0.6572265625,
      "logits/chosen": -0.7369140386581421,
      "logits/rejected": -0.657519519329071,
      "logps/chosen": -0.6993163824081421,
      "logps/rejected": -0.876757800579071,
      "loss": 0.9164,
      "nll_loss": 0.8775390386581421,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.06993408501148224,
      "rewards/margins": 0.01777343824505806,
      "rewards/rejected": -0.08769531548023224,
      "step": 9270
    },
    {
      "epoch": 0.3477543984560904,
      "grad_norm": 1.6091134309407122,
      "learning_rate": 8.304547985373996e-07,
      "log_odds_chosen": 0.5260254144668579,
      "log_odds_ratio": -0.60546875,
      "logits/chosen": -0.7601562738418579,
      "logits/rejected": -0.633105456829071,
      "logps/chosen": -0.681445300579071,
      "logps/rejected": -1.0265624523162842,
      "loss": 0.9407,
      "nll_loss": 0.881640613079071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06812743842601776,
      "rewards/margins": 0.034531403332948685,
      "rewards/rejected": -0.10273437201976776,
      "step": 9280
    },
    {
      "epoch": 0.34812913379925425,
      "grad_norm": 1.6541055669259745,
      "learning_rate": 8.300077165138592e-07,
      "log_odds_chosen": 0.429443359375,
      "log_odds_ratio": -0.611621081829071,
      "logits/chosen": -0.75390625,
      "logits/rejected": -0.6200195550918579,
      "logps/chosen": -0.6812499761581421,
      "logps/rejected": -0.9603515863418579,
      "loss": 0.9463,
      "nll_loss": 0.8912109136581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.068115234375,
      "rewards/margins": 0.02795867994427681,
      "rewards/rejected": -0.09603271633386612,
      "step": 9290
    },
    {
      "epoch": 0.34850386914241815,
      "grad_norm": 1.4501849698749867,
      "learning_rate": 8.295613557843402e-07,
      "log_odds_chosen": 0.4812988340854645,
      "log_odds_ratio": -0.612011730670929,
      "logits/chosen": -0.68505859375,
      "logits/rejected": -0.6141601800918579,
      "logps/chosen": -0.6405273675918579,
      "logps/rejected": -0.9244140386581421,
      "loss": 0.9477,
      "nll_loss": 0.955273449420929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06409911811351776,
      "rewards/margins": 0.02831573411822319,
      "rewards/rejected": -0.0924072265625,
      "step": 9300
    },
    {
      "epoch": 0.34887860448558206,
      "grad_norm": 1.6585488973963987,
      "learning_rate": 8.291157144114419e-07,
      "log_odds_chosen": 0.44664305448532104,
      "log_odds_ratio": -0.624804675579071,
      "logits/chosen": -0.7279297113418579,
      "logits/rejected": -0.6328125,
      "logps/chosen": -0.695117175579071,
      "logps/rejected": -0.986328125,
      "loss": 0.9393,
      "nll_loss": 0.861523449420929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06955566257238388,
      "rewards/margins": 0.02910919114947319,
      "rewards/rejected": -0.09869384765625,
      "step": 9310
    },
    {
      "epoch": 0.34925333982874596,
      "grad_norm": 1.6747026649068237,
      "learning_rate": 8.286707904650417e-07,
      "log_odds_chosen": 0.33551025390625,
      "log_odds_ratio": -0.6405273675918579,
      "logits/chosen": -0.7601562738418579,
      "logits/rejected": -0.681640625,
      "logps/chosen": -0.691210925579071,
      "logps/rejected": -0.8912109136581421,
      "loss": 0.9469,
      "nll_loss": 0.8447265625,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06914062798023224,
      "rewards/margins": 0.02001342736184597,
      "rewards/rejected": -0.08909912407398224,
      "step": 9320
    },
    {
      "epoch": 0.34962807517190986,
      "grad_norm": 1.6632361637639403,
      "learning_rate": 8.282265820222593e-07,
      "log_odds_chosen": 0.5107421875,
      "log_odds_ratio": -0.583300769329071,
      "logits/chosen": -0.748242199420929,
      "logits/rejected": -0.6871093511581421,
      "logps/chosen": -0.6615234613418579,
      "logps/rejected": -0.9847656488418579,
      "loss": 0.9302,
      "nll_loss": 0.8042968511581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06614990532398224,
      "rewards/margins": 0.03233947604894638,
      "rewards/rejected": -0.09846191108226776,
      "step": 9330
    },
    {
      "epoch": 0.3500028105150737,
      "grad_norm": 1.4756627167649314,
      "learning_rate": 8.277830871674222e-07,
      "log_odds_chosen": 0.4212890565395355,
      "log_odds_ratio": -0.610644519329071,
      "logits/chosen": -0.7476562261581421,
      "logits/rejected": -0.653515636920929,
      "logps/chosen": -0.682910144329071,
      "logps/rejected": -0.945507824420929,
      "loss": 0.9459,
      "nll_loss": 0.861132800579071,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06831054389476776,
      "rewards/margins": 0.02626647986471653,
      "rewards/rejected": -0.09454345703125,
      "step": 9340
    },
    {
      "epoch": 0.3503775458582376,
      "grad_norm": 1.6787038509876329,
      "learning_rate": 8.273403039920306e-07,
      "log_odds_chosen": 0.4155334532260895,
      "log_odds_ratio": -0.623730480670929,
      "logits/chosen": -0.7857421636581421,
      "logits/rejected": -0.6958984136581421,
      "logps/chosen": -0.7408202886581421,
      "logps/rejected": -1.0412108898162842,
      "loss": 0.9278,
      "nll_loss": 0.9203125238418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07403564453125,
      "rewards/margins": 0.030042266473174095,
      "rewards/rejected": -0.10407714545726776,
      "step": 9350
    },
    {
      "epoch": 0.3507522812014015,
      "grad_norm": 1.5760261859534686,
      "learning_rate": 8.268982305947231e-07,
      "log_odds_chosen": 0.31403809785842896,
      "log_odds_ratio": -0.653124988079071,
      "logits/chosen": -0.802929699420929,
      "logits/rejected": -0.724609375,
      "logps/chosen": -0.7408202886581421,
      "logps/rejected": -0.9585937261581421,
      "loss": 0.9367,
      "nll_loss": 0.868945300579071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07404784858226776,
      "rewards/margins": 0.02183227613568306,
      "rewards/rejected": -0.09588623046875,
      "step": 9360
    },
    {
      "epoch": 0.3511270165445654,
      "grad_norm": 1.664233149573346,
      "learning_rate": 8.264568650812423e-07,
      "log_odds_chosen": 0.47297364473342896,
      "log_odds_ratio": -0.5816406011581421,
      "logits/chosen": -0.792187511920929,
      "logits/rejected": -0.67431640625,
      "logps/chosen": -0.7088867425918579,
      "logps/rejected": -1.0085937976837158,
      "loss": 0.9451,
      "nll_loss": 0.892773449420929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07086181640625,
      "rewards/margins": 0.03000793419778347,
      "rewards/rejected": -0.10090331733226776,
      "step": 9370
    },
    {
      "epoch": 0.3515017518877293,
      "grad_norm": 1.6745646213740475,
      "learning_rate": 8.26016205564401e-07,
      "log_odds_chosen": 0.589648425579071,
      "log_odds_ratio": -0.5855468511581421,
      "logits/chosen": -0.7958984375,
      "logits/rejected": -0.671875,
      "logps/chosen": -0.7383788824081421,
      "logps/rejected": -1.163476586341858,
      "loss": 0.9471,
      "nll_loss": 0.8880859613418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07379150390625,
      "rewards/margins": 0.04251403734087944,
      "rewards/rejected": -0.11630859225988388,
      "step": 9380
    },
    {
      "epoch": 0.3518764872308932,
      "grad_norm": 1.7842296206851211,
      "learning_rate": 8.25576250164048e-07,
      "log_odds_chosen": 0.564343273639679,
      "log_odds_ratio": -0.576855480670929,
      "logits/chosen": -0.762499988079071,
      "logits/rejected": -0.667285144329071,
      "logps/chosen": -0.6693359613418579,
      "logps/rejected": -1.0380859375,
      "loss": 0.9582,
      "nll_loss": 0.883984386920929,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06689453125,
      "rewards/margins": 0.036895751953125,
      "rewards/rejected": -0.1038818359375,
      "step": 9390
    },
    {
      "epoch": 0.35225122257405705,
      "grad_norm": 1.5967301529927407,
      "learning_rate": 8.251369970070346e-07,
      "log_odds_chosen": 0.35511475801467896,
      "log_odds_ratio": -0.651074230670929,
      "logits/chosen": -0.769335925579071,
      "logits/rejected": -0.7412109375,
      "logps/chosen": -0.6988281011581421,
      "logps/rejected": -0.9232422113418579,
      "loss": 0.9245,
      "nll_loss": 0.886914074420929,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.0699462890625,
      "rewards/margins": 0.022371673956513405,
      "rewards/rejected": -0.09227295219898224,
      "step": 9400
    },
    {
      "epoch": 0.35262595791722096,
      "grad_norm": 1.858516697138284,
      "learning_rate": 8.246984442271813e-07,
      "log_odds_chosen": 0.41352540254592896,
      "log_odds_ratio": -0.6122070550918579,
      "logits/chosen": -0.8128906488418579,
      "logits/rejected": -0.7152343988418579,
      "logps/chosen": -0.719531238079071,
      "logps/rejected": -0.9609375,
      "loss": 0.9503,
      "nll_loss": 0.8941406011581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07199706882238388,
      "rewards/margins": 0.02409820631146431,
      "rewards/rejected": -0.0960693359375,
      "step": 9410
    },
    {
      "epoch": 0.35300069326038486,
      "grad_norm": 1.5408592663575436,
      "learning_rate": 8.242605899652435e-07,
      "log_odds_chosen": 0.5853271484375,
      "log_odds_ratio": -0.5708984136581421,
      "logits/chosen": -0.7861328125,
      "logits/rejected": -0.6792968511581421,
      "logps/chosen": -0.690625011920929,
      "logps/rejected": -1.0695312023162842,
      "loss": 0.947,
      "nll_loss": 0.894726574420929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06900634616613388,
      "rewards/margins": 0.03795471042394638,
      "rewards/rejected": -0.10703124850988388,
      "step": 9420
    },
    {
      "epoch": 0.35337542860354876,
      "grad_norm": 1.7162056570454893,
      "learning_rate": 8.238234323688798e-07,
      "log_odds_chosen": 0.6486572027206421,
      "log_odds_ratio": -0.56787109375,
      "logits/chosen": -0.774218738079071,
      "logits/rejected": -0.6332031488418579,
      "logps/chosen": -0.682812511920929,
      "logps/rejected": -1.112890601158142,
      "loss": 0.9318,
      "nll_loss": 0.849609375,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06822510063648224,
      "rewards/margins": 0.04309539869427681,
      "rewards/rejected": -0.11136474460363388,
      "step": 9430
    },
    {
      "epoch": 0.35375016394671266,
      "grad_norm": 1.6490534637109828,
      "learning_rate": 8.233869695926182e-07,
      "log_odds_chosen": 0.4116455018520355,
      "log_odds_ratio": -0.6290038824081421,
      "logits/chosen": -0.754101574420929,
      "logits/rejected": -0.682421863079071,
      "logps/chosen": -0.666015625,
      "logps/rejected": -0.933789074420929,
      "loss": 0.9509,
      "nll_loss": 0.8955078125,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0665283203125,
      "rewards/margins": 0.02675476111471653,
      "rewards/rejected": -0.09326171875,
      "step": 9440
    },
    {
      "epoch": 0.3541248992898765,
      "grad_norm": 1.689754823601209,
      "learning_rate": 8.229511997978235e-07,
      "log_odds_chosen": 0.426025390625,
      "log_odds_ratio": -0.59912109375,
      "logits/chosen": -0.788281261920929,
      "logits/rejected": -0.6585937738418579,
      "logps/chosen": -0.6644531488418579,
      "logps/rejected": -0.920703113079071,
      "loss": 0.9323,
      "nll_loss": 0.8531249761581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06651611626148224,
      "rewards/margins": 0.025570297613739967,
      "rewards/rejected": -0.09208984673023224,
      "step": 9450
    },
    {
      "epoch": 0.3544996346330404,
      "grad_norm": 1.6520618614333558,
      "learning_rate": 8.22516121152665e-07,
      "log_odds_chosen": 0.44392091035842896,
      "log_odds_ratio": -0.6270507574081421,
      "logits/chosen": -0.7337890863418579,
      "logits/rejected": -0.6246093511581421,
      "logps/chosen": -0.6890624761581421,
      "logps/rejected": -0.9970703125,
      "loss": 0.944,
      "nll_loss": 0.8988281488418579,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.06889648735523224,
      "rewards/margins": 0.03076324425637722,
      "rewards/rejected": -0.09970702975988388,
      "step": 9460
    },
    {
      "epoch": 0.3548743699762043,
      "grad_norm": 1.68421834163081,
      "learning_rate": 8.220817318320836e-07,
      "log_odds_chosen": 0.4399169981479645,
      "log_odds_ratio": -0.6114257574081421,
      "logits/chosen": -0.768359363079071,
      "logits/rejected": -0.654589831829071,
      "logps/chosen": -0.676953136920929,
      "logps/rejected": -0.950976550579071,
      "loss": 0.9333,
      "nll_loss": 0.857617199420929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06768798828125,
      "rewards/margins": 0.027382660657167435,
      "rewards/rejected": -0.0950927734375,
      "step": 9470
    },
    {
      "epoch": 0.3552491053193682,
      "grad_norm": 1.7776771781157168,
      "learning_rate": 8.216480300177611e-07,
      "log_odds_chosen": 0.48634034395217896,
      "log_odds_ratio": -0.5972656011581421,
      "logits/chosen": -0.7173827886581421,
      "logits/rejected": -0.647265613079071,
      "logps/chosen": -0.655078113079071,
      "logps/rejected": -0.9654296636581421,
      "loss": 0.9564,
      "nll_loss": 0.9105468988418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06547851860523224,
      "rewards/margins": 0.031043242663145065,
      "rewards/rejected": -0.09650878608226776,
      "step": 9480
    },
    {
      "epoch": 0.3556238406625321,
      "grad_norm": 1.554108319406554,
      "learning_rate": 8.212150138980857e-07,
      "log_odds_chosen": 0.46037596464157104,
      "log_odds_ratio": -0.6026366949081421,
      "logits/chosen": -0.7378906011581421,
      "logits/rejected": -0.6859375238418579,
      "logps/chosen": -0.67822265625,
      "logps/rejected": -0.977343738079071,
      "loss": 0.9104,
      "nll_loss": 0.8187500238418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06782226264476776,
      "rewards/margins": 0.02995910681784153,
      "rewards/rejected": -0.09768066555261612,
      "step": 9490
    },
    {
      "epoch": 0.35599857600569595,
      "grad_norm": 1.5555859435533315,
      "learning_rate": 8.207826816681233e-07,
      "log_odds_chosen": 0.439697265625,
      "log_odds_ratio": -0.6343749761581421,
      "logits/chosen": -0.6944335699081421,
      "logits/rejected": -0.6514648199081421,
      "logps/chosen": -0.6781250238418579,
      "logps/rejected": -0.962695300579071,
      "loss": 0.9121,
      "nll_loss": 0.857421875,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06778564304113388,
      "rewards/margins": 0.02849426306784153,
      "rewards/rejected": -0.09628906100988388,
      "step": 9500
    },
    {
      "epoch": 0.35637331134885986,
      "grad_norm": 1.5310662625011984,
      "learning_rate": 8.203510315295829e-07,
      "log_odds_chosen": 0.39586180448532104,
      "log_odds_ratio": -0.6244140863418579,
      "logits/chosen": -0.7816406488418579,
      "logits/rejected": -0.6917968988418579,
      "logps/chosen": -0.7298828363418579,
      "logps/rejected": -1.010156273841858,
      "loss": 0.9199,
      "nll_loss": 0.907031238079071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.072998046875,
      "rewards/margins": 0.02801361121237278,
      "rewards/rejected": -0.10100097954273224,
      "step": 9510
    },
    {
      "epoch": 0.35674804669202376,
      "grad_norm": 1.7018918058240118,
      "learning_rate": 8.199200616907878e-07,
      "log_odds_chosen": 0.546875,
      "log_odds_ratio": -0.561328113079071,
      "logits/chosen": -0.7749999761581421,
      "logits/rejected": -0.6856445074081421,
      "logps/chosen": -0.6988281011581421,
      "logps/rejected": -1.0431640148162842,
      "loss": 0.9424,
      "nll_loss": 0.9068359136581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06987304985523224,
      "rewards/margins": 0.03449249267578125,
      "rewards/rejected": -0.1043701171875,
      "step": 9520
    },
    {
      "epoch": 0.35712278203518766,
      "grad_norm": 1.7043498047827292,
      "learning_rate": 8.194897703666421e-07,
      "log_odds_chosen": 0.37883299589157104,
      "log_odds_ratio": -0.6358398199081421,
      "logits/chosen": -0.7701171636581421,
      "logits/rejected": -0.706250011920929,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -0.9544922113418579,
      "loss": 0.9157,
      "nll_loss": 0.8382812738418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07062987983226776,
      "rewards/margins": 0.02479705773293972,
      "rewards/rejected": -0.095458984375,
      "step": 9530
    },
    {
      "epoch": 0.35749751737835156,
      "grad_norm": 1.583708109638968,
      "learning_rate": 8.190601557786015e-07,
      "log_odds_chosen": 0.6196960210800171,
      "log_odds_ratio": -0.5575195550918579,
      "logits/chosen": -0.7725585699081421,
      "logits/rejected": -0.6458984613418579,
      "logps/chosen": -0.7210937738418579,
      "logps/rejected": -1.130273461341858,
      "loss": 0.9364,
      "nll_loss": 0.9244140386581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07204589992761612,
      "rewards/margins": 0.040924072265625,
      "rewards/rejected": -0.11298827826976776,
      "step": 9540
    },
    {
      "epoch": 0.3578722527215154,
      "grad_norm": 1.6106218148156102,
      "learning_rate": 8.186312161546413e-07,
      "log_odds_chosen": 0.49089354276657104,
      "log_odds_ratio": -0.611621081829071,
      "logits/chosen": -0.7574218511581421,
      "logits/rejected": -0.675097644329071,
      "logps/chosen": -0.7564452886581421,
      "logps/rejected": -1.0681641101837158,
      "loss": 0.9504,
      "nll_loss": 0.894335925579071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07563476264476776,
      "rewards/margins": 0.03118591383099556,
      "rewards/rejected": -0.10688476264476776,
      "step": 9550
    },
    {
      "epoch": 0.3582469880646793,
      "grad_norm": 1.6404204662210946,
      "learning_rate": 8.182029497292262e-07,
      "log_odds_chosen": 0.538525402545929,
      "log_odds_ratio": -0.559374988079071,
      "logits/chosen": -0.720019519329071,
      "logits/rejected": -0.636425793170929,
      "logps/chosen": -0.6612304449081421,
      "logps/rejected": -0.9664062261581421,
      "loss": 0.9449,
      "nll_loss": 0.8359375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06617431342601776,
      "rewards/margins": 0.03043823316693306,
      "rewards/rejected": -0.0965576171875,
      "step": 9560
    },
    {
      "epoch": 0.3586217234078432,
      "grad_norm": 1.7271141882453847,
      "learning_rate": 8.177753547432792e-07,
      "log_odds_chosen": 0.513476550579071,
      "log_odds_ratio": -0.5873047113418579,
      "logits/chosen": -0.7466796636581421,
      "logits/rejected": -0.6312500238418579,
      "logps/chosen": -0.6650390625,
      "logps/rejected": -0.979687511920929,
      "loss": 0.9227,
      "nll_loss": 0.874804675579071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06651611626148224,
      "rewards/margins": 0.03146057203412056,
      "rewards/rejected": -0.09794922173023224,
      "step": 9570
    },
    {
      "epoch": 0.3589964587510071,
      "grad_norm": 1.5073580472649968,
      "learning_rate": 8.173484294441524e-07,
      "log_odds_chosen": 0.4779052734375,
      "log_odds_ratio": -0.5982421636581421,
      "logits/chosen": -0.786914050579071,
      "logits/rejected": -0.651171863079071,
      "logps/chosen": -0.7359374761581421,
      "logps/rejected": -1.040429711341858,
      "loss": 0.94,
      "nll_loss": 0.877148449420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07362060248851776,
      "rewards/margins": 0.03046264685690403,
      "rewards/rejected": -0.10407714545726776,
      "step": 9580
    },
    {
      "epoch": 0.359371194094171,
      "grad_norm": 1.754438446970491,
      "learning_rate": 8.169221720855952e-07,
      "log_odds_chosen": 0.4736328125,
      "log_odds_ratio": -0.602734386920929,
      "logits/chosen": -0.737500011920929,
      "logits/rejected": -0.629687488079071,
      "logps/chosen": -0.69140625,
      "logps/rejected": -1.010156273841858,
      "loss": 0.9384,
      "nll_loss": 0.8763672113418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06906738132238388,
      "rewards/margins": 0.03186798095703125,
      "rewards/rejected": -0.10104980319738388,
      "step": 9590
    },
    {
      "epoch": 0.35974592943733485,
      "grad_norm": 1.6418058415467864,
      "learning_rate": 8.164965809277261e-07,
      "log_odds_chosen": 0.644213855266571,
      "log_odds_ratio": -0.539355456829071,
      "logits/chosen": -0.759570300579071,
      "logits/rejected": -0.66943359375,
      "logps/chosen": -0.70166015625,
      "logps/rejected": -1.100195288658142,
      "loss": 0.9284,
      "nll_loss": 0.8412109613418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.070068359375,
      "rewards/margins": 0.03991241380572319,
      "rewards/rejected": -0.11003418266773224,
      "step": 9600
    },
    {
      "epoch": 0.36012066478049876,
      "grad_norm": 1.7003617335292316,
      "learning_rate": 8.160716542370011e-07,
      "log_odds_chosen": 0.608776867389679,
      "log_odds_ratio": -0.574414074420929,
      "logits/chosen": -0.76953125,
      "logits/rejected": -0.6796875,
      "logps/chosen": -0.65576171875,
      "logps/rejected": -1.057226538658142,
      "loss": 0.937,
      "nll_loss": 0.8587890863418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06558837741613388,
      "rewards/margins": 0.04023895412683487,
      "rewards/rejected": -0.10579834133386612,
      "step": 9610
    },
    {
      "epoch": 0.36049540012366266,
      "grad_norm": 1.608577123256023,
      "learning_rate": 8.156473902861856e-07,
      "log_odds_chosen": 0.47984617948532104,
      "log_odds_ratio": -0.605761706829071,
      "logits/chosen": -0.7120116949081421,
      "logits/rejected": -0.6161133050918579,
      "logps/chosen": -0.6690429449081421,
      "logps/rejected": -0.979687511920929,
      "loss": 0.9313,
      "nll_loss": 0.8736327886581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06690673530101776,
      "rewards/margins": 0.031082917004823685,
      "rewards/rejected": -0.0980224609375,
      "step": 9620
    },
    {
      "epoch": 0.36087013546682656,
      "grad_norm": 1.779439766982219,
      "learning_rate": 8.152237873543241e-07,
      "log_odds_chosen": 0.36041259765625,
      "log_odds_ratio": -0.6419922113418579,
      "logits/chosen": -0.806445300579071,
      "logits/rejected": -0.70947265625,
      "logps/chosen": -0.719921886920929,
      "logps/rejected": -0.956250011920929,
      "loss": 0.9191,
      "nll_loss": 0.869921863079071,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07197265326976776,
      "rewards/margins": 0.02368011511862278,
      "rewards/rejected": -0.09565429389476776,
      "step": 9630
    },
    {
      "epoch": 0.36124487080999046,
      "grad_norm": 1.5437113977447214,
      "learning_rate": 8.148008437267104e-07,
      "log_odds_chosen": 0.36015623807907104,
      "log_odds_ratio": -0.652539074420929,
      "logits/chosen": -0.7183593511581421,
      "logits/rejected": -0.661328136920929,
      "logps/chosen": -0.679882824420929,
      "logps/rejected": -0.909960925579071,
      "loss": 0.9605,
      "nll_loss": 0.9296875,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06798096001148224,
      "rewards/margins": 0.02305297926068306,
      "rewards/rejected": -0.091064453125,
      "step": 9640
    },
    {
      "epoch": 0.36161960615315436,
      "grad_norm": 1.72145237057175,
      "learning_rate": 8.143785576948602e-07,
      "log_odds_chosen": 0.39219969511032104,
      "log_odds_ratio": -0.6170898675918579,
      "logits/chosen": -0.73828125,
      "logits/rejected": -0.671679675579071,
      "logps/chosen": -0.6666015386581421,
      "logps/rejected": -0.9097656011581421,
      "loss": 0.8984,
      "nll_loss": 0.8314453363418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06661377102136612,
      "rewards/margins": 0.02435455285012722,
      "rewards/rejected": -0.09096679836511612,
      "step": 9650
    },
    {
      "epoch": 0.3619943414963182,
      "grad_norm": 2.1346521063799386,
      "learning_rate": 8.139569275564796e-07,
      "log_odds_chosen": 0.530322253704071,
      "log_odds_ratio": -0.5809570550918579,
      "logits/chosen": -0.77734375,
      "logits/rejected": -0.667773425579071,
      "logps/chosen": -0.7328125238418579,
      "logps/rejected": -1.0871093273162842,
      "loss": 0.9187,
      "nll_loss": 0.886914074420929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07333984225988388,
      "rewards/margins": 0.03541870042681694,
      "rewards/rejected": -0.10866699367761612,
      "step": 9660
    },
    {
      "epoch": 0.3623690768394821,
      "grad_norm": 1.590343737169822,
      "learning_rate": 8.135359516154388e-07,
      "log_odds_chosen": 0.35893553495407104,
      "log_odds_ratio": -0.643750011920929,
      "logits/chosen": -0.7396484613418579,
      "logits/rejected": -0.6968749761581421,
      "logps/chosen": -0.703125,
      "logps/rejected": -0.9208984375,
      "loss": 0.9282,
      "nll_loss": 0.889843761920929,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.0703125,
      "rewards/margins": 0.02180786058306694,
      "rewards/rejected": -0.09211425483226776,
      "step": 9670
    },
    {
      "epoch": 0.362743812182646,
      "grad_norm": 1.7896406779001865,
      "learning_rate": 8.131156281817418e-07,
      "log_odds_chosen": 0.46588134765625,
      "log_odds_ratio": -0.584179699420929,
      "logits/chosen": -0.7783203125,
      "logits/rejected": -0.6942383050918579,
      "logps/chosen": -0.677050769329071,
      "logps/rejected": -0.97265625,
      "loss": 0.9197,
      "nll_loss": 0.8359375,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06770019233226776,
      "rewards/margins": 0.02960815466940403,
      "rewards/rejected": -0.09733887016773224,
      "step": 9680
    },
    {
      "epoch": 0.3631185475258099,
      "grad_norm": 2.0522174280676895,
      "learning_rate": 8.126959555714979e-07,
      "log_odds_chosen": 0.508557140827179,
      "log_odds_ratio": -0.601367175579071,
      "logits/chosen": -0.80859375,
      "logits/rejected": -0.693066418170929,
      "logps/chosen": -0.706738293170929,
      "logps/rejected": -1.062109351158142,
      "loss": 0.9329,
      "nll_loss": 0.890429675579071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07064209133386612,
      "rewards/margins": 0.035506438463926315,
      "rewards/rejected": -0.106201171875,
      "step": 9690
    },
    {
      "epoch": 0.3634932828689738,
      "grad_norm": 1.7000846756443309,
      "learning_rate": 8.122769321068952e-07,
      "log_odds_chosen": 0.33879393339157104,
      "log_odds_ratio": -0.639355480670929,
      "logits/chosen": -0.7535156011581421,
      "logits/rejected": -0.6947265863418579,
      "logps/chosen": -0.7113281488418579,
      "logps/rejected": -0.9273437261581421,
      "loss": 0.9525,
      "nll_loss": 0.9322265386581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07106933742761612,
      "rewards/margins": 0.02160339429974556,
      "rewards/rejected": -0.09283447265625,
      "step": 9700
    },
    {
      "epoch": 0.36386801821213766,
      "grad_norm": 1.5944393034799411,
      "learning_rate": 8.118585561161698e-07,
      "log_odds_chosen": 0.6305907964706421,
      "log_odds_ratio": -0.54638671875,
      "logits/chosen": -0.776562511920929,
      "logits/rejected": -0.678027331829071,
      "logps/chosen": -0.6543945074081421,
      "logps/rejected": -1.053125023841858,
      "loss": 0.9309,
      "nll_loss": 0.8431640863418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06536865234375,
      "rewards/margins": 0.03986816480755806,
      "rewards/rejected": -0.10529784858226776,
      "step": 9710
    },
    {
      "epoch": 0.36424275355530156,
      "grad_norm": 1.6935412270710641,
      "learning_rate": 8.114408259335793e-07,
      "log_odds_chosen": 0.3953491151332855,
      "log_odds_ratio": -0.619433581829071,
      "logits/chosen": -0.772656261920929,
      "logits/rejected": -0.6900390386581421,
      "logps/chosen": -0.7206054925918579,
      "logps/rejected": -0.984375,
      "loss": 0.9753,
      "nll_loss": 0.922656238079071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07199706882238388,
      "rewards/margins": 0.02633514441549778,
      "rewards/rejected": -0.09840087592601776,
      "step": 9720
    },
    {
      "epoch": 0.36461748889846546,
      "grad_norm": 1.8323115514730597,
      "learning_rate": 8.110237398993754e-07,
      "log_odds_chosen": 0.4774108827114105,
      "log_odds_ratio": -0.58642578125,
      "logits/chosen": -0.79296875,
      "logits/rejected": -0.695507824420929,
      "logps/chosen": -0.701171875,
      "logps/rejected": -1.0041015148162842,
      "loss": 0.9034,
      "nll_loss": 0.8128906488418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07014159858226776,
      "rewards/margins": 0.03022003173828125,
      "rewards/rejected": -0.10039062798023224,
      "step": 9730
    },
    {
      "epoch": 0.36499222424162936,
      "grad_norm": 1.6149217576854975,
      "learning_rate": 8.106072963597751e-07,
      "log_odds_chosen": 0.4264892637729645,
      "log_odds_ratio": -0.631640613079071,
      "logits/chosen": -0.764843761920929,
      "logits/rejected": -0.6953125,
      "logps/chosen": -0.6958984136581421,
      "logps/rejected": -0.9916015863418579,
      "loss": 0.9546,
      "nll_loss": 0.897656261920929,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06964111328125,
      "rewards/margins": 0.029500579461455345,
      "rewards/rejected": -0.09902343899011612,
      "step": 9740
    },
    {
      "epoch": 0.36536695958479326,
      "grad_norm": 1.7668985466011757,
      "learning_rate": 8.101914936669332e-07,
      "log_odds_chosen": 0.4493774473667145,
      "log_odds_ratio": -0.6158202886581421,
      "logits/chosen": -0.745898425579071,
      "logits/rejected": -0.634570300579071,
      "logps/chosen": -0.679882824420929,
      "logps/rejected": -0.9820312261581421,
      "loss": 0.9402,
      "nll_loss": 0.9076172113418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06798096001148224,
      "rewards/margins": 0.030243683606386185,
      "rewards/rejected": -0.09816894680261612,
      "step": 9750
    },
    {
      "epoch": 0.3657416949279571,
      "grad_norm": 1.6710924608920572,
      "learning_rate": 8.09776330178916e-07,
      "log_odds_chosen": 0.37342530488967896,
      "log_odds_ratio": -0.658203125,
      "logits/chosen": -0.776171863079071,
      "logits/rejected": -0.711718738079071,
      "logps/chosen": -0.674609363079071,
      "logps/rejected": -0.908007800579071,
      "loss": 0.9342,
      "nll_loss": 0.8031250238418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06748046725988388,
      "rewards/margins": 0.02331390418112278,
      "rewards/rejected": -0.09086914360523224,
      "step": 9760
    },
    {
      "epoch": 0.366116430271121,
      "grad_norm": 1.6147556286720857,
      "learning_rate": 8.093618042596727e-07,
      "log_odds_chosen": 0.348876953125,
      "log_odds_ratio": -0.6742187738418579,
      "logits/chosen": -0.775585949420929,
      "logits/rejected": -0.716015636920929,
      "logps/chosen": -0.723828136920929,
      "logps/rejected": -0.958789050579071,
      "loss": 0.9452,
      "nll_loss": 0.9234374761581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07227782905101776,
      "rewards/margins": 0.023525238037109375,
      "rewards/rejected": -0.09577636420726776,
      "step": 9770
    },
    {
      "epoch": 0.3664911656142849,
      "grad_norm": 1.65628816701633,
      "learning_rate": 8.089479142790095e-07,
      "log_odds_chosen": 0.4847168028354645,
      "log_odds_ratio": -0.589160144329071,
      "logits/chosen": -0.770703136920929,
      "logits/rejected": -0.7001953125,
      "logps/chosen": -0.672558605670929,
      "logps/rejected": -0.9730468988418579,
      "loss": 0.9351,
      "nll_loss": 0.888476550579071,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.0672607421875,
      "rewards/margins": 0.03005218505859375,
      "rewards/rejected": -0.09721679985523224,
      "step": 9780
    },
    {
      "epoch": 0.3668659009574488,
      "grad_norm": 1.5848808132113166,
      "learning_rate": 8.085346586125621e-07,
      "log_odds_chosen": 0.5480102300643921,
      "log_odds_ratio": -0.587695300579071,
      "logits/chosen": -0.7289062738418579,
      "logits/rejected": -0.6166015863418579,
      "logps/chosen": -0.6708008050918579,
      "logps/rejected": -1.018945336341858,
      "loss": 0.9456,
      "nll_loss": 0.8794921636581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.067138671875,
      "rewards/margins": 0.03486480563879013,
      "rewards/rejected": -0.1019287109375,
      "step": 9790
    },
    {
      "epoch": 0.3672406363006127,
      "grad_norm": 1.6503972035090961,
      "learning_rate": 8.081220356417685e-07,
      "log_odds_chosen": 0.3486328125,
      "log_odds_ratio": -0.6504882574081421,
      "logits/chosen": -0.7978515625,
      "logits/rejected": -0.7265625,
      "logps/chosen": -0.7783203125,
      "logps/rejected": -0.9966796636581421,
      "loss": 0.9408,
      "nll_loss": 0.889453113079071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07784423977136612,
      "rewards/margins": 0.02183685265481472,
      "rewards/rejected": -0.09968261420726776,
      "step": 9800
    },
    {
      "epoch": 0.36761537164377656,
      "grad_norm": 1.4561139176466145,
      "learning_rate": 8.077100437538435e-07,
      "log_odds_chosen": 0.518847644329071,
      "log_odds_ratio": -0.6014648675918579,
      "logits/chosen": -0.767382800579071,
      "logits/rejected": -0.69140625,
      "logps/chosen": -0.7275390625,
      "logps/rejected": -1.062890648841858,
      "loss": 0.9143,
      "nll_loss": 0.888476550579071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07275390625,
      "rewards/margins": 0.03353576734662056,
      "rewards/rejected": -0.1063232421875,
      "step": 9810
    },
    {
      "epoch": 0.36799010698694046,
      "grad_norm": 1.694258134680671,
      "learning_rate": 8.072986813417512e-07,
      "log_odds_chosen": 0.46556395292282104,
      "log_odds_ratio": -0.6094726324081421,
      "logits/chosen": -0.7083984613418579,
      "logits/rejected": -0.6246093511581421,
      "logps/chosen": -0.685253918170929,
      "logps/rejected": -1.0001952648162842,
      "loss": 0.9407,
      "nll_loss": 0.852246105670929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0684814453125,
      "rewards/margins": 0.03154754638671875,
      "rewards/rejected": -0.10007324069738388,
      "step": 9820
    },
    {
      "epoch": 0.36836484233010436,
      "grad_norm": 4.021830010283598,
      "learning_rate": 8.068879468041791e-07,
      "log_odds_chosen": 0.3878417909145355,
      "log_odds_ratio": -0.631640613079071,
      "logits/chosen": -0.7962890863418579,
      "logits/rejected": -0.68212890625,
      "logps/chosen": -0.6917968988418579,
      "logps/rejected": -0.9410156011581421,
      "loss": 0.9464,
      "nll_loss": 0.892578125,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06916503608226776,
      "rewards/margins": 0.02482910081744194,
      "rewards/rejected": -0.09401855617761612,
      "step": 9830
    },
    {
      "epoch": 0.36873957767326826,
      "grad_norm": 1.6235296393222527,
      "learning_rate": 8.064778385455118e-07,
      "log_odds_chosen": 0.572558581829071,
      "log_odds_ratio": -0.5826171636581421,
      "logits/chosen": -0.730273425579071,
      "logits/rejected": -0.6455078125,
      "logps/chosen": -0.740429699420929,
      "logps/rejected": -1.116796851158142,
      "loss": 0.9514,
      "nll_loss": 0.861328125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07401122897863388,
      "rewards/margins": 0.03766326978802681,
      "rewards/rejected": -0.1116943359375,
      "step": 9840
    },
    {
      "epoch": 0.36911431301643216,
      "grad_norm": 1.523456413056534,
      "learning_rate": 8.060683549758054e-07,
      "log_odds_chosen": 0.5011230707168579,
      "log_odds_ratio": -0.5731445550918579,
      "logits/chosen": -0.736132800579071,
      "logits/rejected": -0.6888672113418579,
      "logps/chosen": -0.6646484136581421,
      "logps/rejected": -0.9794921875,
      "loss": 0.9173,
      "nll_loss": 0.844921886920929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06649170070886612,
      "rewards/margins": 0.03151397779583931,
      "rewards/rejected": -0.09794922173023224,
      "step": 9850
    },
    {
      "epoch": 0.36948904835959606,
      "grad_norm": 1.7395056000449856,
      "learning_rate": 8.056594945107608e-07,
      "log_odds_chosen": 0.4041503965854645,
      "log_odds_ratio": -0.635937511920929,
      "logits/chosen": -0.727343738079071,
      "logits/rejected": -0.6499999761581421,
      "logps/chosen": -0.7060546875,
      "logps/rejected": -0.962890625,
      "loss": 0.9536,
      "nll_loss": 0.899609386920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07061767578125,
      "rewards/margins": 0.025699615478515625,
      "rewards/rejected": -0.09628906100988388,
      "step": 9860
    },
    {
      "epoch": 0.3698637837027599,
      "grad_norm": 1.7038304383624117,
      "learning_rate": 8.052512555716987e-07,
      "log_odds_chosen": 0.3252929747104645,
      "log_odds_ratio": -0.6572265625,
      "logits/chosen": -0.7464843988418579,
      "logits/rejected": -0.6465820074081421,
      "logps/chosen": -0.762499988079071,
      "logps/rejected": -0.99609375,
      "loss": 0.9314,
      "nll_loss": 0.9248046875,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07631836086511612,
      "rewards/margins": 0.0233154296875,
      "rewards/rejected": -0.09958495944738388,
      "step": 9870
    },
    {
      "epoch": 0.3702385190459238,
      "grad_norm": 1.789203364795543,
      "learning_rate": 8.048436365855337e-07,
      "log_odds_chosen": 0.5505126714706421,
      "log_odds_ratio": -0.603515625,
      "logits/chosen": -0.8046875,
      "logits/rejected": -0.667773425579071,
      "logps/chosen": -0.713183581829071,
      "logps/rejected": -1.060937523841858,
      "loss": 0.929,
      "nll_loss": 0.8841797113418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07138671725988388,
      "rewards/margins": 0.03475341945886612,
      "rewards/rejected": -0.10610351711511612,
      "step": 9880
    },
    {
      "epoch": 0.3706132543890877,
      "grad_norm": 1.6077082511462588,
      "learning_rate": 8.044366359847486e-07,
      "log_odds_chosen": 0.552197277545929,
      "log_odds_ratio": -0.573046863079071,
      "logits/chosen": -0.7435547113418579,
      "logits/rejected": -0.6634765863418579,
      "logps/chosen": -0.662304699420929,
      "logps/rejected": -1.0087890625,
      "loss": 0.9308,
      "nll_loss": 0.9154297113418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06623534858226776,
      "rewards/margins": 0.03466186672449112,
      "rewards/rejected": -0.10087890923023224,
      "step": 9890
    },
    {
      "epoch": 0.3709879897322516,
      "grad_norm": 1.6988734508126893,
      "learning_rate": 8.040302522073696e-07,
      "log_odds_chosen": 0.30207520723342896,
      "log_odds_ratio": -0.661914050579071,
      "logits/chosen": -0.725390613079071,
      "logits/rejected": -0.656445324420929,
      "logps/chosen": -0.753125011920929,
      "logps/rejected": -0.9654296636581421,
      "loss": 0.9503,
      "nll_loss": 0.8876953125,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.07532958686351776,
      "rewards/margins": 0.021141815930604935,
      "rewards/rejected": -0.096435546875,
      "step": 9900
    },
    {
      "epoch": 0.3713627250754155,
      "grad_norm": 1.8114125504909735,
      "learning_rate": 8.036244836969407e-07,
      "log_odds_chosen": 0.4241088926792145,
      "log_odds_ratio": -0.609082043170929,
      "logits/chosen": -0.758105456829071,
      "logits/rejected": -0.641406238079071,
      "logps/chosen": -0.671679675579071,
      "logps/rejected": -0.955078125,
      "loss": 0.9234,
      "nll_loss": 0.9403320550918579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.067138671875,
      "rewards/margins": 0.02834930457174778,
      "rewards/rejected": -0.09552001953125,
      "step": 9910
    },
    {
      "epoch": 0.37173746041857936,
      "grad_norm": 1.9467008646797919,
      "learning_rate": 8.032193289024989e-07,
      "log_odds_chosen": 0.513232409954071,
      "log_odds_ratio": -0.5835937261581421,
      "logits/chosen": -0.810546875,
      "logits/rejected": -0.707226574420929,
      "logps/chosen": -0.727734386920929,
      "logps/rejected": -1.048242211341858,
      "loss": 0.9499,
      "nll_loss": 0.862109363079071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07279052585363388,
      "rewards/margins": 0.03201904147863388,
      "rewards/rejected": -0.1048583984375,
      "step": 9920
    },
    {
      "epoch": 0.37211219576174326,
      "grad_norm": 1.5251969999753845,
      "learning_rate": 8.02814786278549e-07,
      "log_odds_chosen": 0.5441650152206421,
      "log_odds_ratio": -0.5669921636581421,
      "logits/chosen": -0.7593749761581421,
      "logits/rejected": -0.651171863079071,
      "logps/chosen": -0.6690429449081421,
      "logps/rejected": -0.999804675579071,
      "loss": 0.9317,
      "nll_loss": 0.941210925579071,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06690673530101776,
      "rewards/margins": 0.033129118382930756,
      "rewards/rejected": -0.09995117038488388,
      "step": 9930
    },
    {
      "epoch": 0.37248693110490716,
      "grad_norm": 1.7172045880351583,
      "learning_rate": 8.024108542850394e-07,
      "log_odds_chosen": 0.4966674745082855,
      "log_odds_ratio": -0.6034179925918579,
      "logits/chosen": -0.738085925579071,
      "logits/rejected": -0.646484375,
      "logps/chosen": -0.7095702886581421,
      "logps/rejected": -1.0275390148162842,
      "loss": 0.9397,
      "nll_loss": 0.8853515386581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07098388671875,
      "rewards/margins": 0.03172454982995987,
      "rewards/rejected": -0.10269775241613388,
      "step": 9940
    },
    {
      "epoch": 0.37286166644807106,
      "grad_norm": 1.8601598786586284,
      "learning_rate": 8.020075313873367e-07,
      "log_odds_chosen": 0.2657714784145355,
      "log_odds_ratio": -0.693164050579071,
      "logits/chosen": -0.744140625,
      "logits/rejected": -0.6864258050918579,
      "logps/chosen": -0.753710925579071,
      "logps/rejected": -0.965039074420929,
      "loss": 0.9146,
      "nll_loss": 0.8726562261581421,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.07534179836511612,
      "rewards/margins": 0.021181488409638405,
      "rewards/rejected": -0.09650878608226776,
      "step": 9950
    },
    {
      "epoch": 0.37323640179123496,
      "grad_norm": 1.7019324578231811,
      "learning_rate": 8.016048160562023e-07,
      "log_odds_chosen": 0.46855467557907104,
      "log_odds_ratio": -0.5874999761581421,
      "logits/chosen": -0.7587890625,
      "logits/rejected": -0.664843738079071,
      "logps/chosen": -0.7085937261581421,
      "logps/rejected": -1.0128905773162842,
      "loss": 0.9447,
      "nll_loss": 0.898632824420929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07084961235523224,
      "rewards/margins": 0.030538177117705345,
      "rewards/rejected": -0.10135497897863388,
      "step": 9960
    },
    {
      "epoch": 0.3736111371343988,
      "grad_norm": 1.9425841530095163,
      "learning_rate": 8.012027067677667e-07,
      "log_odds_chosen": 0.27625733613967896,
      "log_odds_ratio": -0.689160168170929,
      "logits/chosen": -0.767773449420929,
      "logits/rejected": -0.66650390625,
      "logps/chosen": -0.7025390863418579,
      "logps/rejected": -0.91796875,
      "loss": 0.9195,
      "nll_loss": 0.853515625,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07023926079273224,
      "rewards/margins": 0.021508026868104935,
      "rewards/rejected": -0.09182129055261612,
      "step": 9970
    },
    {
      "epoch": 0.3739858724775627,
      "grad_norm": 1.492472788763972,
      "learning_rate": 8.008012020035062e-07,
      "log_odds_chosen": 0.3958496153354645,
      "log_odds_ratio": -0.6361328363418579,
      "logits/chosen": -0.762499988079071,
      "logits/rejected": -0.6595703363418579,
      "logps/chosen": -0.779589831829071,
      "logps/rejected": -1.0320312976837158,
      "loss": 0.9066,
      "nll_loss": 0.902539074420929,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.0780029296875,
      "rewards/margins": 0.025176238268613815,
      "rewards/rejected": -0.10324706882238388,
      "step": 9980
    },
    {
      "epoch": 0.3743606078207266,
      "grad_norm": 1.760944655827164,
      "learning_rate": 8.004003002502188e-07,
      "log_odds_chosen": 0.556896984577179,
      "log_odds_ratio": -0.573046863079071,
      "logits/chosen": -0.7865234613418579,
      "logits/rejected": -0.6996093988418579,
      "logps/chosen": -0.691210925579071,
      "logps/rejected": -1.0099608898162842,
      "loss": 0.9374,
      "nll_loss": 0.887499988079071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06912841647863388,
      "rewards/margins": 0.031839750707149506,
      "rewards/rejected": -0.1009521484375,
      "step": 9990
    },
    {
      "epoch": 0.3747353431638905,
      "grad_norm": 1.7144642070868015,
      "learning_rate": 8e-07,
      "log_odds_chosen": 0.4030395448207855,
      "log_odds_ratio": -0.61572265625,
      "logits/chosen": -0.730664074420929,
      "logits/rejected": -0.6756836175918579,
      "logps/chosen": -0.6851562261581421,
      "logps/rejected": -0.9398437738418579,
      "loss": 0.9136,
      "nll_loss": 0.8921874761581421,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06853027641773224,
      "rewards/margins": 0.02547149732708931,
      "rewards/rejected": -0.09398193657398224,
      "step": 10000
    },
    {
      "epoch": 0.3747353431638905,
      "eval_log_odds_chosen": 0.4230777621269226,
      "eval_log_odds_ratio": -0.6252666711807251,
      "eval_logits/chosen": -0.735567569732666,
      "eval_logits/rejected": -0.6452904939651489,
      "eval_logps/chosen": -0.7526504993438721,
      "eval_logps/rejected": -1.0431419610977173,
      "eval_loss": 1.0906352996826172,
      "eval_nll_loss": 1.0300488471984863,
      "eval_rewards/accuracies": 0.5944454073905945,
      "eval_rewards/chosen": -0.07526461780071259,
      "eval_rewards/margins": 0.02905045822262764,
      "eval_rewards/rejected": -0.10431348532438278,
      "eval_runtime": 2431.3959,
      "eval_samples_per_second": 78.07,
      "eval_steps_per_second": 1.22,
      "step": 10000
    },
    {
      "epoch": 0.3751100785070544,
      "grad_norm": 1.5106633064077282,
      "learning_rate": 7.996002997502185e-07,
      "log_odds_chosen": 0.5748535394668579,
      "log_odds_ratio": -0.552539050579071,
      "logits/chosen": -0.8177734613418579,
      "logits/rejected": -0.7002929449081421,
      "logps/chosen": -0.6787109375,
      "logps/rejected": -1.0402343273162842,
      "loss": 0.9456,
      "nll_loss": 0.9013671875,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06785888969898224,
      "rewards/margins": 0.03616790845990181,
      "rewards/rejected": -0.10400390625,
      "step": 10010
    },
    {
      "epoch": 0.37548481385021826,
      "grad_norm": 1.8037918830308661,
      "learning_rate": 7.992011980034937e-07,
      "log_odds_chosen": 0.568359375,
      "log_odds_ratio": -0.5755859613418579,
      "logits/chosen": -0.7710937261581421,
      "logits/rejected": -0.658496081829071,
      "logps/chosen": -0.677929699420929,
      "logps/rejected": -1.0546875,
      "loss": 0.9225,
      "nll_loss": 0.863085925579071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06789550930261612,
      "rewards/margins": 0.037735749036073685,
      "rewards/rejected": -0.10551758110523224,
      "step": 10020
    },
    {
      "epoch": 0.37585954919338216,
      "grad_norm": 1.9577740917771003,
      "learning_rate": 7.98802693267671e-07,
      "log_odds_chosen": 0.550646960735321,
      "log_odds_ratio": -0.565136730670929,
      "logits/chosen": -0.7923828363418579,
      "logits/rejected": -0.671582043170929,
      "logps/chosen": -0.7183593511581421,
      "logps/rejected": -1.0701172351837158,
      "loss": 0.9366,
      "nll_loss": 0.8519531488418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07185058295726776,
      "rewards/margins": 0.03513794019818306,
      "rewards/rejected": -0.10709228366613388,
      "step": 10030
    },
    {
      "epoch": 0.37623428453654606,
      "grad_norm": 1.6947936306418332,
      "learning_rate": 7.984047840557992e-07,
      "log_odds_chosen": 0.5337890386581421,
      "log_odds_ratio": -0.57763671875,
      "logits/chosen": -0.7769531011581421,
      "logits/rejected": -0.672656238079071,
      "logps/chosen": -0.7347656488418579,
      "logps/rejected": -1.092382788658142,
      "loss": 0.9191,
      "nll_loss": 0.8984375,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.0733642578125,
      "rewards/margins": 0.035747528076171875,
      "rewards/rejected": -0.10910644382238388,
      "step": 10040
    },
    {
      "epoch": 0.37660901987970996,
      "grad_norm": 1.7080057833573068,
      "learning_rate": 7.980074688861063e-07,
      "log_odds_chosen": 0.386474609375,
      "log_odds_ratio": -0.65087890625,
      "logits/chosen": -0.736132800579071,
      "logits/rejected": -0.651562511920929,
      "logps/chosen": -0.6968749761581421,
      "logps/rejected": -0.9496093988418579,
      "loss": 0.932,
      "nll_loss": 0.907031238079071,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0697021484375,
      "rewards/margins": 0.0253143310546875,
      "rewards/rejected": -0.09495849907398224,
      "step": 10050
    },
    {
      "epoch": 0.37698375522287386,
      "grad_norm": 1.6733902924320907,
      "learning_rate": 7.976107462819775e-07,
      "log_odds_chosen": 0.36003416776657104,
      "log_odds_ratio": -0.6392577886581421,
      "logits/chosen": -0.761523425579071,
      "logits/rejected": -0.6670898199081421,
      "logps/chosen": -0.7132812738418579,
      "logps/rejected": -0.9361327886581421,
      "loss": 0.9198,
      "nll_loss": 0.8373047113418579,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07136230170726776,
      "rewards/margins": 0.022240448743104935,
      "rewards/rejected": -0.09355469048023224,
      "step": 10060
    },
    {
      "epoch": 0.37735849056603776,
      "grad_norm": 1.6587542131952295,
      "learning_rate": 7.97214614771931e-07,
      "log_odds_chosen": 0.4024414122104645,
      "log_odds_ratio": -0.6148437261581421,
      "logits/chosen": -0.7708984613418579,
      "logits/rejected": -0.682812511920929,
      "logps/chosen": -0.640820324420929,
      "logps/rejected": -0.8832031488418579,
      "loss": 0.8985,
      "nll_loss": 0.7845703363418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06411132961511612,
      "rewards/margins": 0.024200439453125,
      "rewards/rejected": -0.08830566704273224,
      "step": 10070
    },
    {
      "epoch": 0.3777332259092016,
      "grad_norm": 1.7330102093843525,
      "learning_rate": 7.968190728895957e-07,
      "log_odds_chosen": 0.5080810785293579,
      "log_odds_ratio": -0.5762695074081421,
      "logits/chosen": -0.7191406488418579,
      "logits/rejected": -0.6253906488418579,
      "logps/chosen": -0.664355456829071,
      "logps/rejected": -0.9771484136581421,
      "loss": 0.9423,
      "nll_loss": 0.8662109375,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06649170070886612,
      "rewards/margins": 0.03114776685833931,
      "rewards/rejected": -0.0977783203125,
      "step": 10080
    },
    {
      "epoch": 0.3781079612523655,
      "grad_norm": 1.6403170422735946,
      "learning_rate": 7.964241191736886e-07,
      "log_odds_chosen": 0.5564330816268921,
      "log_odds_ratio": -0.599414050579071,
      "logits/chosen": -0.7835937738418579,
      "logits/rejected": -0.7025390863418579,
      "logps/chosen": -0.716601550579071,
      "logps/rejected": -1.0859375,
      "loss": 0.9239,
      "nll_loss": 0.8773437738418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07166747748851776,
      "rewards/margins": 0.03687896579504013,
      "rewards/rejected": -0.10856933891773224,
      "step": 10090
    },
    {
      "epoch": 0.3784826965955294,
      "grad_norm": 1.609955397514594,
      "learning_rate": 7.960297521679913e-07,
      "log_odds_chosen": 0.4701171815395355,
      "log_odds_ratio": -0.596484363079071,
      "logits/chosen": -0.7787109613418579,
      "logits/rejected": -0.680468738079071,
      "logps/chosen": -0.6878906488418579,
      "logps/rejected": -0.9800781011581421,
      "loss": 0.929,
      "nll_loss": 0.841015636920929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06873778998851776,
      "rewards/margins": 0.02916564978659153,
      "rewards/rejected": -0.09797362983226776,
      "step": 10100
    },
    {
      "epoch": 0.3788574319386933,
      "grad_norm": 1.6726424469578933,
      "learning_rate": 7.956359704213283e-07,
      "log_odds_chosen": 0.39002686738967896,
      "log_odds_ratio": -0.633496105670929,
      "logits/chosen": -0.7749999761581421,
      "logits/rejected": -0.6839843988418579,
      "logps/chosen": -0.6322265863418579,
      "logps/rejected": -0.87109375,
      "loss": 0.9443,
      "nll_loss": 0.7835937738418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06319580227136612,
      "rewards/margins": 0.02388915978372097,
      "rewards/rejected": -0.08706054836511612,
      "step": 10110
    },
    {
      "epoch": 0.3792321672818572,
      "grad_norm": 1.7210286490394402,
      "learning_rate": 7.95242772487544e-07,
      "log_odds_chosen": 0.539538562297821,
      "log_odds_ratio": -0.578125,
      "logits/chosen": -0.715624988079071,
      "logits/rejected": -0.619140625,
      "logps/chosen": -0.675976574420929,
      "logps/rejected": -1.0041015148162842,
      "loss": 0.9305,
      "nll_loss": 0.883007824420929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.067626953125,
      "rewards/margins": 0.03277892991900444,
      "rewards/rejected": -0.10036621242761612,
      "step": 10120
    },
    {
      "epoch": 0.37960690262502106,
      "grad_norm": 1.7475505036585113,
      "learning_rate": 7.94850156925481e-07,
      "log_odds_chosen": 0.5713866949081421,
      "log_odds_ratio": -0.5699218511581421,
      "logits/chosen": -0.754687488079071,
      "logits/rejected": -0.661328136920929,
      "logps/chosen": -0.6670898199081421,
      "logps/rejected": -1.014257788658142,
      "loss": 0.9156,
      "nll_loss": 0.8974609375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06672362983226776,
      "rewards/margins": 0.03467864915728569,
      "rewards/rejected": -0.10148926079273224,
      "step": 10130
    },
    {
      "epoch": 0.37998163796818496,
      "grad_norm": 1.7410228240443149,
      "learning_rate": 7.944581222989574e-07,
      "log_odds_chosen": 0.5486816167831421,
      "log_odds_ratio": -0.5692383050918579,
      "logits/chosen": -0.763671875,
      "logits/rejected": -0.6641601324081421,
      "logps/chosen": -0.6602538824081421,
      "logps/rejected": -1.003515601158142,
      "loss": 0.9167,
      "nll_loss": 0.833984375,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06605224311351776,
      "rewards/margins": 0.0342559814453125,
      "rewards/rejected": -0.1002197265625,
      "step": 10140
    },
    {
      "epoch": 0.38035637331134886,
      "grad_norm": 1.7029365564017416,
      "learning_rate": 7.940666671767441e-07,
      "log_odds_chosen": 0.579907238483429,
      "log_odds_ratio": -0.573535144329071,
      "logits/chosen": -0.722363293170929,
      "logits/rejected": -0.6156250238418579,
      "logps/chosen": -0.681640625,
      "logps/rejected": -1.0869140625,
      "loss": 0.9337,
      "nll_loss": 0.8902343511581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06818847358226776,
      "rewards/margins": 0.0406036376953125,
      "rewards/rejected": -0.10871581733226776,
      "step": 10150
    },
    {
      "epoch": 0.38073110865451276,
      "grad_norm": 1.781590165972759,
      "learning_rate": 7.936757901325451e-07,
      "log_odds_chosen": 0.3271240293979645,
      "log_odds_ratio": -0.63134765625,
      "logits/chosen": -0.726855456829071,
      "logits/rejected": -0.6767578125,
      "logps/chosen": -0.714648425579071,
      "logps/rejected": -0.9175781011581421,
      "loss": 0.9484,
      "nll_loss": 0.899218738079071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07145996391773224,
      "rewards/margins": 0.02032012864947319,
      "rewards/rejected": -0.09174804389476776,
      "step": 10160
    },
    {
      "epoch": 0.38110584399767666,
      "grad_norm": 1.5413136650166386,
      "learning_rate": 7.932854897449727e-07,
      "log_odds_chosen": 0.38654786348342896,
      "log_odds_ratio": -0.6201171875,
      "logits/chosen": -0.7353515625,
      "logits/rejected": -0.687304675579071,
      "logps/chosen": -0.725781261920929,
      "logps/rejected": -0.962695300579071,
      "loss": 0.9098,
      "nll_loss": 0.915820300579071,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07255859673023224,
      "rewards/margins": 0.023726653307676315,
      "rewards/rejected": -0.09624023735523224,
      "step": 10170
    },
    {
      "epoch": 0.3814805793408405,
      "grad_norm": 1.6036336277943,
      "learning_rate": 7.928957645975286e-07,
      "log_odds_chosen": 0.4088378846645355,
      "log_odds_ratio": -0.62890625,
      "logits/chosen": -0.796093761920929,
      "logits/rejected": -0.708789050579071,
      "logps/chosen": -0.68359375,
      "logps/rejected": -0.954882800579071,
      "loss": 0.9085,
      "nll_loss": 0.840624988079071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06833495944738388,
      "rewards/margins": 0.02716217003762722,
      "rewards/rejected": -0.09555663913488388,
      "step": 10180
    },
    {
      "epoch": 0.3818553146840044,
      "grad_norm": 1.7591039717032009,
      "learning_rate": 7.925066132785799e-07,
      "log_odds_chosen": 0.3938842713832855,
      "log_odds_ratio": -0.634472668170929,
      "logits/chosen": -0.743359386920929,
      "logits/rejected": -0.6470702886581421,
      "logps/chosen": -0.7035156488418579,
      "logps/rejected": -0.9525390863418579,
      "loss": 0.9236,
      "nll_loss": 0.812304675579071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07034911960363388,
      "rewards/margins": 0.02495117112994194,
      "rewards/rejected": -0.0953369140625,
      "step": 10190
    },
    {
      "epoch": 0.3822300500271683,
      "grad_norm": 1.623809483133093,
      "learning_rate": 7.921180343813395e-07,
      "log_odds_chosen": 0.40972900390625,
      "log_odds_ratio": -0.6226562261581421,
      "logits/chosen": -0.783203125,
      "logits/rejected": -0.6947265863418579,
      "logps/chosen": -0.7124999761581421,
      "logps/rejected": -0.9769531488418579,
      "loss": 0.9178,
      "nll_loss": 0.8759765625,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07130126655101776,
      "rewards/margins": 0.02644500695168972,
      "rewards/rejected": -0.09770508110523224,
      "step": 10200
    },
    {
      "epoch": 0.3826047853703322,
      "grad_norm": 1.6035877401961658,
      "learning_rate": 7.917300265038436e-07,
      "log_odds_chosen": 0.320556640625,
      "log_odds_ratio": -0.6534179449081421,
      "logits/chosen": -0.757617175579071,
      "logits/rejected": -0.674511730670929,
      "logps/chosen": -0.7201172113418579,
      "logps/rejected": -0.9185546636581421,
      "loss": 0.9353,
      "nll_loss": 0.9048827886581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07199706882238388,
      "rewards/margins": 0.01984863355755806,
      "rewards/rejected": -0.09185791015625,
      "step": 10210
    },
    {
      "epoch": 0.3829795207134961,
      "grad_norm": 1.6527137472912083,
      "learning_rate": 7.913425882489307e-07,
      "log_odds_chosen": 0.5667480230331421,
      "log_odds_ratio": -0.579296886920929,
      "logits/chosen": -0.7828124761581421,
      "logits/rejected": -0.677441418170929,
      "logps/chosen": -0.6884765625,
      "logps/rejected": -1.0466797351837158,
      "loss": 0.9015,
      "nll_loss": 0.8226562738418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06881103664636612,
      "rewards/margins": 0.0359039306640625,
      "rewards/rejected": -0.10468749701976776,
      "step": 10220
    },
    {
      "epoch": 0.38335425605665996,
      "grad_norm": 1.7675109288989526,
      "learning_rate": 7.909557182242211e-07,
      "log_odds_chosen": 0.45850831270217896,
      "log_odds_ratio": -0.6226562261581421,
      "logits/chosen": -0.687695324420929,
      "logits/rejected": -0.595996081829071,
      "logps/chosen": -0.651074230670929,
      "logps/rejected": -0.931640625,
      "loss": 0.9202,
      "nll_loss": 0.8822265863418579,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06516113132238388,
      "rewards/margins": 0.02799835242331028,
      "rewards/rejected": -0.09318847954273224,
      "step": 10230
    },
    {
      "epoch": 0.38372899139982386,
      "grad_norm": 1.66878322076802,
      "learning_rate": 7.905694150420947e-07,
      "log_odds_chosen": 0.43730467557907104,
      "log_odds_ratio": -0.607421875,
      "logits/chosen": -0.7103515863418579,
      "logits/rejected": -0.6171875,
      "logps/chosen": -0.6722656488418579,
      "logps/rejected": -0.9615234136581421,
      "loss": 0.9141,
      "nll_loss": 0.9019531011581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06717529147863388,
      "rewards/margins": 0.02892913855612278,
      "rewards/rejected": -0.09614257514476776,
      "step": 10240
    },
    {
      "epoch": 0.38410372674298776,
      "grad_norm": 1.7575838753196626,
      "learning_rate": 7.901836773196717e-07,
      "log_odds_chosen": 0.47613525390625,
      "log_odds_ratio": -0.5897461175918579,
      "logits/chosen": -0.7359374761581421,
      "logits/rejected": -0.6709960699081421,
      "logps/chosen": -0.6527343988418579,
      "logps/rejected": -0.9466797113418579,
      "loss": 0.9192,
      "nll_loss": 0.853710949420929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06524658203125,
      "rewards/margins": 0.02943115308880806,
      "rewards/rejected": -0.09467773139476776,
      "step": 10250
    },
    {
      "epoch": 0.38447846208615166,
      "grad_norm": 1.7231045293508687,
      "learning_rate": 7.897985036787898e-07,
      "log_odds_chosen": 0.5166260004043579,
      "log_odds_ratio": -0.597460925579071,
      "logits/chosen": -0.748828113079071,
      "logits/rejected": -0.630664050579071,
      "logps/chosen": -0.7232421636581421,
      "logps/rejected": -1.051367163658142,
      "loss": 0.9279,
      "nll_loss": 0.903124988079071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07227782905101776,
      "rewards/margins": 0.03284912183880806,
      "rewards/rejected": -0.10512695461511612,
      "step": 10260
    },
    {
      "epoch": 0.38485319742931556,
      "grad_norm": 1.7050988063632029,
      "learning_rate": 7.894138927459853e-07,
      "log_odds_chosen": 0.5685180425643921,
      "log_odds_ratio": -0.58447265625,
      "logits/chosen": -0.743945300579071,
      "logits/rejected": -0.6688476800918579,
      "logps/chosen": -0.6666015386581421,
      "logps/rejected": -1.0167968273162842,
      "loss": 0.9304,
      "nll_loss": 0.8587890863418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06668701022863388,
      "rewards/margins": 0.03504791110754013,
      "rewards/rejected": -0.10174560546875,
      "step": 10270
    },
    {
      "epoch": 0.3852279327724794,
      "grad_norm": 1.6695034803888484,
      "learning_rate": 7.890298431524716e-07,
      "log_odds_chosen": 0.5848144292831421,
      "log_odds_ratio": -0.577441394329071,
      "logits/chosen": -0.7406250238418579,
      "logits/rejected": -0.64697265625,
      "logps/chosen": -0.65966796875,
      "logps/rejected": -1.0310547351837158,
      "loss": 0.9252,
      "nll_loss": 0.890820324420929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06600341945886612,
      "rewards/margins": 0.03713226318359375,
      "rewards/rejected": -0.10311279445886612,
      "step": 10280
    },
    {
      "epoch": 0.3856026681156433,
      "grad_norm": 1.721732655900969,
      "learning_rate": 7.88646353534119e-07,
      "log_odds_chosen": 0.5425781011581421,
      "log_odds_ratio": -0.574511706829071,
      "logits/chosen": -0.7255859375,
      "logits/rejected": -0.61865234375,
      "logps/chosen": -0.6937500238418579,
      "logps/rejected": -1.0654296875,
      "loss": 0.9576,
      "nll_loss": 0.918164074420929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06934814155101776,
      "rewards/margins": 0.037168122828006744,
      "rewards/rejected": -0.10646972805261612,
      "step": 10290
    },
    {
      "epoch": 0.3859774034588072,
      "grad_norm": 1.7461022791009209,
      "learning_rate": 7.882634225314345e-07,
      "log_odds_chosen": 0.5621093511581421,
      "log_odds_ratio": -0.5604492425918579,
      "logits/chosen": -0.7720702886581421,
      "logits/rejected": -0.6416015625,
      "logps/chosen": -0.652148425579071,
      "logps/rejected": -1.0048828125,
      "loss": 0.9216,
      "nll_loss": 0.872851550579071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06524658203125,
      "rewards/margins": 0.03524170070886612,
      "rewards/rejected": -0.10047607123851776,
      "step": 10300
    },
    {
      "epoch": 0.3863521388019711,
      "grad_norm": 1.7236231369588877,
      "learning_rate": 7.87881048789541e-07,
      "log_odds_chosen": 0.47900390625,
      "log_odds_ratio": -0.619140625,
      "logits/chosen": -0.8070312738418579,
      "logits/rejected": -0.6875,
      "logps/chosen": -0.7413085699081421,
      "logps/rejected": -1.070898413658142,
      "loss": 0.9269,
      "nll_loss": 0.891406238079071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07404784858226776,
      "rewards/margins": 0.03296203538775444,
      "rewards/rejected": -0.10710449516773224,
      "step": 10310
    },
    {
      "epoch": 0.386726874145135,
      "grad_norm": 1.7375201437838914,
      "learning_rate": 7.874992309581578e-07,
      "log_odds_chosen": 0.5645996332168579,
      "log_odds_ratio": -0.5704101324081421,
      "logits/chosen": -0.762499988079071,
      "logits/rejected": -0.655468761920929,
      "logps/chosen": -0.625,
      "logps/rejected": -0.958984375,
      "loss": 0.9425,
      "nll_loss": 0.8404296636581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06253661960363388,
      "rewards/margins": 0.033359527587890625,
      "rewards/rejected": -0.09589843451976776,
      "step": 10320
    },
    {
      "epoch": 0.3871016094882989,
      "grad_norm": 1.9570114450982812,
      "learning_rate": 7.871179676915801e-07,
      "log_odds_chosen": 0.650134265422821,
      "log_odds_ratio": -0.5381835699081421,
      "logits/chosen": -0.7293945550918579,
      "logits/rejected": -0.614453136920929,
      "logps/chosen": -0.67333984375,
      "logps/rejected": -1.0759766101837158,
      "loss": 0.9074,
      "nll_loss": 0.878125011920929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06737060844898224,
      "rewards/margins": 0.04031066969037056,
      "rewards/rejected": -0.107666015625,
      "step": 10330
    },
    {
      "epoch": 0.38747634483146276,
      "grad_norm": 1.7112160786704662,
      "learning_rate": 7.867372576486597e-07,
      "log_odds_chosen": 0.49040526151657104,
      "log_odds_ratio": -0.57177734375,
      "logits/chosen": -0.739062488079071,
      "logits/rejected": -0.658007800579071,
      "logps/chosen": -0.6470702886581421,
      "logps/rejected": -0.942578136920929,
      "loss": 0.9298,
      "nll_loss": 0.832812488079071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06475830078125,
      "rewards/margins": 0.02957458421587944,
      "rewards/rejected": -0.09431152045726776,
      "step": 10340
    },
    {
      "epoch": 0.38785108017462666,
      "grad_norm": 1.7083800236676565,
      "learning_rate": 7.863570994927847e-07,
      "log_odds_chosen": 0.523510754108429,
      "log_odds_ratio": -0.59716796875,
      "logits/chosen": -0.7923828363418579,
      "logits/rejected": -0.6695312261581421,
      "logps/chosen": -0.6927734613418579,
      "logps/rejected": -1.0388672351837158,
      "loss": 0.9311,
      "nll_loss": 0.888476550579071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06928710639476776,
      "rewards/margins": 0.03467254713177681,
      "rewards/rejected": -0.10394287109375,
      "step": 10350
    },
    {
      "epoch": 0.38822581551779056,
      "grad_norm": 1.6478370538883038,
      "learning_rate": 7.859774918918594e-07,
      "log_odds_chosen": 0.49134522676467896,
      "log_odds_ratio": -0.585742175579071,
      "logits/chosen": -0.7943359613418579,
      "logits/rejected": -0.686328113079071,
      "logps/chosen": -0.679492175579071,
      "logps/rejected": -1.008203148841858,
      "loss": 0.9402,
      "nll_loss": 0.8734375238418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06795654445886612,
      "rewards/margins": 0.032808683812618256,
      "rewards/rejected": -0.10080566257238388,
      "step": 10360
    },
    {
      "epoch": 0.38860055086095446,
      "grad_norm": 1.76946811485592,
      "learning_rate": 7.855984335182852e-07,
      "log_odds_chosen": 0.4496704041957855,
      "log_odds_ratio": -0.5855468511581421,
      "logits/chosen": -0.7105468511581421,
      "logits/rejected": -0.667675793170929,
      "logps/chosen": -0.6509765386581421,
      "logps/rejected": -0.9193359613418579,
      "loss": 0.9144,
      "nll_loss": 0.830273449420929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06512451171875,
      "rewards/margins": 0.02685241773724556,
      "rewards/rejected": -0.0919189453125,
      "step": 10370
    },
    {
      "epoch": 0.38897528620411836,
      "grad_norm": 1.7203013411908183,
      "learning_rate": 7.852199230489422e-07,
      "log_odds_chosen": 0.44709473848342896,
      "log_odds_ratio": -0.6000000238418579,
      "logits/chosen": -0.763867199420929,
      "logits/rejected": -0.716992199420929,
      "logps/chosen": -0.7320312261581421,
      "logps/rejected": -1.0109374523162842,
      "loss": 0.9176,
      "nll_loss": 0.860156238079071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07320556789636612,
      "rewards/margins": 0.02785186842083931,
      "rewards/rejected": -0.10104980319738388,
      "step": 10380
    },
    {
      "epoch": 0.3893500215472822,
      "grad_norm": 1.620415359681351,
      "learning_rate": 7.848419591651668e-07,
      "log_odds_chosen": 0.4502929747104645,
      "log_odds_ratio": -0.630566418170929,
      "logits/chosen": -0.79296875,
      "logits/rejected": -0.6943359375,
      "logps/chosen": -0.745312511920929,
      "logps/rejected": -1.0607421398162842,
      "loss": 0.9527,
      "nll_loss": 0.9146484136581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07453612983226776,
      "rewards/margins": 0.031591035425662994,
      "rewards/rejected": -0.10610351711511612,
      "step": 10390
    },
    {
      "epoch": 0.3897247568904461,
      "grad_norm": 1.6674705928108662,
      "learning_rate": 7.844645405527361e-07,
      "log_odds_chosen": 0.5246216058731079,
      "log_odds_ratio": -0.5975586175918579,
      "logits/chosen": -0.73681640625,
      "logits/rejected": -0.655957043170929,
      "logps/chosen": -0.666210949420929,
      "logps/rejected": -1.0187499523162842,
      "loss": 0.9131,
      "nll_loss": 0.8353515863418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06663818657398224,
      "rewards/margins": 0.03522491455078125,
      "rewards/rejected": -0.10183105617761612,
      "step": 10400
    },
    {
      "epoch": 0.39009949223361,
      "grad_norm": 1.8575325321770098,
      "learning_rate": 7.840876659018457e-07,
      "log_odds_chosen": 0.4977050721645355,
      "log_odds_ratio": -0.6021484136581421,
      "logits/chosen": -0.7867187261581421,
      "logits/rejected": -0.6787109375,
      "logps/chosen": -0.7376953363418579,
      "logps/rejected": -1.0583984851837158,
      "loss": 0.936,
      "nll_loss": 0.9203125238418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07376708835363388,
      "rewards/margins": 0.032001495361328125,
      "rewards/rejected": -0.10577392578125,
      "step": 10410
    },
    {
      "epoch": 0.3904742275767739,
      "grad_norm": 1.7253039327080224,
      "learning_rate": 7.837113339070922e-07,
      "log_odds_chosen": 0.44818115234375,
      "log_odds_ratio": -0.5967773199081421,
      "logits/chosen": -0.7884765863418579,
      "logits/rejected": -0.6851562261581421,
      "logps/chosen": -0.630664050579071,
      "logps/rejected": -0.909960925579071,
      "loss": 0.9086,
      "nll_loss": 0.790234386920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06302490085363388,
      "rewards/margins": 0.02793731726706028,
      "rewards/rejected": -0.09097900241613388,
      "step": 10420
    },
    {
      "epoch": 0.3908489629199378,
      "grad_norm": 1.724648187914037,
      "learning_rate": 7.833355432674538e-07,
      "log_odds_chosen": 0.5458008050918579,
      "log_odds_ratio": -0.592480480670929,
      "logits/chosen": -0.8062499761581421,
      "logits/rejected": -0.68408203125,
      "logps/chosen": -0.6744140386581421,
      "logps/rejected": -1.040429711341858,
      "loss": 0.9224,
      "nll_loss": 0.7948242425918579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06745605170726776,
      "rewards/margins": 0.03660430759191513,
      "rewards/rejected": -0.10400390625,
      "step": 10430
    },
    {
      "epoch": 0.39122369826310166,
      "grad_norm": 1.5748384630924264,
      "learning_rate": 7.829602926862713e-07,
      "log_odds_chosen": 0.4023681581020355,
      "log_odds_ratio": -0.634472668170929,
      "logits/chosen": -0.7333984375,
      "logits/rejected": -0.653124988079071,
      "logps/chosen": -0.7216796875,
      "logps/rejected": -0.9745117425918579,
      "loss": 0.9167,
      "nll_loss": 0.906054675579071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0721435546875,
      "rewards/margins": 0.02521362341940403,
      "rewards/rejected": -0.09738769382238388,
      "step": 10440
    },
    {
      "epoch": 0.39159843360626556,
      "grad_norm": 1.8245480008627195,
      "learning_rate": 7.825855808712296e-07,
      "log_odds_chosen": 0.506396472454071,
      "log_odds_ratio": -0.57861328125,
      "logits/chosen": -0.8179687261581421,
      "logits/rejected": -0.699414074420929,
      "logps/chosen": -0.670703113079071,
      "logps/rejected": -0.972851574420929,
      "loss": 0.9203,
      "nll_loss": 0.8642578125,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06710205227136612,
      "rewards/margins": 0.03028717078268528,
      "rewards/rejected": -0.09731445461511612,
      "step": 10450
    },
    {
      "epoch": 0.39197316894942946,
      "grad_norm": 1.6531196714959442,
      "learning_rate": 7.822114065343386e-07,
      "log_odds_chosen": 0.4507690370082855,
      "log_odds_ratio": -0.6097656488418579,
      "logits/chosen": -0.8076171875,
      "logits/rejected": -0.684277355670929,
      "logps/chosen": -0.718554675579071,
      "logps/rejected": -1.015234351158142,
      "loss": 0.9317,
      "nll_loss": 0.888867199420929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07188721001148224,
      "rewards/margins": 0.02973022498190403,
      "rewards/rejected": -0.10158691555261612,
      "step": 10460
    },
    {
      "epoch": 0.39234790429259336,
      "grad_norm": 1.6900637169531247,
      "learning_rate": 7.818377683919149e-07,
      "log_odds_chosen": 0.64093017578125,
      "log_odds_ratio": -0.537304699420929,
      "logits/chosen": -0.739453136920929,
      "logits/rejected": -0.607226550579071,
      "logps/chosen": -0.683398425579071,
      "logps/rejected": -1.0945312976837158,
      "loss": 0.9329,
      "nll_loss": 0.8671875,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06832275539636612,
      "rewards/margins": 0.04122619703412056,
      "rewards/rejected": -0.10944823920726776,
      "step": 10470
    },
    {
      "epoch": 0.39272263963575726,
      "grad_norm": 1.7095775970010842,
      "learning_rate": 7.814646651645635e-07,
      "log_odds_chosen": 0.47612303495407104,
      "log_odds_ratio": -0.5848633050918579,
      "logits/chosen": -0.7669922113418579,
      "logits/rejected": -0.661328136920929,
      "logps/chosen": -0.7568359375,
      "logps/rejected": -1.0519530773162842,
      "loss": 0.9449,
      "nll_loss": 0.9312499761581421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07567138969898224,
      "rewards/margins": 0.02945709228515625,
      "rewards/rejected": -0.10517577826976776,
      "step": 10480
    },
    {
      "epoch": 0.3930973749789211,
      "grad_norm": 1.8668977168870606,
      "learning_rate": 7.810920955771586e-07,
      "log_odds_chosen": 0.3665527403354645,
      "log_odds_ratio": -0.6444336175918579,
      "logits/chosen": -0.7876952886581421,
      "logits/rejected": -0.696093738079071,
      "logps/chosen": -0.7337890863418579,
      "logps/rejected": -0.9720703363418579,
      "loss": 0.9366,
      "nll_loss": 0.919140636920929,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07337646186351776,
      "rewards/margins": 0.023906707763671875,
      "rewards/rejected": -0.09721679985523224,
      "step": 10490
    },
    {
      "epoch": 0.393472110322085,
      "grad_norm": 1.664104677029002,
      "learning_rate": 7.807200583588266e-07,
      "log_odds_chosen": 0.5621582269668579,
      "log_odds_ratio": -0.58056640625,
      "logits/chosen": -0.749316394329071,
      "logits/rejected": -0.6751953363418579,
      "logps/chosen": -0.663867175579071,
      "logps/rejected": -1.0017578601837158,
      "loss": 0.9219,
      "nll_loss": 0.8365234136581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06638183444738388,
      "rewards/margins": 0.03380737453699112,
      "rewards/rejected": -0.10018310695886612,
      "step": 10500
    },
    {
      "epoch": 0.3938468456652489,
      "grad_norm": 1.6412167599674161,
      "learning_rate": 7.803485522429261e-07,
      "log_odds_chosen": 0.3815551698207855,
      "log_odds_ratio": -0.619140625,
      "logits/chosen": -0.7568359375,
      "logits/rejected": -0.6498047113418579,
      "logps/chosen": -0.692187488079071,
      "logps/rejected": -0.9498046636581421,
      "loss": 0.9448,
      "nll_loss": 0.880859375,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0692138671875,
      "rewards/margins": 0.0258026123046875,
      "rewards/rejected": -0.09503173828125,
      "step": 10510
    },
    {
      "epoch": 0.3942215810084128,
      "grad_norm": 1.6805290532708557,
      "learning_rate": 7.799775759670318e-07,
      "log_odds_chosen": 0.5530639886856079,
      "log_odds_ratio": -0.5889648199081421,
      "logits/chosen": -0.7789062261581421,
      "logits/rejected": -0.6957031488418579,
      "logps/chosen": -0.708203136920929,
      "logps/rejected": -1.0626952648162842,
      "loss": 0.9302,
      "nll_loss": 0.828125,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07076416164636612,
      "rewards/margins": 0.03548584133386612,
      "rewards/rejected": -0.10625000298023224,
      "step": 10520
    },
    {
      "epoch": 0.3945963163515767,
      "grad_norm": 1.6384794093933732,
      "learning_rate": 7.796071282729149e-07,
      "log_odds_chosen": 0.48530274629592896,
      "log_odds_ratio": -0.59765625,
      "logits/chosen": -0.758593738079071,
      "logits/rejected": -0.6371093988418579,
      "logps/chosen": -0.6849609613418579,
      "logps/rejected": -0.9957031011581421,
      "loss": 0.9448,
      "nll_loss": 0.9281250238418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06855468451976776,
      "rewards/margins": 0.03097229078412056,
      "rewards/rejected": -0.09956054389476776,
      "step": 10530
    },
    {
      "epoch": 0.3949710516947406,
      "grad_norm": 1.7505433525281482,
      "learning_rate": 7.792372079065259e-07,
      "log_odds_chosen": 0.40965574979782104,
      "log_odds_ratio": -0.6195312738418579,
      "logits/chosen": -0.776562511920929,
      "logits/rejected": -0.7056640386581421,
      "logps/chosen": -0.685351550579071,
      "logps/rejected": -0.953417956829071,
      "loss": 0.9255,
      "nll_loss": 0.9232422113418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06854248046875,
      "rewards/margins": 0.026739884167909622,
      "rewards/rejected": -0.09527587890625,
      "step": 10540
    },
    {
      "epoch": 0.39534578703790446,
      "grad_norm": 1.6968527539232812,
      "learning_rate": 7.788678136179767e-07,
      "log_odds_chosen": 0.46270751953125,
      "log_odds_ratio": -0.6172851324081421,
      "logits/chosen": -0.70751953125,
      "logits/rejected": -0.6322265863418579,
      "logps/chosen": -0.693359375,
      "logps/rejected": -1.0275390148162842,
      "loss": 0.9164,
      "nll_loss": 0.9039062261581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06932373344898224,
      "rewards/margins": 0.03347473219037056,
      "rewards/rejected": -0.10277099907398224,
      "step": 10550
    },
    {
      "epoch": 0.39572052238106836,
      "grad_norm": 9.486713184007808,
      "learning_rate": 7.78498944161523e-07,
      "log_odds_chosen": 0.694506824016571,
      "log_odds_ratio": -0.55078125,
      "logits/chosen": -0.742480456829071,
      "logits/rejected": -0.6543945074081421,
      "logps/chosen": -0.6551758050918579,
      "logps/rejected": -1.114648461341858,
      "loss": 0.9368,
      "nll_loss": 0.820507824420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06550292670726776,
      "rewards/margins": 0.0458526611328125,
      "rewards/rejected": -0.11140136420726776,
      "step": 10560
    },
    {
      "epoch": 0.39609525772423226,
      "grad_norm": 1.7330921461799649,
      "learning_rate": 7.781305982955459e-07,
      "log_odds_chosen": 0.44318848848342896,
      "log_odds_ratio": -0.6114257574081421,
      "logits/chosen": -0.768750011920929,
      "logits/rejected": -0.6761718988418579,
      "logps/chosen": -0.6767578125,
      "logps/rejected": -0.9595702886581421,
      "loss": 0.9355,
      "nll_loss": 0.8744140863418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06773681938648224,
      "rewards/margins": 0.02821655198931694,
      "rewards/rejected": -0.09587402641773224,
      "step": 10570
    },
    {
      "epoch": 0.39646999306739616,
      "grad_norm": 1.661484466234196,
      "learning_rate": 7.777627747825355e-07,
      "log_odds_chosen": 0.34697264432907104,
      "log_odds_ratio": -0.63916015625,
      "logits/chosen": -0.7369140386581421,
      "logits/rejected": -0.6494140625,
      "logps/chosen": -0.727832019329071,
      "logps/rejected": -0.942187488079071,
      "loss": 0.9029,
      "nll_loss": 0.880078136920929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07281494140625,
      "rewards/margins": 0.02145538292825222,
      "rewards/rejected": -0.09423828125,
      "step": 10580
    },
    {
      "epoch": 0.39684472841056007,
      "grad_norm": 1.7589203673595162,
      "learning_rate": 7.773954723890725e-07,
      "log_odds_chosen": 0.37598878145217896,
      "log_odds_ratio": -0.616992175579071,
      "logits/chosen": -0.7232421636581421,
      "logits/rejected": -0.6739257574081421,
      "logps/chosen": -0.7061523199081421,
      "logps/rejected": -0.944140613079071,
      "loss": 0.942,
      "nll_loss": 0.877734363079071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07056884467601776,
      "rewards/margins": 0.02380065992474556,
      "rewards/rejected": -0.09442138671875,
      "step": 10590
    },
    {
      "epoch": 0.3972194637537239,
      "grad_norm": 1.8590235158644095,
      "learning_rate": 7.770286898858113e-07,
      "log_odds_chosen": 0.515063464641571,
      "log_odds_ratio": -0.611132800579071,
      "logits/chosen": -0.7588866949081421,
      "logits/rejected": -0.644824206829071,
      "logps/chosen": -0.69140625,
      "logps/rejected": -1.044335961341858,
      "loss": 0.9496,
      "nll_loss": 0.8306640386581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06917724758386612,
      "rewards/margins": 0.03523864597082138,
      "rewards/rejected": -0.10441894829273224,
      "step": 10600
    },
    {
      "epoch": 0.3975941990968878,
      "grad_norm": 1.6876545407976775,
      "learning_rate": 7.766624260474625e-07,
      "log_odds_chosen": 0.5823974609375,
      "log_odds_ratio": -0.5537109375,
      "logits/chosen": -0.705859363079071,
      "logits/rejected": -0.5975586175918579,
      "logps/chosen": -0.683886706829071,
      "logps/rejected": -1.0505859851837158,
      "loss": 0.9285,
      "nll_loss": 0.866406261920929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06840820610523224,
      "rewards/margins": 0.03664856031537056,
      "rewards/rejected": -0.10495605319738388,
      "step": 10610
    },
    {
      "epoch": 0.3979689344400517,
      "grad_norm": 1.6044061450680502,
      "learning_rate": 7.762966796527759e-07,
      "log_odds_chosen": 0.5561279058456421,
      "log_odds_ratio": -0.5755859613418579,
      "logits/chosen": -0.707812488079071,
      "logits/rejected": -0.57568359375,
      "logps/chosen": -0.628222644329071,
      "logps/rejected": -0.984375,
      "loss": 0.9508,
      "nll_loss": 0.8353515863418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06284179538488388,
      "rewards/margins": 0.035607147961854935,
      "rewards/rejected": -0.09840087592601776,
      "step": 10620
    },
    {
      "epoch": 0.3983436697832156,
      "grad_norm": 1.726216557218643,
      "learning_rate": 7.759314494845234e-07,
      "log_odds_chosen": 0.5100952386856079,
      "log_odds_ratio": -0.5997070074081421,
      "logits/chosen": -0.7261718511581421,
      "logits/rejected": -0.623046875,
      "logps/chosen": -0.6832031011581421,
      "logps/rejected": -0.995898425579071,
      "loss": 0.9287,
      "nll_loss": 0.8818359375,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.068359375,
      "rewards/margins": 0.03122863732278347,
      "rewards/rejected": -0.09956054389476776,
      "step": 10630
    },
    {
      "epoch": 0.3987184051263795,
      "grad_norm": 1.7768800879912154,
      "learning_rate": 7.755667343294812e-07,
      "log_odds_chosen": 0.4142211973667145,
      "log_odds_ratio": -0.6351562738418579,
      "logits/chosen": -0.7445312738418579,
      "logits/rejected": -0.6695312261581421,
      "logps/chosen": -0.707812488079071,
      "logps/rejected": -0.9716796875,
      "loss": 0.9229,
      "nll_loss": 0.924023449420929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07078857719898224,
      "rewards/margins": 0.02637634240090847,
      "rewards/rejected": -0.09714355319738388,
      "step": 10640
    },
    {
      "epoch": 0.39909314046954336,
      "grad_norm": 1.7764087077282937,
      "learning_rate": 7.752025329784146e-07,
      "log_odds_chosen": 0.4220336973667145,
      "log_odds_ratio": -0.609082043170929,
      "logits/chosen": -0.7046874761581421,
      "logits/rejected": -0.628613293170929,
      "logps/chosen": -0.678515613079071,
      "logps/rejected": -0.947265625,
      "loss": 0.9385,
      "nll_loss": 0.8662109375,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06785888969898224,
      "rewards/margins": 0.02686157263815403,
      "rewards/rejected": -0.0947265625,
      "step": 10650
    },
    {
      "epoch": 0.39946787581270726,
      "grad_norm": 1.9243959444317233,
      "learning_rate": 7.748388442260596e-07,
      "log_odds_chosen": 0.5599120855331421,
      "log_odds_ratio": -0.579785168170929,
      "logits/chosen": -0.725781261920929,
      "logits/rejected": -0.651171863079071,
      "logps/chosen": -0.678515613079071,
      "logps/rejected": -1.0392577648162842,
      "loss": 0.9236,
      "nll_loss": 0.8583984375,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06794433295726776,
      "rewards/margins": 0.03603057935833931,
      "rewards/rejected": -0.10385742038488388,
      "step": 10660
    },
    {
      "epoch": 0.39984261115587116,
      "grad_norm": 1.7584841955358708,
      "learning_rate": 7.744756668711065e-07,
      "log_odds_chosen": 0.608959972858429,
      "log_odds_ratio": -0.5653320550918579,
      "logits/chosen": -0.7710937261581421,
      "logits/rejected": -0.6587890386581421,
      "logps/chosen": -0.65966796875,
      "logps/rejected": -1.0515625476837158,
      "loss": 0.9195,
      "nll_loss": 0.8941406011581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06601562350988388,
      "rewards/margins": 0.039237212389707565,
      "rewards/rejected": -0.1051025390625,
      "step": 10670
    },
    {
      "epoch": 0.40021734649903506,
      "grad_norm": 1.6899548728290112,
      "learning_rate": 7.741129997161835e-07,
      "log_odds_chosen": 0.4721435606479645,
      "log_odds_ratio": -0.6187499761581421,
      "logits/chosen": -0.7284179925918579,
      "logits/rejected": -0.6522461175918579,
      "logps/chosen": -0.6929687261581421,
      "logps/rejected": -1.0173828601837158,
      "loss": 0.9375,
      "nll_loss": 0.8525390625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06929931789636612,
      "rewards/margins": 0.03244781494140625,
      "rewards/rejected": -0.10178222507238388,
      "step": 10680
    },
    {
      "epoch": 0.40059208184219897,
      "grad_norm": 1.6627519169390386,
      "learning_rate": 7.737508415678403e-07,
      "log_odds_chosen": 0.415771484375,
      "log_odds_ratio": -0.6162109375,
      "logits/chosen": -0.749804675579071,
      "logits/rejected": -0.7007812261581421,
      "logps/chosen": -0.686718761920929,
      "logps/rejected": -0.959179699420929,
      "loss": 0.9423,
      "nll_loss": 0.911914050579071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06864013522863388,
      "rewards/margins": 0.02722625806927681,
      "rewards/rejected": -0.09587402641773224,
      "step": 10690
    },
    {
      "epoch": 0.4009668171853628,
      "grad_norm": 1.7576587730741642,
      "learning_rate": 7.733891912365308e-07,
      "log_odds_chosen": 0.4210449159145355,
      "log_odds_ratio": -0.641796886920929,
      "logits/chosen": -0.7708984613418579,
      "logits/rejected": -0.6597656011581421,
      "logps/chosen": -0.7066406011581421,
      "logps/rejected": -0.9833984375,
      "loss": 0.902,
      "nll_loss": 0.8580077886581421,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07064209133386612,
      "rewards/margins": 0.02773437462747097,
      "rewards/rejected": -0.09833984076976776,
      "step": 10700
    },
    {
      "epoch": 0.4013415525285267,
      "grad_norm": 1.8368111571303691,
      "learning_rate": 7.730280475365979e-07,
      "log_odds_chosen": 0.3921142518520355,
      "log_odds_ratio": -0.6324218511581421,
      "logits/chosen": -0.7660156488418579,
      "logits/rejected": -0.6796875,
      "logps/chosen": -0.7142578363418579,
      "logps/rejected": -0.955859363079071,
      "loss": 0.9116,
      "nll_loss": 0.846875011920929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07144775241613388,
      "rewards/margins": 0.024109650403261185,
      "rewards/rejected": -0.09562988579273224,
      "step": 10710
    },
    {
      "epoch": 0.4017162878716906,
      "grad_norm": 1.630690500095941,
      "learning_rate": 7.726674092862557e-07,
      "log_odds_chosen": 0.3624206483364105,
      "log_odds_ratio": -0.6348632574081421,
      "logits/chosen": -0.8134765625,
      "logits/rejected": -0.703125,
      "logps/chosen": -0.7216796875,
      "logps/rejected": -0.948437511920929,
      "loss": 0.9293,
      "nll_loss": 0.8427734375,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07216797024011612,
      "rewards/margins": 0.02265777625143528,
      "rewards/rejected": -0.09482421725988388,
      "step": 10720
    },
    {
      "epoch": 0.4020910232148545,
      "grad_norm": 1.6080601971062405,
      "learning_rate": 7.723072753075748e-07,
      "log_odds_chosen": 0.511425793170929,
      "log_odds_ratio": -0.6136718988418579,
      "logits/chosen": -0.717578113079071,
      "logits/rejected": -0.628222644329071,
      "logps/chosen": -0.6927734613418579,
      "logps/rejected": -1.0322265625,
      "loss": 0.9304,
      "nll_loss": 0.875292956829071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06929931789636612,
      "rewards/margins": 0.03389892727136612,
      "rewards/rejected": -0.10324706882238388,
      "step": 10730
    },
    {
      "epoch": 0.4024657585580184,
      "grad_norm": 1.7252110855893226,
      "learning_rate": 7.719476444264649e-07,
      "log_odds_chosen": 0.5286620855331421,
      "log_odds_ratio": -0.586132824420929,
      "logits/chosen": -0.7640625238418579,
      "logits/rejected": -0.63720703125,
      "logps/chosen": -0.666210949420929,
      "logps/rejected": -0.991015613079071,
      "loss": 0.9303,
      "nll_loss": 0.8248046636581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06658935546875,
      "rewards/margins": 0.03243713453412056,
      "rewards/rejected": -0.09902343899011612,
      "step": 10740
    },
    {
      "epoch": 0.4028404939011823,
      "grad_norm": 1.7165331722812478,
      "learning_rate": 7.715885154726593e-07,
      "log_odds_chosen": 0.40455323457717896,
      "log_odds_ratio": -0.6170898675918579,
      "logits/chosen": -0.792773425579071,
      "logits/rejected": -0.7215820550918579,
      "logps/chosen": -0.734375,
      "logps/rejected": -0.982617199420929,
      "loss": 0.9225,
      "nll_loss": 0.822265625,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07346191257238388,
      "rewards/margins": 0.02485504187643528,
      "rewards/rejected": -0.09819336235523224,
      "step": 10750
    },
    {
      "epoch": 0.40321522924434616,
      "grad_norm": 1.8250876930038846,
      "learning_rate": 7.71229887279699e-07,
      "log_odds_chosen": 0.3954834043979645,
      "log_odds_ratio": -0.6214843988418579,
      "logits/chosen": -0.737988293170929,
      "logits/rejected": -0.674121081829071,
      "logps/chosen": -0.6695312261581421,
      "logps/rejected": -0.89453125,
      "loss": 0.9256,
      "nll_loss": 0.799609363079071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06697998195886612,
      "rewards/margins": 0.02257385291159153,
      "rewards/rejected": -0.08951415866613388,
      "step": 10760
    },
    {
      "epoch": 0.40358996458751006,
      "grad_norm": 1.8284878065044656,
      "learning_rate": 7.708717586849164e-07,
      "log_odds_chosen": 0.518872082233429,
      "log_odds_ratio": -0.5918945074081421,
      "logits/chosen": -0.750781238079071,
      "logits/rejected": -0.63330078125,
      "logps/chosen": -0.676953136920929,
      "logps/rejected": -1.028906226158142,
      "loss": 0.9237,
      "nll_loss": 0.832812488079071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06770019233226776,
      "rewards/margins": 0.03515777736902237,
      "rewards/rejected": -0.10281982272863388,
      "step": 10770
    },
    {
      "epoch": 0.40396469993067396,
      "grad_norm": 1.7506479788157008,
      "learning_rate": 7.705141285294196e-07,
      "log_odds_chosen": 0.44184571504592896,
      "log_odds_ratio": -0.60693359375,
      "logits/chosen": -0.7632812261581421,
      "logits/rejected": -0.666015625,
      "logps/chosen": -0.6595703363418579,
      "logps/rejected": -0.9410156011581421,
      "loss": 0.9054,
      "nll_loss": 0.8169921636581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06597900390625,
      "rewards/margins": 0.02810363844037056,
      "rewards/rejected": -0.09401855617761612,
      "step": 10780
    },
    {
      "epoch": 0.40433943527383787,
      "grad_norm": 1.7402266448948693,
      "learning_rate": 7.701569956580767e-07,
      "log_odds_chosen": 0.35924071073532104,
      "log_odds_ratio": -0.62744140625,
      "logits/chosen": -0.7632812261581421,
      "logits/rejected": -0.653613269329071,
      "logps/chosen": -0.719921886920929,
      "logps/rejected": -0.958203136920929,
      "loss": 0.9358,
      "nll_loss": 0.892578125,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07200928032398224,
      "rewards/margins": 0.02377624437212944,
      "rewards/rejected": -0.09578857570886612,
      "step": 10790
    },
    {
      "epoch": 0.40471417061700177,
      "grad_norm": 1.7071321843843983,
      "learning_rate": 7.69800358919501e-07,
      "log_odds_chosen": 0.5638061761856079,
      "log_odds_ratio": -0.595019519329071,
      "logits/chosen": -0.7381836175918579,
      "logits/rejected": -0.68310546875,
      "logps/chosen": -0.700390636920929,
      "logps/rejected": -1.077539086341858,
      "loss": 0.9298,
      "nll_loss": 0.8935546875,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07001952826976776,
      "rewards/margins": 0.03764801099896431,
      "rewards/rejected": -0.10780028998851776,
      "step": 10800
    },
    {
      "epoch": 0.4050889059601656,
      "grad_norm": 1.7336781280376852,
      "learning_rate": 7.694442171660333e-07,
      "log_odds_chosen": 0.46613770723342896,
      "log_odds_ratio": -0.6024414300918579,
      "logits/chosen": -0.763867199420929,
      "logits/rejected": -0.645312488079071,
      "logps/chosen": -0.674023449420929,
      "logps/rejected": -0.9603515863418579,
      "loss": 0.9245,
      "nll_loss": 0.8197265863418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06741943210363388,
      "rewards/margins": 0.028566742315888405,
      "rewards/rejected": -0.09599609673023224,
      "step": 10810
    },
    {
      "epoch": 0.4054636413033295,
      "grad_norm": 1.7061570782431035,
      "learning_rate": 7.690885692537282e-07,
      "log_odds_chosen": 0.3679565489292145,
      "log_odds_ratio": -0.6512695550918579,
      "logits/chosen": -0.7373046875,
      "logits/rejected": -0.6328125,
      "logps/chosen": -0.701367199420929,
      "logps/rejected": -0.943164050579071,
      "loss": 0.9286,
      "nll_loss": 0.856249988079071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07015381008386612,
      "rewards/margins": 0.024143218994140625,
      "rewards/rejected": -0.09429931640625,
      "step": 10820
    },
    {
      "epoch": 0.4058383766464934,
      "grad_norm": 1.6761196664154367,
      "learning_rate": 7.687334140423383e-07,
      "log_odds_chosen": 0.385009765625,
      "log_odds_ratio": -0.6451171636581421,
      "logits/chosen": -0.8070312738418579,
      "logits/rejected": -0.7090820074081421,
      "logps/chosen": -0.689257800579071,
      "logps/rejected": -0.948046863079071,
      "loss": 0.9043,
      "nll_loss": 0.853710949420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06887207180261612,
      "rewards/margins": 0.02586975134909153,
      "rewards/rejected": -0.09475097805261612,
      "step": 10830
    },
    {
      "epoch": 0.4062131119896573,
      "grad_norm": 1.711129728451547,
      "learning_rate": 7.683787503952985e-07,
      "log_odds_chosen": 0.43310546875,
      "log_odds_ratio": -0.6142578125,
      "logits/chosen": -0.7476562261581421,
      "logits/rejected": -0.6630859375,
      "logps/chosen": -0.7300781011581421,
      "logps/rejected": -1.0236327648162842,
      "loss": 0.9244,
      "nll_loss": 0.930859386920929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07298584282398224,
      "rewards/margins": 0.029320526868104935,
      "rewards/rejected": -0.1021728515625,
      "step": 10840
    },
    {
      "epoch": 0.4065878473328212,
      "grad_norm": 1.6505553681312257,
      "learning_rate": 7.680245771797108e-07,
      "log_odds_chosen": 0.4707275331020355,
      "log_odds_ratio": -0.606640636920929,
      "logits/chosen": -0.7542968988418579,
      "logits/rejected": -0.65478515625,
      "logps/chosen": -0.6890624761581421,
      "logps/rejected": -1.0105469226837158,
      "loss": 0.9037,
      "nll_loss": 0.867382824420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06895752251148224,
      "rewards/margins": 0.032135009765625,
      "rewards/rejected": -0.10114745795726776,
      "step": 10850
    },
    {
      "epoch": 0.40696258267598506,
      "grad_norm": 1.8465157482408154,
      "learning_rate": 7.676708932663293e-07,
      "log_odds_chosen": 0.536999523639679,
      "log_odds_ratio": -0.5918945074081421,
      "logits/chosen": -0.7494140863418579,
      "logits/rejected": -0.6724609136581421,
      "logps/chosen": -0.668749988079071,
      "logps/rejected": -1.0203125476837158,
      "loss": 0.9233,
      "nll_loss": 0.8310546875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06685791164636612,
      "rewards/margins": 0.035167694091796875,
      "rewards/rejected": -0.10206298530101776,
      "step": 10860
    },
    {
      "epoch": 0.40733731801914896,
      "grad_norm": 1.7363363387247759,
      "learning_rate": 7.67317697529545e-07,
      "log_odds_chosen": 0.506884753704071,
      "log_odds_ratio": -0.579882800579071,
      "logits/chosen": -0.748046875,
      "logits/rejected": -0.6255859136581421,
      "logps/chosen": -0.6675781011581421,
      "logps/rejected": -0.9931640625,
      "loss": 0.8987,
      "nll_loss": 0.789843738079071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06678466498851776,
      "rewards/margins": 0.03255004808306694,
      "rewards/rejected": -0.09934081882238388,
      "step": 10870
    },
    {
      "epoch": 0.40771205336231287,
      "grad_norm": 1.8378601530248304,
      "learning_rate": 7.669649888473704e-07,
      "log_odds_chosen": 0.37407225370407104,
      "log_odds_ratio": -0.643261730670929,
      "logits/chosen": -0.7242187261581421,
      "logits/rejected": -0.6202148199081421,
      "logps/chosen": -0.7196289300918579,
      "logps/rejected": -0.958789050579071,
      "loss": 0.8965,
      "nll_loss": 0.8609374761581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07197265326976776,
      "rewards/margins": 0.023886870592832565,
      "rewards/rejected": -0.09587402641773224,
      "step": 10880
    },
    {
      "epoch": 0.40808678870547677,
      "grad_norm": 1.5982439653229779,
      "learning_rate": 7.666127661014253e-07,
      "log_odds_chosen": 0.40521240234375,
      "log_odds_ratio": -0.6229492425918579,
      "logits/chosen": -0.7699218988418579,
      "logits/rejected": -0.703906238079071,
      "logps/chosen": -0.69921875,
      "logps/rejected": -0.948437511920929,
      "loss": 0.9198,
      "nll_loss": 0.845703125,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06989745795726776,
      "rewards/margins": 0.02490692213177681,
      "rewards/rejected": -0.0948486328125,
      "step": 10890
    },
    {
      "epoch": 0.40846152404864067,
      "grad_norm": 1.6742656871097465,
      "learning_rate": 7.662610281769211e-07,
      "log_odds_chosen": 0.3261352479457855,
      "log_odds_ratio": -0.640820324420929,
      "logits/chosen": -0.7611328363418579,
      "logits/rejected": -0.658007800579071,
      "logps/chosen": -0.716113269329071,
      "logps/rejected": -0.910351574420929,
      "loss": 0.9316,
      "nll_loss": 0.814453125,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07159423828125,
      "rewards/margins": 0.01946868933737278,
      "rewards/rejected": -0.091064453125,
      "step": 10900
    },
    {
      "epoch": 0.4088362593918045,
      "grad_norm": 1.7501186766453345,
      "learning_rate": 7.659097739626465e-07,
      "log_odds_chosen": 0.39599609375,
      "log_odds_ratio": -0.63134765625,
      "logits/chosen": -0.778027355670929,
      "logits/rejected": -0.654589831829071,
      "logps/chosen": -0.68212890625,
      "logps/rejected": -0.942187488079071,
      "loss": 0.9265,
      "nll_loss": 0.8765624761581421,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06821288913488388,
      "rewards/margins": 0.02605590783059597,
      "rewards/rejected": -0.09431152045726776,
      "step": 10910
    },
    {
      "epoch": 0.4092109947349684,
      "grad_norm": 1.7451067214523015,
      "learning_rate": 7.655590023509527e-07,
      "log_odds_chosen": 0.657580554485321,
      "log_odds_ratio": -0.539843738079071,
      "logits/chosen": -0.747265636920929,
      "logits/rejected": -0.650097668170929,
      "logps/chosen": -0.744140625,
      "logps/rejected": -1.16796875,
      "loss": 0.9411,
      "nll_loss": 0.899218738079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07442627102136612,
      "rewards/margins": 0.04235992580652237,
      "rewards/rejected": -0.1168212890625,
      "step": 10920
    },
    {
      "epoch": 0.4095857300781323,
      "grad_norm": 1.7073738330411368,
      "learning_rate": 7.652087122377384e-07,
      "log_odds_chosen": 0.6322387456893921,
      "log_odds_ratio": -0.56396484375,
      "logits/chosen": -0.724804699420929,
      "logits/rejected": -0.6060546636581421,
      "logps/chosen": -0.6522461175918579,
      "logps/rejected": -1.0408203601837158,
      "loss": 0.9209,
      "nll_loss": 0.892578125,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06522216647863388,
      "rewards/margins": 0.03889770433306694,
      "rewards/rejected": -0.10415039211511612,
      "step": 10930
    },
    {
      "epoch": 0.4099604654212962,
      "grad_norm": 1.8080699401521085,
      "learning_rate": 7.648589025224355e-07,
      "log_odds_chosen": 0.4336181581020355,
      "log_odds_ratio": -0.6197265386581421,
      "logits/chosen": -0.7789062261581421,
      "logits/rejected": -0.6541992425918579,
      "logps/chosen": -0.742382824420929,
      "logps/rejected": -1.0535156726837158,
      "loss": 0.9555,
      "nll_loss": 0.886523425579071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07424316555261612,
      "rewards/margins": 0.03102874755859375,
      "rewards/rejected": -0.10537109524011612,
      "step": 10940
    },
    {
      "epoch": 0.4103352007644601,
      "grad_norm": 1.9478776554920092,
      "learning_rate": 7.645095721079945e-07,
      "log_odds_chosen": 0.5228637456893921,
      "log_odds_ratio": -0.5858398675918579,
      "logits/chosen": -0.7548828125,
      "logits/rejected": -0.63134765625,
      "logps/chosen": -0.712109386920929,
      "logps/rejected": -1.041601538658142,
      "loss": 0.9323,
      "nll_loss": 0.8314453363418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07121582329273224,
      "rewards/margins": 0.03296966478228569,
      "rewards/rejected": -0.1041259765625,
      "step": 10950
    },
    {
      "epoch": 0.41070993610762396,
      "grad_norm": 2.111957765015368,
      "learning_rate": 7.641607199008701e-07,
      "log_odds_chosen": 0.37608641386032104,
      "log_odds_ratio": -0.6250976324081421,
      "logits/chosen": -0.663867175579071,
      "logits/rejected": -0.618457019329071,
      "logps/chosen": -0.703906238079071,
      "logps/rejected": -0.9322265386581421,
      "loss": 0.9223,
      "nll_loss": 0.8330078125,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07038573920726776,
      "rewards/margins": 0.02280273474752903,
      "rewards/rejected": -0.09317626804113388,
      "step": 10960
    },
    {
      "epoch": 0.41108467145078786,
      "grad_norm": 1.6763279520622252,
      "learning_rate": 7.638123448110066e-07,
      "log_odds_chosen": 0.3846191465854645,
      "log_odds_ratio": -0.6216796636581421,
      "logits/chosen": -0.800000011920929,
      "logits/rejected": -0.699414074420929,
      "logps/chosen": -0.677539050579071,
      "logps/rejected": -0.9306640625,
      "loss": 0.9004,
      "nll_loss": 0.8291015625,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06781005859375,
      "rewards/margins": 0.025337982922792435,
      "rewards/rejected": -0.09311523288488388,
      "step": 10970
    },
    {
      "epoch": 0.41145940679395177,
      "grad_norm": 1.7092476320604748,
      "learning_rate": 7.634644457518242e-07,
      "log_odds_chosen": 0.50341796875,
      "log_odds_ratio": -0.586132824420929,
      "logits/chosen": -0.757617175579071,
      "logits/rejected": -0.669921875,
      "logps/chosen": -0.6851562261581421,
      "logps/rejected": -0.984570324420929,
      "loss": 0.9234,
      "nll_loss": 0.8871093988418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06851806491613388,
      "rewards/margins": 0.02991333045065403,
      "rewards/rejected": -0.09843750298023224,
      "step": 10980
    },
    {
      "epoch": 0.41183414213711567,
      "grad_norm": 1.7639348581645196,
      "learning_rate": 7.631170216402039e-07,
      "log_odds_chosen": 0.526684582233429,
      "log_odds_ratio": -0.570996105670929,
      "logits/chosen": -0.755175769329071,
      "logits/rejected": -0.6279296875,
      "logps/chosen": -0.6795898675918579,
      "logps/rejected": -1.018945336341858,
      "loss": 0.926,
      "nll_loss": 0.8505859375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06793212890625,
      "rewards/margins": 0.033954620361328125,
      "rewards/rejected": -0.1019287109375,
      "step": 10990
    },
    {
      "epoch": 0.41220887748027957,
      "grad_norm": 1.7148832038023307,
      "learning_rate": 7.627700713964739e-07,
      "log_odds_chosen": 0.4753173887729645,
      "log_odds_ratio": -0.612011730670929,
      "logits/chosen": -0.6884765625,
      "logits/rejected": -0.606738269329071,
      "logps/chosen": -0.7109375,
      "logps/rejected": -1.0158202648162842,
      "loss": 0.9208,
      "nll_loss": 0.845898449420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.071044921875,
      "rewards/margins": 0.03054504469037056,
      "rewards/rejected": -0.10161133110523224,
      "step": 11000
    },
    {
      "epoch": 0.41258361282344347,
      "grad_norm": 1.6823320762341336,
      "learning_rate": 7.624235939443953e-07,
      "log_odds_chosen": 0.39989012479782104,
      "log_odds_ratio": -0.6214843988418579,
      "logits/chosen": -0.7484375238418579,
      "logits/rejected": -0.6373046636581421,
      "logps/chosen": -0.70849609375,
      "logps/rejected": -0.9769531488418579,
      "loss": 0.9197,
      "nll_loss": 0.9248046875,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07086181640625,
      "rewards/margins": 0.02682952955365181,
      "rewards/rejected": -0.09765625,
      "step": 11010
    },
    {
      "epoch": 0.4129583481666073,
      "grad_norm": 1.6834815693870524,
      "learning_rate": 7.620775882111482e-07,
      "log_odds_chosen": 0.41606444120407104,
      "log_odds_ratio": -0.6167968511581421,
      "logits/chosen": -0.768750011920929,
      "logits/rejected": -0.6558593511581421,
      "logps/chosen": -0.7123047113418579,
      "logps/rejected": -0.967968761920929,
      "loss": 0.9238,
      "nll_loss": 0.8509765863418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07121582329273224,
      "rewards/margins": 0.02560119703412056,
      "rewards/rejected": -0.09689941257238388,
      "step": 11020
    },
    {
      "epoch": 0.4133330835097712,
      "grad_norm": 1.6664366910219017,
      "learning_rate": 7.617320531273181e-07,
      "log_odds_chosen": 0.49580079317092896,
      "log_odds_ratio": -0.6058593988418579,
      "logits/chosen": -0.745410144329071,
      "logits/rejected": -0.6415039300918579,
      "logps/chosen": -0.7251952886581421,
      "logps/rejected": -1.0576171875,
      "loss": 0.9309,
      "nll_loss": 0.8509765863418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07254638522863388,
      "rewards/margins": 0.03318939357995987,
      "rewards/rejected": -0.10573730617761612,
      "step": 11030
    },
    {
      "epoch": 0.4137078188529351,
      "grad_norm": 1.6667804556686427,
      "learning_rate": 7.613869876268809e-07,
      "log_odds_chosen": 0.4564208984375,
      "log_odds_ratio": -0.6053711175918579,
      "logits/chosen": -0.762890636920929,
      "logits/rejected": -0.7115234136581421,
      "logps/chosen": -0.717968761920929,
      "logps/rejected": -1.0,
      "loss": 0.9171,
      "nll_loss": 0.8941406011581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07177734375,
      "rewards/margins": 0.028211211785674095,
      "rewards/rejected": -0.10004882514476776,
      "step": 11040
    },
    {
      "epoch": 0.414082554196099,
      "grad_norm": 1.803601868651761,
      "learning_rate": 7.610423906471905e-07,
      "log_odds_chosen": 0.5626220703125,
      "log_odds_ratio": -0.5743163824081421,
      "logits/chosen": -0.7466796636581421,
      "logits/rejected": -0.64404296875,
      "logps/chosen": -0.691113293170929,
      "logps/rejected": -1.0460937023162842,
      "loss": 0.9168,
      "nll_loss": 0.888671875,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06910400092601776,
      "rewards/margins": 0.03553314134478569,
      "rewards/rejected": -0.10466308891773224,
      "step": 11050
    },
    {
      "epoch": 0.4144572895392629,
      "grad_norm": 1.64966321355691,
      "learning_rate": 7.606982611289639e-07,
      "log_odds_chosen": 0.38090819120407104,
      "log_odds_ratio": -0.6348632574081421,
      "logits/chosen": -0.739453136920929,
      "logits/rejected": -0.638964831829071,
      "logps/chosen": -0.701855480670929,
      "logps/rejected": -0.955273449420929,
      "loss": 0.9359,
      "nll_loss": 0.8857421875,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.0701904296875,
      "rewards/margins": 0.02529602125287056,
      "rewards/rejected": -0.09547118842601776,
      "step": 11060
    },
    {
      "epoch": 0.41483202488242676,
      "grad_norm": 1.7710760886991983,
      "learning_rate": 7.60354598016268e-07,
      "log_odds_chosen": 0.4047607481479645,
      "log_odds_ratio": -0.64501953125,
      "logits/chosen": -0.777148425579071,
      "logits/rejected": -0.6669921875,
      "logps/chosen": -0.671875,
      "logps/rejected": -0.9306640625,
      "loss": 0.9146,
      "nll_loss": 0.8882812261581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06722412258386612,
      "rewards/margins": 0.025902558118104935,
      "rewards/rejected": -0.09306640923023224,
      "step": 11070
    },
    {
      "epoch": 0.41520676022559067,
      "grad_norm": 1.7488086178537636,
      "learning_rate": 7.600114002565063e-07,
      "log_odds_chosen": 0.48875731229782104,
      "log_odds_ratio": -0.611132800579071,
      "logits/chosen": -0.799609363079071,
      "logits/rejected": -0.6693359613418579,
      "logps/chosen": -0.704882800579071,
      "logps/rejected": -1.0173828601837158,
      "loss": 0.9307,
      "nll_loss": 0.824023425579071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07059326022863388,
      "rewards/margins": 0.031214142218232155,
      "rewards/rejected": -0.10166015475988388,
      "step": 11080
    },
    {
      "epoch": 0.41558149556875457,
      "grad_norm": 1.7468184467338823,
      "learning_rate": 7.596686668004049e-07,
      "log_odds_chosen": 0.397705078125,
      "log_odds_ratio": -0.635546863079071,
      "logits/chosen": -0.7261718511581421,
      "logits/rejected": -0.6045898199081421,
      "logps/chosen": -0.7494140863418579,
      "logps/rejected": -1.0193359851837158,
      "loss": 0.9346,
      "nll_loss": 0.8910156488418579,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07491455227136612,
      "rewards/margins": 0.02699737623333931,
      "rewards/rejected": -0.10195312649011612,
      "step": 11090
    },
    {
      "epoch": 0.41595623091191847,
      "grad_norm": 1.768733466707403,
      "learning_rate": 7.593263966019991e-07,
      "log_odds_chosen": 0.4928832948207855,
      "log_odds_ratio": -0.5987304449081421,
      "logits/chosen": -0.698925793170929,
      "logits/rejected": -0.6494140625,
      "logps/chosen": -0.6817382574081421,
      "logps/rejected": -0.976367175579071,
      "loss": 0.9361,
      "nll_loss": 0.864062488079071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06813964992761612,
      "rewards/margins": 0.0294952392578125,
      "rewards/rejected": -0.09755859524011612,
      "step": 11100
    },
    {
      "epoch": 0.41633096625508237,
      "grad_norm": 1.736760529497024,
      "learning_rate": 7.589845886186201e-07,
      "log_odds_chosen": 0.5287719964981079,
      "log_odds_ratio": -0.559863269329071,
      "logits/chosen": -0.71240234375,
      "logits/rejected": -0.62451171875,
      "logps/chosen": -0.700976550579071,
      "logps/rejected": -1.025976538658142,
      "loss": 0.9026,
      "nll_loss": 0.861328125,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07011719048023224,
      "rewards/margins": 0.03253631666302681,
      "rewards/rejected": -0.10273437201976776,
      "step": 11110
    },
    {
      "epoch": 0.4167057015982462,
      "grad_norm": 3.4932009998511826,
      "learning_rate": 7.586432418108816e-07,
      "log_odds_chosen": 0.4507690370082855,
      "log_odds_ratio": -0.6004883050918579,
      "logits/chosen": -0.7318359613418579,
      "logits/rejected": -0.630078136920929,
      "logps/chosen": -0.6585937738418579,
      "logps/rejected": -0.952929675579071,
      "loss": 0.9127,
      "nll_loss": 0.83984375,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06583251804113388,
      "rewards/margins": 0.02946167066693306,
      "rewards/rejected": -0.09539794921875,
      "step": 11120
    },
    {
      "epoch": 0.4170804369414101,
      "grad_norm": 2.0357532495283266,
      "learning_rate": 7.583023551426664e-07,
      "log_odds_chosen": 0.2879638671875,
      "log_odds_ratio": -0.66943359375,
      "logits/chosen": -0.6781250238418579,
      "logits/rejected": -0.58984375,
      "logps/chosen": -0.740234375,
      "logps/rejected": -0.9175781011581421,
      "loss": 0.9403,
      "nll_loss": 0.8558593988418579,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07403564453125,
      "rewards/margins": 0.01766052283346653,
      "rewards/rejected": -0.0916748046875,
      "step": 11130
    },
    {
      "epoch": 0.417455172284574,
      "grad_norm": 1.8329797076556196,
      "learning_rate": 7.579619275811138e-07,
      "log_odds_chosen": 0.4352050721645355,
      "log_odds_ratio": -0.6390625238418579,
      "logits/chosen": -0.7328125238418579,
      "logits/rejected": -0.6507812738418579,
      "logps/chosen": -0.6781250238418579,
      "logps/rejected": -0.980175793170929,
      "loss": 0.9473,
      "nll_loss": 0.866406261920929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06782226264476776,
      "rewards/margins": 0.03017425537109375,
      "rewards/rejected": -0.09799804538488388,
      "step": 11140
    },
    {
      "epoch": 0.4178299076277379,
      "grad_norm": 1.7287860881429669,
      "learning_rate": 7.576219580966055e-07,
      "log_odds_chosen": 0.48088377714157104,
      "log_odds_ratio": -0.6011718511581421,
      "logits/chosen": -0.75146484375,
      "logits/rejected": -0.6416991949081421,
      "logps/chosen": -0.719531238079071,
      "logps/rejected": -1.03125,
      "loss": 0.9131,
      "nll_loss": 0.839648425579071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07192382961511612,
      "rewards/margins": 0.03111877478659153,
      "rewards/rejected": -0.1031494140625,
      "step": 11150
    },
    {
      "epoch": 0.4182046429709018,
      "grad_norm": 1.8099109163259508,
      "learning_rate": 7.57282445662753e-07,
      "log_odds_chosen": 0.628796398639679,
      "log_odds_ratio": -0.532031238079071,
      "logits/chosen": -0.780468761920929,
      "logits/rejected": -0.635058581829071,
      "logps/chosen": -0.6568359136581421,
      "logps/rejected": -1.0330078601837158,
      "loss": 0.9197,
      "nll_loss": 0.8101562261581421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06566162407398224,
      "rewards/margins": 0.03760375827550888,
      "rewards/rejected": -0.10329589992761612,
      "step": 11160
    },
    {
      "epoch": 0.41857937831406566,
      "grad_norm": 1.67990387754498,
      "learning_rate": 7.569433892563852e-07,
      "log_odds_chosen": 0.551239013671875,
      "log_odds_ratio": -0.5693359375,
      "logits/chosen": -0.7162109613418579,
      "logits/rejected": -0.6109374761581421,
      "logps/chosen": -0.66943359375,
      "logps/rejected": -1.0166015625,
      "loss": 0.9279,
      "nll_loss": 0.8968750238418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06688232719898224,
      "rewards/margins": 0.03473968431353569,
      "rewards/rejected": -0.10166015475988388,
      "step": 11170
    },
    {
      "epoch": 0.41895411365722957,
      "grad_norm": 1.718712820881308,
      "learning_rate": 7.566047878575343e-07,
      "log_odds_chosen": 0.515820324420929,
      "log_odds_ratio": -0.571972668170929,
      "logits/chosen": -0.702929675579071,
      "logits/rejected": -0.6187499761581421,
      "logps/chosen": -0.685742199420929,
      "logps/rejected": -0.9937499761581421,
      "loss": 0.9255,
      "nll_loss": 0.821484386920929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06857910007238388,
      "rewards/margins": 0.0308074951171875,
      "rewards/rejected": -0.09943847358226776,
      "step": 11180
    },
    {
      "epoch": 0.41932884900039347,
      "grad_norm": 1.7556025221350147,
      "learning_rate": 7.562666404494236e-07,
      "log_odds_chosen": 0.3600830137729645,
      "log_odds_ratio": -0.6294921636581421,
      "logits/chosen": -0.6944335699081421,
      "logits/rejected": -0.633593738079071,
      "logps/chosen": -0.698925793170929,
      "logps/rejected": -0.9115234613418579,
      "loss": 0.9143,
      "nll_loss": 0.8707031011581421,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.0699462890625,
      "rewards/margins": 0.021247100085020065,
      "rewards/rejected": -0.09116210788488388,
      "step": 11190
    },
    {
      "epoch": 0.41970358434355737,
      "grad_norm": 1.7852569391173057,
      "learning_rate": 7.559289460184543e-07,
      "log_odds_chosen": 0.39848631620407104,
      "log_odds_ratio": -0.616015613079071,
      "logits/chosen": -0.7193359136581421,
      "logits/rejected": -0.6060546636581421,
      "logps/chosen": -0.7279297113418579,
      "logps/rejected": -0.9742187261581421,
      "loss": 0.9367,
      "nll_loss": 0.916210949420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07288818061351776,
      "rewards/margins": 0.02458648756146431,
      "rewards/rejected": -0.09743652492761612,
      "step": 11200
    },
    {
      "epoch": 0.42007831968672127,
      "grad_norm": 1.7364642094924831,
      "learning_rate": 7.555917035541937e-07,
      "log_odds_chosen": 0.4654907286167145,
      "log_odds_ratio": -0.5912109613418579,
      "logits/chosen": -0.7310546636581421,
      "logits/rejected": -0.6265624761581421,
      "logps/chosen": -0.6826171875,
      "logps/rejected": -0.9755859375,
      "loss": 0.9327,
      "nll_loss": 0.9019531011581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06827392429113388,
      "rewards/margins": 0.02926177904009819,
      "rewards/rejected": -0.09746094048023224,
      "step": 11210
    },
    {
      "epoch": 0.42045305502988517,
      "grad_norm": 1.7317787731458694,
      "learning_rate": 7.552549120493609e-07,
      "log_odds_chosen": 0.547314465045929,
      "log_odds_ratio": -0.579785168170929,
      "logits/chosen": -0.7544921636581421,
      "logits/rejected": -0.6142578125,
      "logps/chosen": -0.677929699420929,
      "logps/rejected": -1.026757836341858,
      "loss": 0.9012,
      "nll_loss": 0.835742175579071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06779785454273224,
      "rewards/margins": 0.03486328199505806,
      "rewards/rejected": -0.10268554836511612,
      "step": 11220
    },
    {
      "epoch": 0.420827790373049,
      "grad_norm": 1.6398804055293474,
      "learning_rate": 7.549185704998158e-07,
      "log_odds_chosen": 0.4226440489292145,
      "log_odds_ratio": -0.65771484375,
      "logits/chosen": -0.666796863079071,
      "logits/rejected": -0.6167968511581421,
      "logps/chosen": -0.7333984375,
      "logps/rejected": -0.984179675579071,
      "loss": 0.9115,
      "nll_loss": 0.8330078125,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07331542670726776,
      "rewards/margins": 0.025139618664979935,
      "rewards/rejected": -0.09846191108226776,
      "step": 11230
    },
    {
      "epoch": 0.4212025257162129,
      "grad_norm": 1.7471470304730545,
      "learning_rate": 7.545826779045449e-07,
      "log_odds_chosen": 0.392822265625,
      "log_odds_ratio": -0.649707019329071,
      "logits/chosen": -0.7109375,
      "logits/rejected": -0.587890625,
      "logps/chosen": -0.702343761920929,
      "logps/rejected": -0.971875011920929,
      "loss": 0.9338,
      "nll_loss": 0.8890625238418579,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07020263373851776,
      "rewards/margins": 0.02700042724609375,
      "rewards/rejected": -0.09733887016773224,
      "step": 11240
    },
    {
      "epoch": 0.4215772610593768,
      "grad_norm": 1.8734966979480865,
      "learning_rate": 7.542472332656506e-07,
      "log_odds_chosen": 0.36790770292282104,
      "log_odds_ratio": -0.614941418170929,
      "logits/chosen": -0.71240234375,
      "logits/rejected": -0.629101574420929,
      "logps/chosen": -0.6708008050918579,
      "logps/rejected": -0.8896484375,
      "loss": 0.9114,
      "nll_loss": 0.829296886920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06704101711511612,
      "rewards/margins": 0.021874237805604935,
      "rewards/rejected": -0.08897705376148224,
      "step": 11250
    },
    {
      "epoch": 0.4219519964025407,
      "grad_norm": 1.8604065602162945,
      "learning_rate": 7.539122355883373e-07,
      "log_odds_chosen": 0.515795886516571,
      "log_odds_ratio": -0.6087890863418579,
      "logits/chosen": -0.7652343511581421,
      "logits/rejected": -0.6810547113418579,
      "logps/chosen": -0.7154296636581421,
      "logps/rejected": -1.0568358898162842,
      "loss": 0.9215,
      "nll_loss": 0.87890625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.071533203125,
      "rewards/margins": 0.03406066820025444,
      "rewards/rejected": -0.10566405951976776,
      "step": 11260
    },
    {
      "epoch": 0.4223267317457046,
      "grad_norm": 1.667110089800858,
      "learning_rate": 7.535776838808995e-07,
      "log_odds_chosen": 0.4514404237270355,
      "log_odds_ratio": -0.6249023675918579,
      "logits/chosen": -0.7044922113418579,
      "logits/rejected": -0.636914074420929,
      "logps/chosen": -0.729296863079071,
      "logps/rejected": -1.008203148841858,
      "loss": 0.9051,
      "nll_loss": 0.9300781488418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07293701171875,
      "rewards/margins": 0.02790679968893528,
      "rewards/rejected": -0.10090331733226776,
      "step": 11270
    },
    {
      "epoch": 0.42270146708886847,
      "grad_norm": 1.7306405054538077,
      "learning_rate": 7.532435771547094e-07,
      "log_odds_chosen": 0.456298828125,
      "log_odds_ratio": -0.595019519329071,
      "logits/chosen": -0.748242199420929,
      "logits/rejected": -0.6429687738418579,
      "logps/chosen": -0.6958984136581421,
      "logps/rejected": -0.972851574420929,
      "loss": 0.9167,
      "nll_loss": 0.869921863079071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.069580078125,
      "rewards/margins": 0.02763671800494194,
      "rewards/rejected": -0.09724120795726776,
      "step": 11280
    },
    {
      "epoch": 0.42307620243203237,
      "grad_norm": 1.7358601658523871,
      "learning_rate": 7.52909914424205e-07,
      "log_odds_chosen": 0.48676759004592896,
      "log_odds_ratio": -0.5926758050918579,
      "logits/chosen": -0.747851550579071,
      "logits/rejected": -0.6581054925918579,
      "logps/chosen": -0.700976550579071,
      "logps/rejected": -1.0076172351837158,
      "loss": 0.9262,
      "nll_loss": 0.8765624761581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07005615532398224,
      "rewards/margins": 0.03067169152200222,
      "rewards/rejected": -0.10073242336511612,
      "step": 11290
    },
    {
      "epoch": 0.42345093777519627,
      "grad_norm": 1.754433007951108,
      "learning_rate": 7.525766947068777e-07,
      "log_odds_chosen": 0.531982421875,
      "log_odds_ratio": -0.58251953125,
      "logits/chosen": -0.7349609136581421,
      "logits/rejected": -0.65087890625,
      "logps/chosen": -0.654101550579071,
      "logps/rejected": -1.003320336341858,
      "loss": 0.9334,
      "nll_loss": 0.834179699420929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06535644829273224,
      "rewards/margins": 0.03496246412396431,
      "rewards/rejected": -0.10032959282398224,
      "step": 11300
    },
    {
      "epoch": 0.42382567311836017,
      "grad_norm": 1.5911894221952927,
      "learning_rate": 7.522439170232598e-07,
      "log_odds_chosen": 0.522778332233429,
      "log_odds_ratio": -0.599804699420929,
      "logits/chosen": -0.680859386920929,
      "logits/rejected": -0.6015625,
      "logps/chosen": -0.698046863079071,
      "logps/rejected": -1.0439453125,
      "loss": 0.9138,
      "nll_loss": 0.902148425579071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06977538764476776,
      "rewards/margins": 0.034575652331113815,
      "rewards/rejected": -0.1043701171875,
      "step": 11310
    },
    {
      "epoch": 0.42420040846152407,
      "grad_norm": 1.7427273152156697,
      "learning_rate": 7.519115803969124e-07,
      "log_odds_chosen": 0.579052746295929,
      "log_odds_ratio": -0.5621093511581421,
      "logits/chosen": -0.763867199420929,
      "logits/rejected": -0.657519519329071,
      "logps/chosen": -0.6993163824081421,
      "logps/rejected": -1.062890648841858,
      "loss": 0.9347,
      "nll_loss": 0.8685547113418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06993408501148224,
      "rewards/margins": 0.03641357272863388,
      "rewards/rejected": -0.10629882663488388,
      "step": 11320
    },
    {
      "epoch": 0.4245751438046879,
      "grad_norm": 1.7170560241341906,
      "learning_rate": 7.515796838544139e-07,
      "log_odds_chosen": 0.48109132051467896,
      "log_odds_ratio": -0.6161133050918579,
      "logits/chosen": -0.716113269329071,
      "logits/rejected": -0.6470702886581421,
      "logps/chosen": -0.68359375,
      "logps/rejected": -0.9996093511581421,
      "loss": 0.9191,
      "nll_loss": 0.8628906011581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06829833984375,
      "rewards/margins": 0.03166351467370987,
      "rewards/rejected": -0.09992675483226776,
      "step": 11330
    },
    {
      "epoch": 0.4249498791478518,
      "grad_norm": 1.8695316543984932,
      "learning_rate": 7.51248226425348e-07,
      "log_odds_chosen": 0.5184081792831421,
      "log_odds_ratio": -0.582324206829071,
      "logits/chosen": -0.74609375,
      "logits/rejected": -0.6533203125,
      "logps/chosen": -0.6348632574081421,
      "logps/rejected": -0.964648425579071,
      "loss": 0.9127,
      "nll_loss": 0.821484386920929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06348876655101776,
      "rewards/margins": 0.03296051174402237,
      "rewards/rejected": -0.09653320163488388,
      "step": 11340
    },
    {
      "epoch": 0.4253246144910157,
      "grad_norm": 1.7822783655592884,
      "learning_rate": 7.509172071422913e-07,
      "log_odds_chosen": 0.376708984375,
      "log_odds_ratio": -0.6351562738418579,
      "logits/chosen": -0.699414074420929,
      "logits/rejected": -0.615234375,
      "logps/chosen": -0.724804699420929,
      "logps/rejected": -0.9765625,
      "loss": 0.9151,
      "nll_loss": 0.840039074420929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.072509765625,
      "rewards/margins": 0.02519073523581028,
      "rewards/rejected": -0.09768066555261612,
      "step": 11350
    },
    {
      "epoch": 0.4256993498341796,
      "grad_norm": 1.8804291662842445,
      "learning_rate": 7.505866250408015e-07,
      "log_odds_chosen": 0.40484619140625,
      "log_odds_ratio": -0.616894543170929,
      "logits/chosen": -0.704296886920929,
      "logits/rejected": -0.63232421875,
      "logps/chosen": -0.6498047113418579,
      "logps/rejected": -0.901171863079071,
      "loss": 0.9403,
      "nll_loss": 0.8734375238418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06500244140625,
      "rewards/margins": 0.025131989270448685,
      "rewards/rejected": -0.09007568657398224,
      "step": 11360
    },
    {
      "epoch": 0.4260740851773435,
      "grad_norm": 1.6371783421735588,
      "learning_rate": 7.50256479159406e-07,
      "log_odds_chosen": 0.34141844511032104,
      "log_odds_ratio": -0.663378894329071,
      "logits/chosen": -0.7294921875,
      "logits/rejected": -0.6441406011581421,
      "logps/chosen": -0.701171875,
      "logps/rejected": -0.9423828125,
      "loss": 0.9169,
      "nll_loss": 0.87109375,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07008056342601776,
      "rewards/margins": 0.024105072021484375,
      "rewards/rejected": -0.09418945014476776,
      "step": 11370
    },
    {
      "epoch": 0.42644882052050737,
      "grad_norm": 1.661254901387014,
      "learning_rate": 7.499267685395902e-07,
      "log_odds_chosen": 0.535888671875,
      "log_odds_ratio": -0.570019543170929,
      "logits/chosen": -0.7119140625,
      "logits/rejected": -0.6590820550918579,
      "logps/chosen": -0.6470702886581421,
      "logps/rejected": -0.958789050579071,
      "loss": 0.9028,
      "nll_loss": 0.776171863079071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06478271633386612,
      "rewards/margins": 0.0311279296875,
      "rewards/rejected": -0.09592285007238388,
      "step": 11380
    },
    {
      "epoch": 0.42682355586367127,
      "grad_norm": 2.371406137505394,
      "learning_rate": 7.495974922257845e-07,
      "log_odds_chosen": 0.34392088651657104,
      "log_odds_ratio": -0.6338866949081421,
      "logits/chosen": -0.8158203363418579,
      "logits/rejected": -0.7250000238418579,
      "logps/chosen": -0.669921875,
      "logps/rejected": -0.892871081829071,
      "loss": 0.9266,
      "nll_loss": 0.841796875,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06697998195886612,
      "rewards/margins": 0.022225189954042435,
      "rewards/rejected": -0.08919677883386612,
      "step": 11390
    },
    {
      "epoch": 0.42719829120683517,
      "grad_norm": 1.844499752834359,
      "learning_rate": 7.492686492653552e-07,
      "log_odds_chosen": 0.5052490234375,
      "log_odds_ratio": -0.5947265625,
      "logits/chosen": -0.7548828125,
      "logits/rejected": -0.6666015386581421,
      "logps/chosen": -0.741406261920929,
      "logps/rejected": -1.074804663658142,
      "loss": 0.9141,
      "nll_loss": 0.8759765625,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.0740966796875,
      "rewards/margins": 0.033277131617069244,
      "rewards/rejected": -0.10747070610523224,
      "step": 11400
    },
    {
      "epoch": 0.42757302654999907,
      "grad_norm": 1.7860469495342584,
      "learning_rate": 7.489402387085902e-07,
      "log_odds_chosen": 0.46989744901657104,
      "log_odds_ratio": -0.631542980670929,
      "logits/chosen": -0.6875976324081421,
      "logits/rejected": -0.6229492425918579,
      "logps/chosen": -0.718066394329071,
      "logps/rejected": -1.042578101158142,
      "loss": 0.9142,
      "nll_loss": 0.9046875238418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07187499850988388,
      "rewards/margins": 0.03242645412683487,
      "rewards/rejected": -0.104248046875,
      "step": 11410
    },
    {
      "epoch": 0.42794776189316297,
      "grad_norm": 1.6385656430254447,
      "learning_rate": 7.486122596086891e-07,
      "log_odds_chosen": 0.4237304627895355,
      "log_odds_ratio": -0.626660168170929,
      "logits/chosen": -0.7998046875,
      "logits/rejected": -0.705371081829071,
      "logps/chosen": -0.727343738079071,
      "logps/rejected": -1.004296898841858,
      "loss": 0.9234,
      "nll_loss": 0.91015625,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07272949069738388,
      "rewards/margins": 0.027738189324736595,
      "rewards/rejected": -0.10051269829273224,
      "step": 11420
    },
    {
      "epoch": 0.42832249723632687,
      "grad_norm": 5.596493528192786,
      "learning_rate": 7.482847110217516e-07,
      "log_odds_chosen": 0.505541980266571,
      "log_odds_ratio": -0.5811523199081421,
      "logits/chosen": -0.7289062738418579,
      "logits/rejected": -0.623730480670929,
      "logps/chosen": -0.6446288824081421,
      "logps/rejected": -0.952929675579071,
      "loss": 0.9027,
      "nll_loss": 0.8169921636581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06440429389476776,
      "rewards/margins": 0.03086242638528347,
      "rewards/rejected": -0.09523925930261612,
      "step": 11430
    },
    {
      "epoch": 0.4286972325794907,
      "grad_norm": 1.8480957971097454,
      "learning_rate": 7.479575920067657e-07,
      "log_odds_chosen": 0.4446044862270355,
      "log_odds_ratio": -0.6156250238418579,
      "logits/chosen": -0.7274414300918579,
      "logits/rejected": -0.649218738079071,
      "logps/chosen": -0.7083984613418579,
      "logps/rejected": -1.0099608898162842,
      "loss": 0.935,
      "nll_loss": 0.8941406011581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07086181640625,
      "rewards/margins": 0.03011169470846653,
      "rewards/rejected": -0.10097656399011612,
      "step": 11440
    },
    {
      "epoch": 0.4290719679226546,
      "grad_norm": 1.751795928443859,
      "learning_rate": 7.476309016255964e-07,
      "log_odds_chosen": 0.49107664823532104,
      "log_odds_ratio": -0.603320300579071,
      "logits/chosen": -0.758984386920929,
      "logits/rejected": -0.6640625,
      "logps/chosen": -0.725781261920929,
      "logps/rejected": -1.0480468273162842,
      "loss": 0.947,
      "nll_loss": 0.879101574420929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07260742038488388,
      "rewards/margins": 0.03219451755285263,
      "rewards/rejected": -0.10480956733226776,
      "step": 11450
    },
    {
      "epoch": 0.4294467032658185,
      "grad_norm": 1.839054667162349,
      "learning_rate": 7.473046389429744e-07,
      "log_odds_chosen": 0.518322765827179,
      "log_odds_ratio": -0.5811523199081421,
      "logits/chosen": -0.7408202886581421,
      "logits/rejected": -0.626757800579071,
      "logps/chosen": -0.7259765863418579,
      "logps/rejected": -1.054296851158142,
      "loss": 0.9158,
      "nll_loss": 0.927929699420929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07261963188648224,
      "rewards/margins": 0.03286285325884819,
      "rewards/rejected": -0.10544433444738388,
      "step": 11460
    },
    {
      "epoch": 0.4298214386089824,
      "grad_norm": 1.8053188723914908,
      "learning_rate": 7.469788030264852e-07,
      "log_odds_chosen": 0.44526368379592896,
      "log_odds_ratio": -0.616406261920929,
      "logits/chosen": -0.76220703125,
      "logits/rejected": -0.6810547113418579,
      "logps/chosen": -0.693554699420929,
      "logps/rejected": -0.9869140386581421,
      "loss": 0.9152,
      "nll_loss": 0.856249988079071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06932373344898224,
      "rewards/margins": 0.02938079833984375,
      "rewards/rejected": -0.09864501655101776,
      "step": 11470
    },
    {
      "epoch": 0.4301961739521463,
      "grad_norm": 1.6204929613692716,
      "learning_rate": 7.466533929465574e-07,
      "log_odds_chosen": 0.575927734375,
      "log_odds_ratio": -0.5752929449081421,
      "logits/chosen": -0.738085925579071,
      "logits/rejected": -0.639843761920929,
      "logps/chosen": -0.667285144329071,
      "logps/rejected": -1.0369141101837158,
      "loss": 0.9186,
      "nll_loss": 0.8369140625,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06671142578125,
      "rewards/margins": 0.03705902025103569,
      "rewards/rejected": -0.10378418117761612,
      "step": 11480
    },
    {
      "epoch": 0.43057090929531017,
      "grad_norm": 1.8309358561683577,
      "learning_rate": 7.463284077764519e-07,
      "log_odds_chosen": 0.3561157286167145,
      "log_odds_ratio": -0.6552734375,
      "logits/chosen": -0.773632824420929,
      "logits/rejected": -0.645703136920929,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -0.958789050579071,
      "loss": 0.9186,
      "nll_loss": 0.8539062738418579,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07158203423023224,
      "rewards/margins": 0.024283599108457565,
      "rewards/rejected": -0.09592285007238388,
      "step": 11490
    },
    {
      "epoch": 0.43094564463847407,
      "grad_norm": 1.7973502572336357,
      "learning_rate": 7.460038465922511e-07,
      "log_odds_chosen": 0.5218139886856079,
      "log_odds_ratio": -0.57080078125,
      "logits/chosen": -0.700390636920929,
      "logits/rejected": -0.6207031011581421,
      "logps/chosen": -0.6650390625,
      "logps/rejected": -0.9791015386581421,
      "loss": 0.9294,
      "nll_loss": 0.824511706829071,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06645508110523224,
      "rewards/margins": 0.03143005445599556,
      "rewards/rejected": -0.09794922173023224,
      "step": 11500
    },
    {
      "epoch": 0.43132037998163797,
      "grad_norm": 1.9264299109718062,
      "learning_rate": 7.456797084728466e-07,
      "log_odds_chosen": 0.49467772245407104,
      "log_odds_ratio": -0.609375,
      "logits/chosen": -0.735058605670929,
      "logits/rejected": -0.649707019329071,
      "logps/chosen": -0.6934570074081421,
      "logps/rejected": -1.0041015148162842,
      "loss": 0.9006,
      "nll_loss": 0.8388671875,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06934814155101776,
      "rewards/margins": 0.03102722205221653,
      "rewards/rejected": -0.10036621242761612,
      "step": 11510
    },
    {
      "epoch": 0.43169511532480187,
      "grad_norm": 1.7024742288595258,
      "learning_rate": 7.453559924999299e-07,
      "log_odds_chosen": 0.58154296875,
      "log_odds_ratio": -0.563183605670929,
      "logits/chosen": -0.7513672113418579,
      "logits/rejected": -0.6507812738418579,
      "logps/chosen": -0.637402355670929,
      "logps/rejected": -1.002539038658142,
      "loss": 0.9104,
      "nll_loss": 0.8177734613418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06374511867761612,
      "rewards/margins": 0.0365447998046875,
      "rewards/rejected": -0.100341796875,
      "step": 11520
    },
    {
      "epoch": 0.43206985066796577,
      "grad_norm": 1.8481252169008489,
      "learning_rate": 7.450326977579804e-07,
      "log_odds_chosen": 0.4598754942417145,
      "log_odds_ratio": -0.60791015625,
      "logits/chosen": -0.737109363079071,
      "logits/rejected": -0.659960925579071,
      "logps/chosen": -0.71484375,
      "logps/rejected": -0.999804675579071,
      "loss": 0.9232,
      "nll_loss": 0.8603515625,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07145996391773224,
      "rewards/margins": 0.02853546105325222,
      "rewards/rejected": -0.10000000149011612,
      "step": 11530
    },
    {
      "epoch": 0.4324445860111296,
      "grad_norm": 1.832755050462322,
      "learning_rate": 7.447098233342549e-07,
      "log_odds_chosen": 0.46049803495407104,
      "log_odds_ratio": -0.5912109613418579,
      "logits/chosen": -0.732226550579071,
      "logits/rejected": -0.640332043170929,
      "logps/chosen": -0.668164074420929,
      "logps/rejected": -0.9644531011581421,
      "loss": 0.9247,
      "nll_loss": 0.9156249761581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06684570014476776,
      "rewards/margins": 0.02964477613568306,
      "rewards/rejected": -0.09648437798023224,
      "step": 11540
    },
    {
      "epoch": 0.4328193213542935,
      "grad_norm": 2.1112527114002786,
      "learning_rate": 7.443873683187767e-07,
      "log_odds_chosen": 0.3845581114292145,
      "log_odds_ratio": -0.6620117425918579,
      "logits/chosen": -0.736328125,
      "logits/rejected": -0.6629883050918579,
      "logps/chosen": -0.684765636920929,
      "logps/rejected": -0.961718738079071,
      "loss": 0.9145,
      "nll_loss": 0.839648425579071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06850586086511612,
      "rewards/margins": 0.027655791491270065,
      "rewards/rejected": -0.09614257514476776,
      "step": 11550
    },
    {
      "epoch": 0.4331940566974574,
      "grad_norm": 1.6232569799035699,
      "learning_rate": 7.440653318043245e-07,
      "log_odds_chosen": 0.40608519315719604,
      "log_odds_ratio": -0.632617175579071,
      "logits/chosen": -0.774609386920929,
      "logits/rejected": -0.671191394329071,
      "logps/chosen": -0.7005859613418579,
      "logps/rejected": -0.966796875,
      "loss": 0.9231,
      "nll_loss": 0.8560546636581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07010497897863388,
      "rewards/margins": 0.026535797864198685,
      "rewards/rejected": -0.09660644829273224,
      "step": 11560
    },
    {
      "epoch": 0.4335687920406213,
      "grad_norm": 1.8060157049560888,
      "learning_rate": 7.437437128864224e-07,
      "log_odds_chosen": 0.4255615174770355,
      "log_odds_ratio": -0.607373058795929,
      "logits/chosen": -0.8121093511581421,
      "logits/rejected": -0.6851562261581421,
      "logps/chosen": -0.7060546875,
      "logps/rejected": -0.987109363079071,
      "loss": 0.9276,
      "nll_loss": 0.896484375,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07059326022863388,
      "rewards/margins": 0.02809295617043972,
      "rewards/rejected": -0.09871826320886612,
      "step": 11570
    },
    {
      "epoch": 0.4339435273837852,
      "grad_norm": 1.8111213471192311,
      "learning_rate": 7.434225106633287e-07,
      "log_odds_chosen": 0.5215209722518921,
      "log_odds_ratio": -0.6000000238418579,
      "logits/chosen": -0.790820300579071,
      "logits/rejected": -0.68701171875,
      "logps/chosen": -0.6435546875,
      "logps/rejected": -0.97998046875,
      "loss": 0.9127,
      "nll_loss": 0.840624988079071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06436767429113388,
      "rewards/margins": 0.03364105150103569,
      "rewards/rejected": -0.09801025688648224,
      "step": 11580
    },
    {
      "epoch": 0.43431826272694907,
      "grad_norm": 1.7207581794399893,
      "learning_rate": 7.431017242360253e-07,
      "log_odds_chosen": 0.332763671875,
      "log_odds_ratio": -0.653613269329071,
      "logits/chosen": -0.734570324420929,
      "logits/rejected": -0.6387695074081421,
      "logps/chosen": -0.687207043170929,
      "logps/rejected": -0.8970702886581421,
      "loss": 0.9251,
      "nll_loss": 0.81640625,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06866455078125,
      "rewards/margins": 0.020964812487363815,
      "rewards/rejected": -0.08967284858226776,
      "step": 11590
    },
    {
      "epoch": 0.43469299807011297,
      "grad_norm": 1.7713835413809282,
      "learning_rate": 7.427813527082074e-07,
      "log_odds_chosen": 0.4171142578125,
      "log_odds_ratio": -0.613476574420929,
      "logits/chosen": -0.741406261920929,
      "logits/rejected": -0.6615234613418579,
      "logps/chosen": -0.686816394329071,
      "logps/rejected": -0.956835925579071,
      "loss": 0.9093,
      "nll_loss": 0.866406261920929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06867675483226776,
      "rewards/margins": 0.02702026441693306,
      "rewards/rejected": -0.09565429389476776,
      "step": 11600
    },
    {
      "epoch": 0.43506773341327687,
      "grad_norm": 1.7566251444067376,
      "learning_rate": 7.424613951862727e-07,
      "log_odds_chosen": 0.4626403748989105,
      "log_odds_ratio": -0.640576183795929,
      "logits/chosen": -0.733593761920929,
      "logits/rejected": -0.6163085699081421,
      "logps/chosen": -0.699999988079071,
      "logps/rejected": -0.9800781011581421,
      "loss": 0.9045,
      "nll_loss": 0.8369140625,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06998290866613388,
      "rewards/margins": 0.02801818773150444,
      "rewards/rejected": -0.09801025688648224,
      "step": 11610
    },
    {
      "epoch": 0.43544246875644077,
      "grad_norm": 1.8359572932744297,
      "learning_rate": 7.42141850779311e-07,
      "log_odds_chosen": 0.47242432832717896,
      "log_odds_ratio": -0.5814453363418579,
      "logits/chosen": -0.7669922113418579,
      "logits/rejected": -0.673144519329071,
      "logps/chosen": -0.662304699420929,
      "logps/rejected": -0.9593750238418579,
      "loss": 0.9128,
      "nll_loss": 0.820507824420929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06623534858226776,
      "rewards/margins": 0.02966613695025444,
      "rewards/rejected": -0.09589843451976776,
      "step": 11620
    },
    {
      "epoch": 0.43581720409960467,
      "grad_norm": 1.7911445064749016,
      "learning_rate": 7.418227185990941e-07,
      "log_odds_chosen": 0.570361316204071,
      "log_odds_ratio": -0.5611327886581421,
      "logits/chosen": -0.770703136920929,
      "logits/rejected": -0.64794921875,
      "logps/chosen": -0.6773437261581421,
      "logps/rejected": -1.0486328601837158,
      "loss": 0.9187,
      "nll_loss": 0.8578125238418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0677490234375,
      "rewards/margins": 0.03709258884191513,
      "rewards/rejected": -0.10490722954273224,
      "step": 11630
    },
    {
      "epoch": 0.4361919394427685,
      "grad_norm": 1.8221734157376148,
      "learning_rate": 7.415039977600647e-07,
      "log_odds_chosen": 0.4067626893520355,
      "log_odds_ratio": -0.612500011920929,
      "logits/chosen": -0.7470703125,
      "logits/rejected": -0.653124988079071,
      "logps/chosen": -0.69189453125,
      "logps/rejected": -0.9336913824081421,
      "loss": 0.9053,
      "nll_loss": 0.7998046875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0692138671875,
      "rewards/margins": 0.02418365515768528,
      "rewards/rejected": -0.09345702826976776,
      "step": 11640
    },
    {
      "epoch": 0.4365666747859324,
      "grad_norm": 2.0128265118813085,
      "learning_rate": 7.411856873793271e-07,
      "log_odds_chosen": 0.3412841856479645,
      "log_odds_ratio": -0.6314452886581421,
      "logits/chosen": -0.7308593988418579,
      "logits/rejected": -0.6529296636581421,
      "logps/chosen": -0.6812499761581421,
      "logps/rejected": -0.8853515386581421,
      "loss": 0.9105,
      "nll_loss": 0.826953113079071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06813964992761612,
      "rewards/margins": 0.02035369910299778,
      "rewards/rejected": -0.08847656100988388,
      "step": 11650
    },
    {
      "epoch": 0.4369414101290963,
      "grad_norm": 1.714106506981703,
      "learning_rate": 7.408677865766361e-07,
      "log_odds_chosen": 0.38951414823532104,
      "log_odds_ratio": -0.6371093988418579,
      "logits/chosen": -0.690722644329071,
      "logits/rejected": -0.584277331829071,
      "logps/chosen": -0.6919921636581421,
      "logps/rejected": -0.9505859613418579,
      "loss": 0.9147,
      "nll_loss": 0.9154297113418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06922607123851776,
      "rewards/margins": 0.02577972412109375,
      "rewards/rejected": -0.094970703125,
      "step": 11660
    },
    {
      "epoch": 0.4373161454722602,
      "grad_norm": 1.9344658708385392,
      "learning_rate": 7.405502944743868e-07,
      "log_odds_chosen": 0.5467529296875,
      "log_odds_ratio": -0.569042980670929,
      "logits/chosen": -0.716113269329071,
      "logits/rejected": -0.6319335699081421,
      "logps/chosen": -0.676953136920929,
      "logps/rejected": -1.0236327648162842,
      "loss": 0.9249,
      "nll_loss": 0.8802734613418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06773681938648224,
      "rewards/margins": 0.03461456298828125,
      "rewards/rejected": -0.1024169921875,
      "step": 11670
    },
    {
      "epoch": 0.4376908808154241,
      "grad_norm": 1.7092977725299163,
      "learning_rate": 7.402332101976052e-07,
      "log_odds_chosen": 0.31702882051467896,
      "log_odds_ratio": -0.6689453125,
      "logits/chosen": -0.708789050579071,
      "logits/rejected": -0.6380859613418579,
      "logps/chosen": -0.685546875,
      "logps/rejected": -0.9019531011581421,
      "loss": 0.916,
      "nll_loss": 0.908007800579071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06856689602136612,
      "rewards/margins": 0.021582793444395065,
      "rewards/rejected": -0.09017334133386612,
      "step": 11680
    },
    {
      "epoch": 0.438065616158588,
      "grad_norm": 1.671134943621317,
      "learning_rate": 7.399165328739372e-07,
      "log_odds_chosen": 0.551806628704071,
      "log_odds_ratio": -0.5791015625,
      "logits/chosen": -0.718945324420929,
      "logits/rejected": -0.652539074420929,
      "logps/chosen": -0.6463867425918579,
      "logps/rejected": -0.997265636920929,
      "loss": 0.915,
      "nll_loss": 0.818164050579071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06467285007238388,
      "rewards/margins": 0.03512115404009819,
      "rewards/rejected": -0.09965820610523224,
      "step": 11690
    },
    {
      "epoch": 0.43844035150175187,
      "grad_norm": 2.1362020073660277,
      "learning_rate": 7.396002616336387e-07,
      "log_odds_chosen": 0.582080066204071,
      "log_odds_ratio": -0.581250011920929,
      "logits/chosen": -0.708789050579071,
      "logits/rejected": -0.6114257574081421,
      "logps/chosen": -0.708203136920929,
      "logps/rejected": -1.0968749523162842,
      "loss": 0.9328,
      "nll_loss": 0.872265636920929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07086181640625,
      "rewards/margins": 0.03878479078412056,
      "rewards/rejected": -0.10965576022863388,
      "step": 11700
    },
    {
      "epoch": 0.43881508684491577,
      "grad_norm": 1.752422420183024,
      "learning_rate": 7.392843956095663e-07,
      "log_odds_chosen": 0.7308349609375,
      "log_odds_ratio": -0.534375011920929,
      "logits/chosen": -0.815234363079071,
      "logits/rejected": -0.6366211175918579,
      "logps/chosen": -0.6595703363418579,
      "logps/rejected": -1.146093726158142,
      "loss": 0.9236,
      "nll_loss": 0.8363281488418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06594238430261612,
      "rewards/margins": 0.04851074144244194,
      "rewards/rejected": -0.11455078423023224,
      "step": 11710
    },
    {
      "epoch": 0.43918982218807967,
      "grad_norm": 1.7513741264044977,
      "learning_rate": 7.389689339371664e-07,
      "log_odds_chosen": 0.40089112520217896,
      "log_odds_ratio": -0.630664050579071,
      "logits/chosen": -0.7181640863418579,
      "logits/rejected": -0.6451171636581421,
      "logps/chosen": -0.6866210699081421,
      "logps/rejected": -0.94921875,
      "loss": 0.9004,
      "nll_loss": 0.797070324420929,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.06865234673023224,
      "rewards/margins": 0.026226043701171875,
      "rewards/rejected": -0.09495849907398224,
      "step": 11720
    },
    {
      "epoch": 0.43956455753124357,
      "grad_norm": 1.7059963005091892,
      "learning_rate": 7.386538757544653e-07,
      "log_odds_chosen": 0.637805163860321,
      "log_odds_ratio": -0.545117199420929,
      "logits/chosen": -0.733105480670929,
      "logits/rejected": -0.633984386920929,
      "logps/chosen": -0.6650390625,
      "logps/rejected": -1.072265625,
      "loss": 0.9125,
      "nll_loss": 0.8896484375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06646728515625,
      "rewards/margins": 0.04077911376953125,
      "rewards/rejected": -0.10727538913488388,
      "step": 11730
    },
    {
      "epoch": 0.4399392928744075,
      "grad_norm": 1.833012059194792,
      "learning_rate": 7.3833922020206e-07,
      "log_odds_chosen": 0.633471667766571,
      "log_odds_ratio": -0.5687500238418579,
      "logits/chosen": -0.720703125,
      "logits/rejected": -0.60498046875,
      "logps/chosen": -0.7066406011581421,
      "logps/rejected": -1.129296898841858,
      "loss": 0.906,
      "nll_loss": 0.812695324420929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07061767578125,
      "rewards/margins": 0.04226074367761612,
      "rewards/rejected": -0.11281738430261612,
      "step": 11740
    },
    {
      "epoch": 0.4403140282175713,
      "grad_norm": 1.9938723407066976,
      "learning_rate": 7.38024966423108e-07,
      "log_odds_chosen": 0.551403820514679,
      "log_odds_ratio": -0.6060546636581421,
      "logits/chosen": -0.75,
      "logits/rejected": -0.6502929925918579,
      "logps/chosen": -0.705273449420929,
      "logps/rejected": -1.0525391101837158,
      "loss": 0.9062,
      "nll_loss": 0.8460937738418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.070556640625,
      "rewards/margins": 0.03468017652630806,
      "rewards/rejected": -0.10526122897863388,
      "step": 11750
    },
    {
      "epoch": 0.4406887635607352,
      "grad_norm": 1.6480557702058816,
      "learning_rate": 7.377111135633174e-07,
      "log_odds_chosen": 0.4205078184604645,
      "log_odds_ratio": -0.617480456829071,
      "logits/chosen": -0.763671875,
      "logits/rejected": -0.68505859375,
      "logps/chosen": -0.6982421875,
      "logps/rejected": -0.9375,
      "loss": 0.9281,
      "nll_loss": 0.8150390386581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06984863430261612,
      "rewards/margins": 0.02398834191262722,
      "rewards/rejected": -0.09382323920726776,
      "step": 11760
    },
    {
      "epoch": 0.4410634989038991,
      "grad_norm": 1.7444823987259208,
      "learning_rate": 7.373976607709372e-07,
      "log_odds_chosen": 0.43629151582717896,
      "log_odds_ratio": -0.6211913824081421,
      "logits/chosen": -0.767773449420929,
      "logits/rejected": -0.6446288824081421,
      "logps/chosen": -0.6998046636581421,
      "logps/rejected": -1.0037109851837158,
      "loss": 0.899,
      "nll_loss": 0.8160156011581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07001952826976776,
      "rewards/margins": 0.03041534498333931,
      "rewards/rejected": -0.10042724758386612,
      "step": 11770
    },
    {
      "epoch": 0.441438234247063,
      "grad_norm": 1.7773349691081954,
      "learning_rate": 7.370846071967476e-07,
      "log_odds_chosen": 0.5493530035018921,
      "log_odds_ratio": -0.55712890625,
      "logits/chosen": -0.734375,
      "logits/rejected": -0.6172851324081421,
      "logps/chosen": -0.67529296875,
      "logps/rejected": -1.010156273841858,
      "loss": 0.9223,
      "nll_loss": 0.8232421875,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06755371391773224,
      "rewards/margins": 0.03350830078125,
      "rewards/rejected": -0.10104980319738388,
      "step": 11780
    },
    {
      "epoch": 0.4418129695902269,
      "grad_norm": 1.6842320713818948,
      "learning_rate": 7.367719519940501e-07,
      "log_odds_chosen": 0.6343017816543579,
      "log_odds_ratio": -0.539355456829071,
      "logits/chosen": -0.700976550579071,
      "logits/rejected": -0.572949230670929,
      "logps/chosen": -0.6787109375,
      "logps/rejected": -1.101953148841858,
      "loss": 0.9197,
      "nll_loss": 0.865234375,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06793212890625,
      "rewards/margins": 0.0422821044921875,
      "rewards/rejected": -0.11013183742761612,
      "step": 11790
    },
    {
      "epoch": 0.44218770493339077,
      "grad_norm": 1.7726805709996796,
      "learning_rate": 7.364596943186587e-07,
      "log_odds_chosen": 0.2435302734375,
      "log_odds_ratio": -0.674511730670929,
      "logits/chosen": -0.73046875,
      "logits/rejected": -0.63330078125,
      "logps/chosen": -0.7035156488418579,
      "logps/rejected": -0.8734375238418579,
      "loss": 0.934,
      "nll_loss": 0.918749988079071,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07033691555261612,
      "rewards/margins": 0.01701507531106472,
      "rewards/rejected": -0.08739013969898224,
      "step": 11800
    },
    {
      "epoch": 0.44256244027655467,
      "grad_norm": 1.7361369604467454,
      "learning_rate": 7.36147833328889e-07,
      "log_odds_chosen": 0.42634278535842896,
      "log_odds_ratio": -0.6175781488418579,
      "logits/chosen": -0.7705078125,
      "logits/rejected": -0.688281238079071,
      "logps/chosen": -0.676074206829071,
      "logps/rejected": -0.9486328363418579,
      "loss": 0.9181,
      "nll_loss": 0.8734375238418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06763915717601776,
      "rewards/margins": 0.02721710130572319,
      "rewards/rejected": -0.09483642876148224,
      "step": 11810
    },
    {
      "epoch": 0.44293717561971857,
      "grad_norm": 1.6342671894470326,
      "learning_rate": 7.358363681855503e-07,
      "log_odds_chosen": 0.4116577208042145,
      "log_odds_ratio": -0.6258789300918579,
      "logits/chosen": -0.7535156011581421,
      "logits/rejected": -0.62158203125,
      "logps/chosen": -0.7000976800918579,
      "logps/rejected": -0.9775390625,
      "loss": 0.926,
      "nll_loss": 0.8681640625,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07003173977136612,
      "rewards/margins": 0.02761535719037056,
      "rewards/rejected": -0.09768066555261612,
      "step": 11820
    },
    {
      "epoch": 0.44331191096288247,
      "grad_norm": 1.8002050785436698,
      "learning_rate": 7.355252980519345e-07,
      "log_odds_chosen": 0.3738647401332855,
      "log_odds_ratio": -0.6366211175918579,
      "logits/chosen": -0.771679699420929,
      "logits/rejected": -0.654492199420929,
      "logps/chosen": -0.7225586175918579,
      "logps/rejected": -0.9658203125,
      "loss": 0.9397,
      "nll_loss": 0.8646484613418579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07224120944738388,
      "rewards/margins": 0.0242156982421875,
      "rewards/rejected": -0.09644775092601776,
      "step": 11830
    },
    {
      "epoch": 0.4436866463060464,
      "grad_norm": 1.7430441031499382,
      "learning_rate": 7.352146220938078e-07,
      "log_odds_chosen": 0.3446289002895355,
      "log_odds_ratio": -0.649121105670929,
      "logits/chosen": -0.682421863079071,
      "logits/rejected": -0.63232421875,
      "logps/chosen": -0.7181640863418579,
      "logps/rejected": -0.961718738079071,
      "loss": 0.9052,
      "nll_loss": 0.853320300579071,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07171630859375,
      "rewards/margins": 0.02442474290728569,
      "rewards/rejected": -0.09616699069738388,
      "step": 11840
    },
    {
      "epoch": 0.4440613816492102,
      "grad_norm": 1.8179581485036729,
      "learning_rate": 7.349043394794005e-07,
      "log_odds_chosen": 0.6130737066268921,
      "log_odds_ratio": -0.561230480670929,
      "logits/chosen": -0.7503906488418579,
      "logits/rejected": -0.640332043170929,
      "logps/chosen": -0.6639648675918579,
      "logps/rejected": -1.0285155773162842,
      "loss": 0.9138,
      "nll_loss": 0.848437488079071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06641845405101776,
      "rewards/margins": 0.03639679029583931,
      "rewards/rejected": -0.10283203423023224,
      "step": 11850
    },
    {
      "epoch": 0.4444361169923741,
      "grad_norm": 1.7811231654499478,
      "learning_rate": 7.345944493793987e-07,
      "log_odds_chosen": 0.6668456792831421,
      "log_odds_ratio": -0.5611327886581421,
      "logits/chosen": -0.7591797113418579,
      "logits/rejected": -0.6767578125,
      "logps/chosen": -0.649609386920929,
      "logps/rejected": -1.0802733898162842,
      "loss": 0.8949,
      "nll_loss": 0.7904297113418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06495361030101776,
      "rewards/margins": 0.04300536960363388,
      "rewards/rejected": -0.10798339545726776,
      "step": 11860
    },
    {
      "epoch": 0.444810852335538,
      "grad_norm": 1.8296286271460396,
      "learning_rate": 7.342849509669337e-07,
      "log_odds_chosen": 0.51361083984375,
      "log_odds_ratio": -0.6048828363418579,
      "logits/chosen": -0.73046875,
      "logits/rejected": -0.618359386920929,
      "logps/chosen": -0.6634765863418579,
      "logps/rejected": -1.0009765625,
      "loss": 0.9178,
      "nll_loss": 0.862109363079071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06636963039636612,
      "rewards/margins": 0.03377380222082138,
      "rewards/rejected": -0.10014648735523224,
      "step": 11870
    },
    {
      "epoch": 0.4451855876787019,
      "grad_norm": 1.8545001031158135,
      "learning_rate": 7.339758434175737e-07,
      "log_odds_chosen": 0.537243664264679,
      "log_odds_ratio": -0.5882812738418579,
      "logits/chosen": -0.7593749761581421,
      "logits/rejected": -0.680371105670929,
      "logps/chosen": -0.691210925579071,
      "logps/rejected": -1.0314452648162842,
      "loss": 0.8885,
      "nll_loss": 0.830273449420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06914062798023224,
      "rewards/margins": 0.034029386937618256,
      "rewards/rejected": -0.1031494140625,
      "step": 11880
    },
    {
      "epoch": 0.4455603230218658,
      "grad_norm": 1.7554959310487568,
      "learning_rate": 7.336671259093143e-07,
      "log_odds_chosen": 0.5464843511581421,
      "log_odds_ratio": -0.5826171636581421,
      "logits/chosen": -0.744921863079071,
      "logits/rejected": -0.670605480670929,
      "logps/chosen": -0.6885741949081421,
      "logps/rejected": -1.0166015625,
      "loss": 0.9146,
      "nll_loss": 0.865039050579071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06879882514476776,
      "rewards/margins": 0.03287658840417862,
      "rewards/rejected": -0.10166015475988388,
      "step": 11890
    },
    {
      "epoch": 0.4459350583650297,
      "grad_norm": 1.749867694512219,
      "learning_rate": 7.33358797622569e-07,
      "log_odds_chosen": 0.46192628145217896,
      "log_odds_ratio": -0.602734386920929,
      "logits/chosen": -0.766796886920929,
      "logits/rejected": -0.6504882574081421,
      "logps/chosen": -0.6841796636581421,
      "logps/rejected": -0.9644531011581421,
      "loss": 0.9081,
      "nll_loss": 0.8326171636581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.068359375,
      "rewards/margins": 0.02805938757956028,
      "rewards/rejected": -0.09639892727136612,
      "step": 11900
    },
    {
      "epoch": 0.44630979370819357,
      "grad_norm": 1.7708806786661049,
      "learning_rate": 7.330508577401606e-07,
      "log_odds_chosen": 0.570483386516571,
      "log_odds_ratio": -0.570507824420929,
      "logits/chosen": -0.776171863079071,
      "logits/rejected": -0.632617175579071,
      "logps/chosen": -0.678027331829071,
      "logps/rejected": -1.065820336341858,
      "loss": 0.926,
      "nll_loss": 0.842968761920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06781005859375,
      "rewards/margins": 0.03869171068072319,
      "rewards/rejected": -0.10646972805261612,
      "step": 11910
    },
    {
      "epoch": 0.44668452905135747,
      "grad_norm": 1.6672551863142122,
      "learning_rate": 7.327433054473117e-07,
      "log_odds_chosen": 0.551953136920929,
      "log_odds_ratio": -0.5796874761581421,
      "logits/chosen": -0.69921875,
      "logits/rejected": -0.6073242425918579,
      "logps/chosen": -0.6939452886581421,
      "logps/rejected": -1.05078125,
      "loss": 0.8994,
      "nll_loss": 0.866992175579071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06949462741613388,
      "rewards/margins": 0.03563995286822319,
      "rewards/rejected": -0.10517577826976776,
      "step": 11920
    },
    {
      "epoch": 0.44705926439452137,
      "grad_norm": 1.7036895289443987,
      "learning_rate": 7.324361399316357e-07,
      "log_odds_chosen": 0.6391845941543579,
      "log_odds_ratio": -0.564746081829071,
      "logits/chosen": -0.750195324420929,
      "logits/rejected": -0.63037109375,
      "logps/chosen": -0.6856445074081421,
      "logps/rejected": -1.1134765148162842,
      "loss": 0.9071,
      "nll_loss": 0.8628906011581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06856689602136612,
      "rewards/margins": 0.0428924560546875,
      "rewards/rejected": -0.11141357570886612,
      "step": 11930
    },
    {
      "epoch": 0.4474339997376853,
      "grad_norm": 2.006403702681125,
      "learning_rate": 7.321293603831281e-07,
      "log_odds_chosen": 0.4027954041957855,
      "log_odds_ratio": -0.6329101324081421,
      "logits/chosen": -0.7466796636581421,
      "logits/rejected": -0.650585949420929,
      "logps/chosen": -0.6900390386581421,
      "logps/rejected": -0.9697265625,
      "loss": 0.9121,
      "nll_loss": 0.853515625,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.06903076171875,
      "rewards/margins": 0.027982329949736595,
      "rewards/rejected": -0.096923828125,
      "step": 11940
    },
    {
      "epoch": 0.4478087350808492,
      "grad_norm": 1.6435692296119102,
      "learning_rate": 7.318229659941572e-07,
      "log_odds_chosen": 0.4843383729457855,
      "log_odds_ratio": -0.61767578125,
      "logits/chosen": -0.7220703363418579,
      "logits/rejected": -0.63720703125,
      "logps/chosen": -0.7046874761581421,
      "logps/rejected": -1.018945336341858,
      "loss": 0.926,
      "nll_loss": 0.883984386920929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07050780951976776,
      "rewards/margins": 0.03142852708697319,
      "rewards/rejected": -0.10197754204273224,
      "step": 11950
    },
    {
      "epoch": 0.448183470424013,
      "grad_norm": 1.881244492008659,
      "learning_rate": 7.315169559594551e-07,
      "log_odds_chosen": 0.48426514863967896,
      "log_odds_ratio": -0.59326171875,
      "logits/chosen": -0.7591797113418579,
      "logits/rejected": -0.657421886920929,
      "logps/chosen": -0.721875011920929,
      "logps/rejected": -1.0646483898162842,
      "loss": 0.8912,
      "nll_loss": 0.8394531011581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.0721435546875,
      "rewards/margins": 0.03436126559972763,
      "rewards/rejected": -0.10646972805261612,
      "step": 11960
    },
    {
      "epoch": 0.4485582057671769,
      "grad_norm": 1.7286153013382692,
      "learning_rate": 7.31211329476109e-07,
      "log_odds_chosen": 0.5782226324081421,
      "log_odds_ratio": -0.5826171636581421,
      "logits/chosen": -0.784960925579071,
      "logits/rejected": -0.662792980670929,
      "logps/chosen": -0.65966796875,
      "logps/rejected": -1.040624976158142,
      "loss": 0.9248,
      "nll_loss": 0.8150390386581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06601562350988388,
      "rewards/margins": 0.03813476487994194,
      "rewards/rejected": -0.10405273735523224,
      "step": 11970
    },
    {
      "epoch": 0.4489329411103408,
      "grad_norm": 1.757537981006684,
      "learning_rate": 7.309060857435526e-07,
      "log_odds_chosen": 0.47987061738967896,
      "log_odds_ratio": -0.5826171636581421,
      "logits/chosen": -0.719531238079071,
      "logits/rejected": -0.610156238079071,
      "logps/chosen": -0.671093761920929,
      "logps/rejected": -0.9966796636581421,
      "loss": 0.9036,
      "nll_loss": 0.9037109613418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06705322116613388,
      "rewards/margins": 0.03254394605755806,
      "rewards/rejected": -0.09958495944738388,
      "step": 11980
    },
    {
      "epoch": 0.4493076764535047,
      "grad_norm": 1.807299623805114,
      "learning_rate": 7.30601223963557e-07,
      "log_odds_chosen": 0.6166137456893921,
      "log_odds_ratio": -0.559863269329071,
      "logits/chosen": -0.6973632574081421,
      "logits/rejected": -0.5966796875,
      "logps/chosen": -0.6583007574081421,
      "logps/rejected": -1.0490233898162842,
      "loss": 0.9095,
      "nll_loss": 0.79931640625,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.0657958984375,
      "rewards/margins": 0.03911743313074112,
      "rewards/rejected": -0.10495605319738388,
      "step": 11990
    },
    {
      "epoch": 0.4496824117966686,
      "grad_norm": 2.099060647306707,
      "learning_rate": 7.302967433402214e-07,
      "log_odds_chosen": 0.6213012933731079,
      "log_odds_ratio": -0.5439453125,
      "logits/chosen": -0.7583984136581421,
      "logits/rejected": -0.6142578125,
      "logps/chosen": -0.6607421636581421,
      "logps/rejected": -1.062109351158142,
      "loss": 0.8987,
      "nll_loss": 0.8560546636581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06614990532398224,
      "rewards/margins": 0.04019317775964737,
      "rewards/rejected": -0.10629882663488388,
      "step": 12000
    },
    {
      "epoch": 0.45005714713983247,
      "grad_norm": 1.73514596336096,
      "learning_rate": 7.299926430799657e-07,
      "log_odds_chosen": 0.643176257610321,
      "log_odds_ratio": -0.54833984375,
      "logits/chosen": -0.7372070550918579,
      "logits/rejected": -0.6214843988418579,
      "logps/chosen": -0.680468738079071,
      "logps/rejected": -1.111718773841858,
      "loss": 0.9119,
      "nll_loss": 0.8783203363418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06800536811351776,
      "rewards/margins": 0.04313812404870987,
      "rewards/rejected": -0.11118163913488388,
      "step": 12010
    },
    {
      "epoch": 0.45043188248299637,
      "grad_norm": 1.8386884284819032,
      "learning_rate": 7.296889223915205e-07,
      "log_odds_chosen": 0.455810546875,
      "log_odds_ratio": -0.623242199420929,
      "logits/chosen": -0.7691406011581421,
      "logits/rejected": -0.6512695550918579,
      "logps/chosen": -0.703417956829071,
      "logps/rejected": -1.022070288658142,
      "loss": 0.9095,
      "nll_loss": 0.824999988079071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07033691555261612,
      "rewards/margins": 0.03179015964269638,
      "rewards/rejected": -0.10218505561351776,
      "step": 12020
    },
    {
      "epoch": 0.45080661782616027,
      "grad_norm": 1.9202806761313265,
      "learning_rate": 7.293855804859192e-07,
      "log_odds_chosen": 0.6503661870956421,
      "log_odds_ratio": -0.557324230670929,
      "logits/chosen": -0.7662109136581421,
      "logits/rejected": -0.6439453363418579,
      "logps/chosen": -0.699023425579071,
      "logps/rejected": -1.115820288658142,
      "loss": 0.9129,
      "nll_loss": 0.830859363079071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06988525390625,
      "rewards/margins": 0.04171142727136612,
      "rewards/rejected": -0.11164550483226776,
      "step": 12030
    },
    {
      "epoch": 0.4511813531693242,
      "grad_norm": 1.7172056939710376,
      "learning_rate": 7.290826165764892e-07,
      "log_odds_chosen": 0.4920410215854645,
      "log_odds_ratio": -0.599902331829071,
      "logits/chosen": -0.758984386920929,
      "logits/rejected": -0.6444336175918579,
      "logps/chosen": -0.6976562738418579,
      "logps/rejected": -1.029296875,
      "loss": 0.9188,
      "nll_loss": 0.8466796875,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06972656399011612,
      "rewards/margins": 0.03313903883099556,
      "rewards/rejected": -0.10288085788488388,
      "step": 12040
    },
    {
      "epoch": 0.4515560885124881,
      "grad_norm": 1.7029578590378698,
      "learning_rate": 7.28780029878843e-07,
      "log_odds_chosen": 0.5387817621231079,
      "log_odds_ratio": -0.5697265863418579,
      "logits/chosen": -0.7212890386581421,
      "logits/rejected": -0.609570324420929,
      "logps/chosen": -0.63525390625,
      "logps/rejected": -0.9716796875,
      "loss": 0.9289,
      "nll_loss": 0.862109363079071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06354980170726776,
      "rewards/margins": 0.03359375149011612,
      "rewards/rejected": -0.09715576469898224,
      "step": 12050
    },
    {
      "epoch": 0.4519308238556519,
      "grad_norm": 1.7762598218514771,
      "learning_rate": 7.284778196108706e-07,
      "log_odds_chosen": 0.5853515863418579,
      "log_odds_ratio": -0.56396484375,
      "logits/chosen": -0.7269531488418579,
      "logits/rejected": -0.658984363079071,
      "logps/chosen": -0.68017578125,
      "logps/rejected": -1.039648413658142,
      "loss": 0.9155,
      "nll_loss": 0.862500011920929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06801757961511612,
      "rewards/margins": 0.03592529147863388,
      "rewards/rejected": -0.10397949069738388,
      "step": 12060
    },
    {
      "epoch": 0.4523055591988158,
      "grad_norm": 1.9157786502420728,
      "learning_rate": 7.281759849927299e-07,
      "log_odds_chosen": 0.37077635526657104,
      "log_odds_ratio": -0.6275390386581421,
      "logits/chosen": -0.706347644329071,
      "logits/rejected": -0.606738269329071,
      "logps/chosen": -0.735546886920929,
      "logps/rejected": -0.964648425579071,
      "loss": 0.913,
      "nll_loss": 0.9007812738418579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.073486328125,
      "rewards/margins": 0.02286071702837944,
      "rewards/rejected": -0.09641113132238388,
      "step": 12070
    },
    {
      "epoch": 0.4526802945419797,
      "grad_norm": 2.0106471317910364,
      "learning_rate": 7.278745252468389e-07,
      "log_odds_chosen": 0.48077392578125,
      "log_odds_ratio": -0.638476550579071,
      "logits/chosen": -0.7445312738418579,
      "logits/rejected": -0.693164050579071,
      "logps/chosen": -0.7132812738418579,
      "logps/rejected": -1.059179663658142,
      "loss": 0.9116,
      "nll_loss": 0.84375,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07135009765625,
      "rewards/margins": 0.03465576097369194,
      "rewards/rejected": -0.10599365085363388,
      "step": 12080
    },
    {
      "epoch": 0.4530550298851436,
      "grad_norm": 1.7913210397980635,
      "learning_rate": 7.275734395978672e-07,
      "log_odds_chosen": 0.3414550721645355,
      "log_odds_ratio": -0.662890613079071,
      "logits/chosen": -0.7103515863418579,
      "logits/rejected": -0.6392577886581421,
      "logps/chosen": -0.720898449420929,
      "logps/rejected": -0.973925769329071,
      "loss": 0.9165,
      "nll_loss": 0.893359363079071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07209472358226776,
      "rewards/margins": 0.02530365064740181,
      "rewards/rejected": -0.097412109375,
      "step": 12090
    },
    {
      "epoch": 0.4534297652283075,
      "grad_norm": 1.7267410821877966,
      "learning_rate": 7.272727272727272e-07,
      "log_odds_chosen": 0.48798829317092896,
      "log_odds_ratio": -0.599609375,
      "logits/chosen": -0.7701171636581421,
      "logits/rejected": -0.64306640625,
      "logps/chosen": -0.6753906011581421,
      "logps/rejected": -0.9892578125,
      "loss": 0.9138,
      "nll_loss": 0.8257812261581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06760253757238388,
      "rewards/margins": 0.03142700344324112,
      "rewards/rejected": -0.09897460788488388,
      "step": 12100
    },
    {
      "epoch": 0.4538045005714714,
      "grad_norm": 1.8432318864506645,
      "learning_rate": 7.269723875005668e-07,
      "log_odds_chosen": 0.50518798828125,
      "log_odds_ratio": -0.5687500238418579,
      "logits/chosen": -0.7349609136581421,
      "logits/rejected": -0.6348632574081421,
      "logps/chosen": -0.687792956829071,
      "logps/rejected": -0.984570324420929,
      "loss": 0.9027,
      "nll_loss": 0.8423827886581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06877441704273224,
      "rewards/margins": 0.02978820726275444,
      "rewards/rejected": -0.0985107421875,
      "step": 12110
    },
    {
      "epoch": 0.45417923591463527,
      "grad_norm": 1.819031767490864,
      "learning_rate": 7.266724195127595e-07,
      "log_odds_chosen": 0.449249267578125,
      "log_odds_ratio": -0.6029297113418579,
      "logits/chosen": -0.760937511920929,
      "logits/rejected": -0.6483398675918579,
      "logps/chosen": -0.658886730670929,
      "logps/rejected": -0.9541015625,
      "loss": 0.9286,
      "nll_loss": 0.806445300579071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06586913764476776,
      "rewards/margins": 0.029615020379424095,
      "rewards/rejected": -0.09544678032398224,
      "step": 12120
    },
    {
      "epoch": 0.45455397125779917,
      "grad_norm": 1.8519428884612863,
      "learning_rate": 7.26372822542898e-07,
      "log_odds_chosen": 0.39849853515625,
      "log_odds_ratio": -0.6356445550918579,
      "logits/chosen": -0.757617175579071,
      "logits/rejected": -0.6583007574081421,
      "logps/chosen": -0.672167956829071,
      "logps/rejected": -0.936718761920929,
      "loss": 0.8988,
      "nll_loss": 0.8290039300918579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06721191108226776,
      "rewards/margins": 0.0264434814453125,
      "rewards/rejected": -0.09365234524011612,
      "step": 12130
    },
    {
      "epoch": 0.4549287066009631,
      "grad_norm": 1.924386417330333,
      "learning_rate": 7.260735958267845e-07,
      "log_odds_chosen": 0.45146483182907104,
      "log_odds_ratio": -0.593457043170929,
      "logits/chosen": -0.7544921636581421,
      "logits/rejected": -0.6329101324081421,
      "logps/chosen": -0.7041015625,
      "logps/rejected": -0.9730468988418579,
      "loss": 0.9388,
      "nll_loss": 0.7950195074081421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07039795070886612,
      "rewards/margins": 0.026868438348174095,
      "rewards/rejected": -0.09731445461511612,
      "step": 12140
    },
    {
      "epoch": 0.455303441944127,
      "grad_norm": 1.876670282876624,
      "learning_rate": 7.257747386024231e-07,
      "log_odds_chosen": 0.569042980670929,
      "log_odds_ratio": -0.574511706829071,
      "logits/chosen": -0.736132800579071,
      "logits/rejected": -0.63330078125,
      "logps/chosen": -0.677539050579071,
      "logps/rejected": -1.0185546875,
      "loss": 0.9255,
      "nll_loss": 0.8374999761581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06777343899011612,
      "rewards/margins": 0.0340423583984375,
      "rewards/rejected": -0.10178222507238388,
      "step": 12150
    },
    {
      "epoch": 0.4556781772872909,
      "grad_norm": 1.684988728351077,
      "learning_rate": 7.254762501100117e-07,
      "log_odds_chosen": 0.5340820550918579,
      "log_odds_ratio": -0.556445300579071,
      "logits/chosen": -0.7689453363418579,
      "logits/rejected": -0.6397460699081421,
      "logps/chosen": -0.67724609375,
      "logps/rejected": -1.009765625,
      "loss": 0.907,
      "nll_loss": 0.845898449420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06770019233226776,
      "rewards/margins": 0.03328552097082138,
      "rewards/rejected": -0.10097656399011612,
      "step": 12160
    },
    {
      "epoch": 0.4560529126304547,
      "grad_norm": 1.8892920392887995,
      "learning_rate": 7.251781295919335e-07,
      "log_odds_chosen": 0.5813232660293579,
      "log_odds_ratio": -0.574999988079071,
      "logits/chosen": -0.7457031011581421,
      "logits/rejected": -0.6405273675918579,
      "logps/chosen": -0.663867175579071,
      "logps/rejected": -1.007421851158142,
      "loss": 0.8948,
      "nll_loss": 0.804492175579071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06638183444738388,
      "rewards/margins": 0.034423828125,
      "rewards/rejected": -0.10078124701976776,
      "step": 12170
    },
    {
      "epoch": 0.4564276479736186,
      "grad_norm": 1.8060788825225136,
      "learning_rate": 7.248803762927498e-07,
      "log_odds_chosen": 0.4081664979457855,
      "log_odds_ratio": -0.6036132574081421,
      "logits/chosen": -0.7613281011581421,
      "logits/rejected": -0.6978515386581421,
      "logps/chosen": -0.7210937738418579,
      "logps/rejected": -0.9808593988418579,
      "loss": 0.9211,
      "nll_loss": 0.8335937261581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07210693508386612,
      "rewards/margins": 0.02596435509622097,
      "rewards/rejected": -0.09809570014476776,
      "step": 12180
    },
    {
      "epoch": 0.4568023833167825,
      "grad_norm": 2.0106791941063373,
      "learning_rate": 7.245829894591907e-07,
      "log_odds_chosen": 0.28289794921875,
      "log_odds_ratio": -0.65771484375,
      "logits/chosen": -0.719921886920929,
      "logits/rejected": -0.6558593511581421,
      "logps/chosen": -0.6748046875,
      "logps/rejected": -0.859179675579071,
      "loss": 0.9042,
      "nll_loss": 0.8363281488418579,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06751708686351776,
      "rewards/margins": 0.01849365234375,
      "rewards/rejected": -0.08602295070886612,
      "step": 12190
    },
    {
      "epoch": 0.4571771186599464,
      "grad_norm": 1.8201027445214661,
      "learning_rate": 7.242859683401482e-07,
      "log_odds_chosen": 0.4569091796875,
      "log_odds_ratio": -0.5970703363418579,
      "logits/chosen": -0.782031238079071,
      "logits/rejected": -0.6527343988418579,
      "logps/chosen": -0.7046874761581421,
      "logps/rejected": -0.9791015386581421,
      "loss": 0.8909,
      "nll_loss": 0.810546875,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07053222507238388,
      "rewards/margins": 0.027391815558075905,
      "rewards/rejected": -0.09793701022863388,
      "step": 12200
    },
    {
      "epoch": 0.4575518540031103,
      "grad_norm": 2.0268148799375316,
      "learning_rate": 7.239893121866677e-07,
      "log_odds_chosen": 0.40941160917282104,
      "log_odds_ratio": -0.61572265625,
      "logits/chosen": -0.706835925579071,
      "logits/rejected": -0.6197265386581421,
      "logps/chosen": -0.642773449420929,
      "logps/rejected": -0.9056640863418579,
      "loss": 0.9229,
      "nll_loss": 0.8460937738418579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06428222358226776,
      "rewards/margins": 0.02627258375287056,
      "rewards/rejected": -0.090576171875,
      "step": 12210
    },
    {
      "epoch": 0.45792658934627417,
      "grad_norm": 1.960512070359854,
      "learning_rate": 7.236930202519399e-07,
      "log_odds_chosen": 0.663867175579071,
      "log_odds_ratio": -0.53173828125,
      "logits/chosen": -0.7191406488418579,
      "logits/rejected": -0.60888671875,
      "logps/chosen": -0.677734375,
      "logps/rejected": -1.079687476158142,
      "loss": 0.9326,
      "nll_loss": 0.8759765625,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06770019233226776,
      "rewards/margins": 0.04023437574505806,
      "rewards/rejected": -0.10800781100988388,
      "step": 12220
    },
    {
      "epoch": 0.45830132468943807,
      "grad_norm": 1.8699986821759138,
      "learning_rate": 7.233970917912936e-07,
      "log_odds_chosen": 0.6252075433731079,
      "log_odds_ratio": -0.56396484375,
      "logits/chosen": -0.6832031011581421,
      "logits/rejected": -0.5824218988418579,
      "logps/chosen": -0.697460949420929,
      "logps/rejected": -1.108007788658142,
      "loss": 0.9055,
      "nll_loss": 0.8599609136581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06977538764476776,
      "rewards/margins": 0.04105377197265625,
      "rewards/rejected": -0.11088867485523224,
      "step": 12230
    },
    {
      "epoch": 0.458676060032602,
      "grad_norm": 2.002565300691114,
      "learning_rate": 7.231015260621871e-07,
      "log_odds_chosen": 0.4617919921875,
      "log_odds_ratio": -0.611132800579071,
      "logits/chosen": -0.70849609375,
      "logits/rejected": -0.6348632574081421,
      "logps/chosen": -0.6947265863418579,
      "logps/rejected": -0.972851574420929,
      "loss": 0.9131,
      "nll_loss": 0.816210925579071,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06949462741613388,
      "rewards/margins": 0.027796173468232155,
      "rewards/rejected": -0.09732665866613388,
      "step": 12240
    },
    {
      "epoch": 0.4590507953757659,
      "grad_norm": 1.7986583553100657,
      "learning_rate": 7.228063223242011e-07,
      "log_odds_chosen": 0.581469714641571,
      "log_odds_ratio": -0.55322265625,
      "logits/chosen": -0.7845703363418579,
      "logits/rejected": -0.686328113079071,
      "logps/chosen": -0.671093761920929,
      "logps/rejected": -1.038476586341858,
      "loss": 0.893,
      "nll_loss": 0.8128906488418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06715087592601776,
      "rewards/margins": 0.03670806810259819,
      "rewards/rejected": -0.10380859673023224,
      "step": 12250
    },
    {
      "epoch": 0.4594255307189298,
      "grad_norm": 1.6724075419712896,
      "learning_rate": 7.225114798390295e-07,
      "log_odds_chosen": 0.5145263671875,
      "log_odds_ratio": -0.5713866949081421,
      "logits/chosen": -0.7914062738418579,
      "logits/rejected": -0.6714843511581421,
      "logps/chosen": -0.64404296875,
      "logps/rejected": -0.9384765625,
      "loss": 0.8965,
      "nll_loss": 0.800585925579071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06437988579273224,
      "rewards/margins": 0.02946624718606472,
      "rewards/rejected": -0.09389648586511612,
      "step": 12260
    },
    {
      "epoch": 0.4598002660620936,
      "grad_norm": 1.721421378731988,
      "learning_rate": 7.222169978704737e-07,
      "log_odds_chosen": 0.286376953125,
      "log_odds_ratio": -0.683300793170929,
      "logits/chosen": -0.7178710699081421,
      "logits/rejected": -0.624218761920929,
      "logps/chosen": -0.7535156011581421,
      "logps/rejected": -0.948046863079071,
      "loss": 0.914,
      "nll_loss": 0.8636718988418579,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07528076320886612,
      "rewards/margins": 0.0194854736328125,
      "rewards/rejected": -0.09470214694738388,
      "step": 12270
    },
    {
      "epoch": 0.4601750014052575,
      "grad_norm": 1.6727616949336093,
      "learning_rate": 7.219228756844335e-07,
      "log_odds_chosen": 0.3019653260707855,
      "log_odds_ratio": -0.6729491949081421,
      "logits/chosen": -0.749804675579071,
      "logits/rejected": -0.6548827886581421,
      "logps/chosen": -0.712109386920929,
      "logps/rejected": -0.927929699420929,
      "loss": 0.9183,
      "nll_loss": 0.888671875,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07119140774011612,
      "rewards/margins": 0.021529387682676315,
      "rewards/rejected": -0.09272460639476776,
      "step": 12280
    },
    {
      "epoch": 0.4605497367484214,
      "grad_norm": 1.757878603375529,
      "learning_rate": 7.216291125488994e-07,
      "log_odds_chosen": 0.3138183653354645,
      "log_odds_ratio": -0.653613269329071,
      "logits/chosen": -0.7787109613418579,
      "logits/rejected": -0.659960925579071,
      "logps/chosen": -0.682910144329071,
      "logps/rejected": -0.886914074420929,
      "loss": 0.9096,
      "nll_loss": 0.806835949420929,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06827392429113388,
      "rewards/margins": 0.020395660772919655,
      "rewards/rejected": -0.08876953274011612,
      "step": 12290
    },
    {
      "epoch": 0.4609244720915853,
      "grad_norm": 1.7405144404147432,
      "learning_rate": 7.213357077339458e-07,
      "log_odds_chosen": 0.5154784917831421,
      "log_odds_ratio": -0.588671863079071,
      "logits/chosen": -0.8070312738418579,
      "logits/rejected": -0.6943359375,
      "logps/chosen": -0.6869140863418579,
      "logps/rejected": -1.0048828125,
      "loss": 0.8936,
      "nll_loss": 0.83984375,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06870117038488388,
      "rewards/margins": 0.03185577318072319,
      "rewards/rejected": -0.10051269829273224,
      "step": 12300
    },
    {
      "epoch": 0.4612992074347492,
      "grad_norm": 1.963642015356952,
      "learning_rate": 7.210426605117224e-07,
      "log_odds_chosen": 0.609997570514679,
      "log_odds_ratio": -0.552929699420929,
      "logits/chosen": -0.746289074420929,
      "logits/rejected": -0.6499999761581421,
      "logps/chosen": -0.626757800579071,
      "logps/rejected": -0.9800781011581421,
      "loss": 0.9105,
      "nll_loss": 0.8466796875,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06263427436351776,
      "rewards/margins": 0.03534088283777237,
      "rewards/rejected": -0.09794922173023224,
      "step": 12310
    },
    {
      "epoch": 0.4616739427779131,
      "grad_norm": 1.8175067919694248,
      "learning_rate": 7.207499701564471e-07,
      "log_odds_chosen": 0.5956786870956421,
      "log_odds_ratio": -0.5589843988418579,
      "logits/chosen": -0.729785144329071,
      "logits/rejected": -0.6522461175918579,
      "logps/chosen": -0.6375976800918579,
      "logps/rejected": -1.0177733898162842,
      "loss": 0.9122,
      "nll_loss": 0.828125,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.063720703125,
      "rewards/margins": 0.038069915026426315,
      "rewards/rejected": -0.10187987983226776,
      "step": 12320
    },
    {
      "epoch": 0.46204867812107697,
      "grad_norm": 1.8954157191312544,
      "learning_rate": 7.204576359443989e-07,
      "log_odds_chosen": 0.509448230266571,
      "log_odds_ratio": -0.59228515625,
      "logits/chosen": -0.754199206829071,
      "logits/rejected": -0.6522461175918579,
      "logps/chosen": -0.6954101324081421,
      "logps/rejected": -1.0246093273162842,
      "loss": 0.9254,
      "nll_loss": 0.8521484136581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06955566257238388,
      "rewards/margins": 0.03299865871667862,
      "rewards/rejected": -0.10255126655101776,
      "step": 12330
    },
    {
      "epoch": 0.4624234134642409,
      "grad_norm": 1.751446632094498,
      "learning_rate": 7.201656571539094e-07,
      "log_odds_chosen": 0.460693359375,
      "log_odds_ratio": -0.6083984375,
      "logits/chosen": -0.717089831829071,
      "logits/rejected": -0.6532226800918579,
      "logps/chosen": -0.6527343988418579,
      "logps/rejected": -0.927929699420929,
      "loss": 0.8769,
      "nll_loss": 0.782031238079071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06525878608226776,
      "rewards/margins": 0.027501678094267845,
      "rewards/rejected": -0.09270019829273224,
      "step": 12340
    },
    {
      "epoch": 0.4627981488074048,
      "grad_norm": 1.8221764071829674,
      "learning_rate": 7.19874033065356e-07,
      "log_odds_chosen": 0.5782715082168579,
      "log_odds_ratio": -0.56787109375,
      "logits/chosen": -0.7103515863418579,
      "logits/rejected": -0.6415039300918579,
      "logps/chosen": -0.6982421875,
      "logps/rejected": -1.0339844226837158,
      "loss": 0.9041,
      "nll_loss": 0.838085949420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06989745795726776,
      "rewards/margins": 0.03350677341222763,
      "rewards/rejected": -0.10341797024011612,
      "step": 12350
    },
    {
      "epoch": 0.4631728841505687,
      "grad_norm": 1.6911286535935455,
      "learning_rate": 7.195827629611545e-07,
      "log_odds_chosen": 0.44197386503219604,
      "log_odds_ratio": -0.618359386920929,
      "logits/chosen": -0.7066406011581421,
      "logits/rejected": -0.6436523199081421,
      "logps/chosen": -0.682324230670929,
      "logps/rejected": -0.953906238079071,
      "loss": 0.9111,
      "nll_loss": 0.837890625,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06824950873851776,
      "rewards/margins": 0.02721862867474556,
      "rewards/rejected": -0.09544678032398224,
      "step": 12360
    },
    {
      "epoch": 0.4635476194937326,
      "grad_norm": 1.8453746801917394,
      "learning_rate": 7.19291846125751e-07,
      "log_odds_chosen": 0.48359376192092896,
      "log_odds_ratio": -0.603710949420929,
      "logits/chosen": -0.6683593988418579,
      "logits/rejected": -0.598828136920929,
      "logps/chosen": -0.675976574420929,
      "logps/rejected": -0.9583984613418579,
      "loss": 0.9151,
      "nll_loss": 0.845898449420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06766357272863388,
      "rewards/margins": 0.02822265587747097,
      "rewards/rejected": -0.09589843451976776,
      "step": 12370
    },
    {
      "epoch": 0.4639223548368964,
      "grad_norm": 1.8170530212655052,
      "learning_rate": 7.190012818456154e-07,
      "log_odds_chosen": 0.4704956114292145,
      "log_odds_ratio": -0.6055663824081421,
      "logits/chosen": -0.740429699420929,
      "logits/rejected": -0.6324218511581421,
      "logps/chosen": -0.7544921636581421,
      "logps/rejected": -1.066796898841858,
      "loss": 0.9028,
      "nll_loss": 0.869335949420929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07546386867761612,
      "rewards/margins": 0.03126830980181694,
      "rewards/rejected": -0.10671386867761612,
      "step": 12380
    },
    {
      "epoch": 0.4642970901800603,
      "grad_norm": 1.8387634312267764,
      "learning_rate": 7.187110694092334e-07,
      "log_odds_chosen": 0.4258789122104645,
      "log_odds_ratio": -0.6117187738418579,
      "logits/chosen": -0.749218761920929,
      "logits/rejected": -0.646777331829071,
      "logps/chosen": -0.7523437738418579,
      "logps/rejected": -1.043554663658142,
      "loss": 0.9044,
      "nll_loss": 0.854296863079071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07513427734375,
      "rewards/margins": 0.029126739129424095,
      "rewards/rejected": -0.10432128608226776,
      "step": 12390
    },
    {
      "epoch": 0.4646718255232242,
      "grad_norm": 1.8333953419075224,
      "learning_rate": 7.184212081070996e-07,
      "log_odds_chosen": 0.4822631776332855,
      "log_odds_ratio": -0.59326171875,
      "logits/chosen": -0.747265636920929,
      "logits/rejected": -0.6436523199081421,
      "logps/chosen": -0.669140636920929,
      "logps/rejected": -0.970898449420929,
      "loss": 0.9061,
      "nll_loss": 0.795117199420929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06691894680261612,
      "rewards/margins": 0.030182648450136185,
      "rewards/rejected": -0.09703369438648224,
      "step": 12400
    },
    {
      "epoch": 0.4650465608663881,
      "grad_norm": 1.878724003449855,
      "learning_rate": 7.181316972317097e-07,
      "log_odds_chosen": 0.51739501953125,
      "log_odds_ratio": -0.5733398199081421,
      "logits/chosen": -0.702343761920929,
      "logits/rejected": -0.586621105670929,
      "logps/chosen": -0.674511730670929,
      "logps/rejected": -1.010351538658142,
      "loss": 0.9005,
      "nll_loss": 0.818359375,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06743164360523224,
      "rewards/margins": 0.03354644775390625,
      "rewards/rejected": -0.10097656399011612,
      "step": 12410
    },
    {
      "epoch": 0.465421296209552,
      "grad_norm": 1.8740911741271735,
      "learning_rate": 7.17842536077554e-07,
      "log_odds_chosen": 0.52532958984375,
      "log_odds_ratio": -0.5819336175918579,
      "logits/chosen": -0.7392578125,
      "logits/rejected": -0.6502929925918579,
      "logps/chosen": -0.64599609375,
      "logps/rejected": -0.953320324420929,
      "loss": 0.9046,
      "nll_loss": 0.8394531011581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06464843451976776,
      "rewards/margins": 0.03076476976275444,
      "rewards/rejected": -0.09534911811351776,
      "step": 12420
    },
    {
      "epoch": 0.4657960315527159,
      "grad_norm": 1.6145577471439534,
      "learning_rate": 7.175537239411094e-07,
      "log_odds_chosen": 0.50555419921875,
      "log_odds_ratio": -0.59130859375,
      "logits/chosen": -0.7564452886581421,
      "logits/rejected": -0.65234375,
      "logps/chosen": -0.685546875,
      "logps/rejected": -0.9853515625,
      "loss": 0.8937,
      "nll_loss": 0.862109363079071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06853027641773224,
      "rewards/margins": 0.029994964599609375,
      "rewards/rejected": -0.09848632663488388,
      "step": 12430
    },
    {
      "epoch": 0.4661707668958798,
      "grad_norm": 1.9068300980978,
      "learning_rate": 7.172652601208325e-07,
      "log_odds_chosen": 0.5633789300918579,
      "log_odds_ratio": -0.580761730670929,
      "logits/chosen": -0.7154296636581421,
      "logits/rejected": -0.60302734375,
      "logps/chosen": -0.6761718988418579,
      "logps/rejected": -1.0597655773162842,
      "loss": 0.9084,
      "nll_loss": 0.8783203363418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06752929836511612,
      "rewards/margins": 0.03841857984662056,
      "rewards/rejected": -0.10611572116613388,
      "step": 12440
    },
    {
      "epoch": 0.4665455022390437,
      "grad_norm": 2.013620533901831,
      "learning_rate": 7.169771439171534e-07,
      "log_odds_chosen": 0.4790893495082855,
      "log_odds_ratio": -0.5962890386581421,
      "logits/chosen": -0.782031238079071,
      "logits/rejected": -0.637402355670929,
      "logps/chosen": -0.6958984136581421,
      "logps/rejected": -1.010156273841858,
      "loss": 0.9125,
      "nll_loss": 0.8355468511581421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06953124701976776,
      "rewards/margins": 0.03140563890337944,
      "rewards/rejected": -0.10090331733226776,
      "step": 12450
    },
    {
      "epoch": 0.4669202375822076,
      "grad_norm": 1.836964834227377,
      "learning_rate": 7.166893746324661e-07,
      "log_odds_chosen": 0.45960694551467896,
      "log_odds_ratio": -0.5948241949081421,
      "logits/chosen": -0.7269531488418579,
      "logits/rejected": -0.589062511920929,
      "logps/chosen": -0.720898449420929,
      "logps/rejected": -1.0320312976837158,
      "loss": 0.8955,
      "nll_loss": 0.8763672113418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07210693508386612,
      "rewards/margins": 0.03101348876953125,
      "rewards/rejected": -0.10305175930261612,
      "step": 12460
    },
    {
      "epoch": 0.4672949729253715,
      "grad_norm": 2.0348817533177463,
      "learning_rate": 7.164019515711245e-07,
      "log_odds_chosen": 0.47600096464157104,
      "log_odds_ratio": -0.576464831829071,
      "logits/chosen": -0.779980480670929,
      "logits/rejected": -0.6832031011581421,
      "logps/chosen": -0.692187488079071,
      "logps/rejected": -0.9800781011581421,
      "loss": 0.9024,
      "nll_loss": 0.799023449420929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06925048679113388,
      "rewards/margins": 0.02871399000287056,
      "rewards/rejected": -0.09794922173023224,
      "step": 12470
    },
    {
      "epoch": 0.4676697082685353,
      "grad_norm": 1.9789238509599776,
      "learning_rate": 7.161148740394328e-07,
      "log_odds_chosen": 0.52545166015625,
      "log_odds_ratio": -0.5677734613418579,
      "logits/chosen": -0.725781261920929,
      "logits/rejected": -0.650390625,
      "logps/chosen": -0.6981445550918579,
      "logps/rejected": -1.0400390625,
      "loss": 0.8966,
      "nll_loss": 0.8472656011581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06981201469898224,
      "rewards/margins": 0.03432006761431694,
      "rewards/rejected": -0.10410156100988388,
      "step": 12480
    },
    {
      "epoch": 0.4680444436116992,
      "grad_norm": 2.2866171375768385,
      "learning_rate": 7.158281413456402e-07,
      "log_odds_chosen": 0.4609008729457855,
      "log_odds_ratio": -0.6166015863418579,
      "logits/chosen": -0.7554687261581421,
      "logits/rejected": -0.658496081829071,
      "logps/chosen": -0.69140625,
      "logps/rejected": -0.9775390625,
      "loss": 0.9147,
      "nll_loss": 0.816699206829071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.069091796875,
      "rewards/margins": 0.02858123742043972,
      "rewards/rejected": -0.09776611626148224,
      "step": 12490
    },
    {
      "epoch": 0.4684191789548631,
      "grad_norm": 1.7094716579199811,
      "learning_rate": 7.155417527999326e-07,
      "log_odds_chosen": 0.501513659954071,
      "log_odds_ratio": -0.600292980670929,
      "logits/chosen": -0.7464843988418579,
      "logits/rejected": -0.624316394329071,
      "logps/chosen": -0.688671886920929,
      "logps/rejected": -1.004296898841858,
      "loss": 0.9054,
      "nll_loss": 0.885546863079071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06883545219898224,
      "rewards/margins": 0.031577300280332565,
      "rewards/rejected": -0.10039062798023224,
      "step": 12500
    },
    {
      "epoch": 0.468793914298027,
      "grad_norm": 1.8730106607553516,
      "learning_rate": 7.152557077144268e-07,
      "log_odds_chosen": 0.526379406452179,
      "log_odds_ratio": -0.584667980670929,
      "logits/chosen": -0.7291015386581421,
      "logits/rejected": -0.625683605670929,
      "logps/chosen": -0.7012695074081421,
      "logps/rejected": -1.0390625,
      "loss": 0.9045,
      "nll_loss": 0.8192383050918579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07012939453125,
      "rewards/margins": 0.03389587253332138,
      "rewards/rejected": -0.10396728664636612,
      "step": 12510
    },
    {
      "epoch": 0.4691686496411909,
      "grad_norm": 1.8991084171156207,
      "learning_rate": 7.149700054031623e-07,
      "log_odds_chosen": 0.4475952088832855,
      "log_odds_ratio": -0.6102539300918579,
      "logits/chosen": -0.768359363079071,
      "logits/rejected": -0.692187488079071,
      "logps/chosen": -0.705859363079071,
      "logps/rejected": -0.991015613079071,
      "loss": 0.9138,
      "nll_loss": 0.8736327886581421,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07059326022863388,
      "rewards/margins": 0.028466034680604935,
      "rewards/rejected": -0.09904785454273224,
      "step": 12520
    },
    {
      "epoch": 0.4695433849843548,
      "grad_norm": 1.8282733415737957,
      "learning_rate": 7.146846451820958e-07,
      "log_odds_chosen": 0.4583740234375,
      "log_odds_ratio": -0.60888671875,
      "logits/chosen": -0.752148449420929,
      "logits/rejected": -0.6732422113418579,
      "logps/chosen": -0.686816394329071,
      "logps/rejected": -0.986132800579071,
      "loss": 0.9106,
      "nll_loss": 0.8447265625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06857910007238388,
      "rewards/margins": 0.030037689954042435,
      "rewards/rejected": -0.09865722805261612,
      "step": 12530
    },
    {
      "epoch": 0.4699181203275187,
      "grad_norm": 2.0323998277656106,
      "learning_rate": 7.14399626369093e-07,
      "log_odds_chosen": 0.37858885526657104,
      "log_odds_ratio": -0.6636718511581421,
      "logits/chosen": -0.7886718511581421,
      "logits/rejected": -0.7313476800918579,
      "logps/chosen": -0.683300793170929,
      "logps/rejected": -0.947460949420929,
      "loss": 0.9049,
      "nll_loss": 0.8228515386581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06833495944738388,
      "rewards/margins": 0.02643432654440403,
      "rewards/rejected": -0.0947265625,
      "step": 12540
    },
    {
      "epoch": 0.4702928556706826,
      "grad_norm": 1.7384741489793902,
      "learning_rate": 7.141149482839228e-07,
      "log_odds_chosen": 0.39390867948532104,
      "log_odds_ratio": -0.6180664300918579,
      "logits/chosen": -0.7027343511581421,
      "logits/rejected": -0.642382800579071,
      "logps/chosen": -0.683398425579071,
      "logps/rejected": -0.925976574420929,
      "loss": 0.9194,
      "nll_loss": 0.8726562261581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06829833984375,
      "rewards/margins": 0.024304961785674095,
      "rewards/rejected": -0.09259033203125,
      "step": 12550
    },
    {
      "epoch": 0.4706675910138465,
      "grad_norm": 1.7940356441515386,
      "learning_rate": 7.138306102482496e-07,
      "log_odds_chosen": 0.41179198026657104,
      "log_odds_ratio": -0.638476550579071,
      "logits/chosen": -0.7349609136581421,
      "logits/rejected": -0.666210949420929,
      "logps/chosen": -0.724804699420929,
      "logps/rejected": -1.017968773841858,
      "loss": 0.8903,
      "nll_loss": 0.904101550579071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.072509765625,
      "rewards/margins": 0.02933807298541069,
      "rewards/rejected": -0.101806640625,
      "step": 12560
    },
    {
      "epoch": 0.4710423263570104,
      "grad_norm": 1.842244741041177,
      "learning_rate": 7.135466115856274e-07,
      "log_odds_chosen": 0.5885009765625,
      "log_odds_ratio": -0.5596679449081421,
      "logits/chosen": -0.7662109136581421,
      "logits/rejected": -0.6395508050918579,
      "logps/chosen": -0.6385742425918579,
      "logps/rejected": -0.9970703125,
      "loss": 0.8839,
      "nll_loss": 0.816210925579071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06392822414636612,
      "rewards/margins": 0.03582458570599556,
      "rewards/rejected": -0.09968261420726776,
      "step": 12570
    },
    {
      "epoch": 0.4714170617001743,
      "grad_norm": 1.9686057444426028,
      "learning_rate": 7.13262951621492e-07,
      "log_odds_chosen": 0.5504516363143921,
      "log_odds_ratio": -0.5728515386581421,
      "logits/chosen": -0.7320312261581421,
      "logits/rejected": -0.6253906488418579,
      "logps/chosen": -0.6761718988418579,
      "logps/rejected": -1.025781273841858,
      "loss": 0.9089,
      "nll_loss": 0.8617187738418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06757812201976776,
      "rewards/margins": 0.03497314453125,
      "rewards/rejected": -0.10251464694738388,
      "step": 12580
    },
    {
      "epoch": 0.4717917970433381,
      "grad_norm": 1.6521371691877063,
      "learning_rate": 7.129796296831554e-07,
      "log_odds_chosen": 0.504687488079071,
      "log_odds_ratio": -0.591503918170929,
      "logits/chosen": -0.727734386920929,
      "logits/rejected": -0.635546863079071,
      "logps/chosen": -0.6934570074081421,
      "logps/rejected": -1.009179711341858,
      "loss": 0.9074,
      "nll_loss": 0.8570312261581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.0693359375,
      "rewards/margins": 0.03160705417394638,
      "rewards/rejected": -0.10096435248851776,
      "step": 12590
    },
    {
      "epoch": 0.472166532386502,
      "grad_norm": 1.7138510346498201,
      "learning_rate": 7.126966450997984e-07,
      "log_odds_chosen": 0.4675048887729645,
      "log_odds_ratio": -0.6029297113418579,
      "logits/chosen": -0.719433605670929,
      "logits/rejected": -0.634082019329071,
      "logps/chosen": -0.7171875238418579,
      "logps/rejected": -1.0203125476837158,
      "loss": 0.9031,
      "nll_loss": 0.8541015386581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07167968899011612,
      "rewards/margins": 0.030318450182676315,
      "rewards/rejected": -0.10205078125,
      "step": 12600
    },
    {
      "epoch": 0.4725412677296659,
      "grad_norm": 1.9814509945754266,
      "learning_rate": 7.124139972024637e-07,
      "log_odds_chosen": 0.4554443359375,
      "log_odds_ratio": -0.598437488079071,
      "logits/chosen": -0.667187511920929,
      "logits/rejected": -0.597460925579071,
      "logps/chosen": -0.7201172113418579,
      "logps/rejected": -0.998828113079071,
      "loss": 0.9108,
      "nll_loss": 0.933398425579071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07199706882238388,
      "rewards/margins": 0.02782898023724556,
      "rewards/rejected": -0.09987793117761612,
      "step": 12610
    },
    {
      "epoch": 0.4729160030728298,
      "grad_norm": 1.732408779335823,
      "learning_rate": 7.121316853240503e-07,
      "log_odds_chosen": 0.42829591035842896,
      "log_odds_ratio": -0.620898425579071,
      "logits/chosen": -0.7138671875,
      "logits/rejected": -0.6070312261581421,
      "logps/chosen": -0.740039050579071,
      "logps/rejected": -1.0046875476837158,
      "loss": 0.9076,
      "nll_loss": 0.8470703363418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07401122897863388,
      "rewards/margins": 0.026486968621611595,
      "rewards/rejected": -0.10048828274011612,
      "step": 12620
    },
    {
      "epoch": 0.47329073841599373,
      "grad_norm": 1.7810872011040226,
      "learning_rate": 7.118497087993057e-07,
      "log_odds_chosen": 0.4443359375,
      "log_odds_ratio": -0.6156250238418579,
      "logits/chosen": -0.7147461175918579,
      "logits/rejected": -0.6185547113418579,
      "logps/chosen": -0.667187511920929,
      "logps/rejected": -0.958203136920929,
      "loss": 0.9079,
      "nll_loss": 0.8365234136581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06672362983226776,
      "rewards/margins": 0.02904663048684597,
      "rewards/rejected": -0.09578857570886612,
      "step": 12630
    },
    {
      "epoch": 0.4736654737591576,
      "grad_norm": 1.8387720085366814,
      "learning_rate": 7.1156806696482e-07,
      "log_odds_chosen": 0.41925048828125,
      "log_odds_ratio": -0.63037109375,
      "logits/chosen": -0.716015636920929,
      "logits/rejected": -0.628125011920929,
      "logps/chosen": -0.725390613079071,
      "logps/rejected": -0.9986327886581421,
      "loss": 0.9118,
      "nll_loss": 0.849804699420929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07252196967601776,
      "rewards/margins": 0.02736205980181694,
      "rewards/rejected": -0.09992675483226776,
      "step": 12640
    },
    {
      "epoch": 0.4740402091023215,
      "grad_norm": 1.9462867500260361,
      "learning_rate": 7.112867591590192e-07,
      "log_odds_chosen": 0.45428466796875,
      "log_odds_ratio": -0.6011718511581421,
      "logits/chosen": -0.7490234375,
      "logits/rejected": -0.6480468511581421,
      "logps/chosen": -0.6205078363418579,
      "logps/rejected": -0.894335925579071,
      "loss": 0.8955,
      "nll_loss": 0.7919921875,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06207275390625,
      "rewards/margins": 0.02728576585650444,
      "rewards/rejected": -0.08935546875,
      "step": 12650
    },
    {
      "epoch": 0.4744149444454854,
      "grad_norm": 1.657991008351906,
      "learning_rate": 7.110057847221588e-07,
      "log_odds_chosen": 0.5748046636581421,
      "log_odds_ratio": -0.5884765386581421,
      "logits/chosen": -0.703417956829071,
      "logits/rejected": -0.606640636920929,
      "logps/chosen": -0.669921875,
      "logps/rejected": -1.033593773841858,
      "loss": 0.9146,
      "nll_loss": 0.8060547113418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06697998195886612,
      "rewards/margins": 0.036363981664180756,
      "rewards/rejected": -0.10341797024011612,
      "step": 12660
    },
    {
      "epoch": 0.4747896797886493,
      "grad_norm": 1.6564569890002019,
      "learning_rate": 7.107251429963166e-07,
      "log_odds_chosen": 0.4851928651332855,
      "log_odds_ratio": -0.5810546875,
      "logits/chosen": -0.74365234375,
      "logits/rejected": -0.644335925579071,
      "logps/chosen": -0.673632800579071,
      "logps/rejected": -0.968554675579071,
      "loss": 0.925,
      "nll_loss": 0.8326171636581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06737060844898224,
      "rewards/margins": 0.02951507642865181,
      "rewards/rejected": -0.09689941257238388,
      "step": 12670
    },
    {
      "epoch": 0.4751644151318132,
      "grad_norm": 2.0837690160598132,
      "learning_rate": 7.104448333253878e-07,
      "log_odds_chosen": 0.3381103575229645,
      "log_odds_ratio": -0.649707019329071,
      "logits/chosen": -0.669726550579071,
      "logits/rejected": -0.619335949420929,
      "logps/chosen": -0.7132812738418579,
      "logps/rejected": -0.9183593988418579,
      "loss": 0.9002,
      "nll_loss": 0.890625,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07133789360523224,
      "rewards/margins": 0.020475005730986595,
      "rewards/rejected": -0.09178467094898224,
      "step": 12680
    },
    {
      "epoch": 0.475539150474977,
      "grad_norm": 1.9373688509660876,
      "learning_rate": 7.101648550550766e-07,
      "log_odds_chosen": 0.47370606660842896,
      "log_odds_ratio": -0.595019519329071,
      "logits/chosen": -0.7666015625,
      "logits/rejected": -0.6844726800918579,
      "logps/chosen": -0.699999988079071,
      "logps/rejected": -0.995312511920929,
      "loss": 0.9016,
      "nll_loss": 0.8490234613418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07004394382238388,
      "rewards/margins": 0.02954254113137722,
      "rewards/rejected": -0.099609375,
      "step": 12690
    },
    {
      "epoch": 0.4759138858181409,
      "grad_norm": 1.7758799668461935,
      "learning_rate": 7.098852075328911e-07,
      "log_odds_chosen": 0.4109741151332855,
      "log_odds_ratio": -0.626953125,
      "logits/chosen": -0.720703125,
      "logits/rejected": -0.5972656011581421,
      "logps/chosen": -0.7021484375,
      "logps/rejected": -0.968554675579071,
      "loss": 0.8986,
      "nll_loss": 0.782421886920929,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07023926079273224,
      "rewards/margins": 0.026598358526825905,
      "rewards/rejected": -0.09687499701976776,
      "step": 12700
    },
    {
      "epoch": 0.4762886211613048,
      "grad_norm": 1.8479706791945958,
      "learning_rate": 7.096058901081364e-07,
      "log_odds_chosen": 0.46258544921875,
      "log_odds_ratio": -0.5960937738418579,
      "logits/chosen": -0.67236328125,
      "logits/rejected": -0.6044921875,
      "logps/chosen": -0.6717773675918579,
      "logps/rejected": -0.943554699420929,
      "loss": 0.9159,
      "nll_loss": 0.8568359613418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06724853813648224,
      "rewards/margins": 0.02718505822122097,
      "rewards/rejected": -0.09438476711511612,
      "step": 12710
    },
    {
      "epoch": 0.4766633565044687,
      "grad_norm": 2.1587337220231904,
      "learning_rate": 7.093269021319087e-07,
      "log_odds_chosen": 0.577526867389679,
      "log_odds_ratio": -0.566601574420929,
      "logits/chosen": -0.7103515863418579,
      "logits/rejected": -0.5794433355331421,
      "logps/chosen": -0.654101550579071,
      "logps/rejected": -1.017578125,
      "loss": 0.8852,
      "nll_loss": 0.8828125,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06540527194738388,
      "rewards/margins": 0.036347199231386185,
      "rewards/rejected": -0.10174560546875,
      "step": 12720
    },
    {
      "epoch": 0.47703809184763263,
      "grad_norm": 1.8449093509989083,
      "learning_rate": 7.090482429570884e-07,
      "log_odds_chosen": 0.4601074159145355,
      "log_odds_ratio": -0.609667956829071,
      "logits/chosen": -0.698437511920929,
      "logits/rejected": -0.6109374761581421,
      "logps/chosen": -0.670605480670929,
      "logps/rejected": -0.9561523199081421,
      "loss": 0.8908,
      "nll_loss": 0.8433593511581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06704101711511612,
      "rewards/margins": 0.02857360802590847,
      "rewards/rejected": -0.09565429389476776,
      "step": 12730
    },
    {
      "epoch": 0.4774128271907965,
      "grad_norm": 1.975140042155029,
      "learning_rate": 7.087699119383339e-07,
      "log_odds_chosen": 0.6280761957168579,
      "log_odds_ratio": -0.5652831792831421,
      "logits/chosen": -0.746777355670929,
      "logits/rejected": -0.640820324420929,
      "logps/chosen": -0.6859375238418579,
      "logps/rejected": -1.0812499523162842,
      "loss": 0.8983,
      "nll_loss": 0.8402343988418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06854248046875,
      "rewards/margins": 0.03958740085363388,
      "rewards/rejected": -0.10808105766773224,
      "step": 12740
    },
    {
      "epoch": 0.4777875625339604,
      "grad_norm": 2.0071293626614737,
      "learning_rate": 7.084919084320762e-07,
      "log_odds_chosen": 0.564379870891571,
      "log_odds_ratio": -0.5799804925918579,
      "logits/chosen": -0.757128894329071,
      "logits/rejected": -0.6044921875,
      "logps/chosen": -0.674023449420929,
      "logps/rejected": -1.0255858898162842,
      "loss": 0.9082,
      "nll_loss": 0.8617187738418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06740722805261612,
      "rewards/margins": 0.03512725979089737,
      "rewards/rejected": -0.1025390625,
      "step": 12750
    },
    {
      "epoch": 0.4781622978771243,
      "grad_norm": 1.8251793270951544,
      "learning_rate": 7.08214231796511e-07,
      "log_odds_chosen": 0.5157470703125,
      "log_odds_ratio": -0.5966796875,
      "logits/chosen": -0.796191394329071,
      "logits/rejected": -0.6767578125,
      "logps/chosen": -0.6767578125,
      "logps/rejected": -1.0144531726837158,
      "loss": 0.906,
      "nll_loss": 0.865234375,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06761474907398224,
      "rewards/margins": 0.03385315090417862,
      "rewards/rejected": -0.10148926079273224,
      "step": 12760
    },
    {
      "epoch": 0.4785370332202882,
      "grad_norm": 1.9063798737388153,
      "learning_rate": 7.079368813915939e-07,
      "log_odds_chosen": 0.44171142578125,
      "log_odds_ratio": -0.603320300579071,
      "logits/chosen": -0.7193359136581421,
      "logits/rejected": -0.6231445074081421,
      "logps/chosen": -0.6683593988418579,
      "logps/rejected": -0.946484386920929,
      "loss": 0.9057,
      "nll_loss": 0.8896484375,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06678466498851776,
      "rewards/margins": 0.02780761756002903,
      "rewards/rejected": -0.09461669623851776,
      "step": 12770
    },
    {
      "epoch": 0.4789117685634521,
      "grad_norm": 1.789331524727393,
      "learning_rate": 7.076598565790337e-07,
      "log_odds_chosen": 0.3042236268520355,
      "log_odds_ratio": -0.6513671875,
      "logits/chosen": -0.7440429925918579,
      "logits/rejected": -0.657519519329071,
      "logps/chosen": -0.708789050579071,
      "logps/rejected": -0.897656261920929,
      "loss": 0.9092,
      "nll_loss": 0.8246093988418579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07086181640625,
      "rewards/margins": 0.018932342529296875,
      "rewards/rejected": -0.08977051079273224,
      "step": 12780
    },
    {
      "epoch": 0.479286503906616,
      "grad_norm": 2.341572403392424,
      "learning_rate": 7.073831567222859e-07,
      "log_odds_chosen": 0.5513671636581421,
      "log_odds_ratio": -0.55712890625,
      "logits/chosen": -0.7261718511581421,
      "logits/rejected": -0.627636730670929,
      "logps/chosen": -0.651171863079071,
      "logps/rejected": -0.991406261920929,
      "loss": 0.9103,
      "nll_loss": 0.8427734375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06510009616613388,
      "rewards/margins": 0.03399810940027237,
      "rewards/rejected": -0.09907226264476776,
      "step": 12790
    },
    {
      "epoch": 0.4796612392497798,
      "grad_norm": 1.84223526715124,
      "learning_rate": 7.071067811865475e-07,
      "log_odds_chosen": 0.46455079317092896,
      "log_odds_ratio": -0.6319335699081421,
      "logits/chosen": -0.7291015386581421,
      "logits/rejected": -0.6259765625,
      "logps/chosen": -0.706347644329071,
      "logps/rejected": -1.038671851158142,
      "loss": 0.8892,
      "nll_loss": 0.8160156011581421,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07062987983226776,
      "rewards/margins": 0.03325653076171875,
      "rewards/rejected": -0.10389403998851776,
      "step": 12800
    },
    {
      "epoch": 0.4800359745929437,
      "grad_norm": 1.7853587029080378,
      "learning_rate": 7.068307293387497e-07,
      "log_odds_chosen": 0.47343748807907104,
      "log_odds_ratio": -0.5790039300918579,
      "logits/chosen": -0.7623046636581421,
      "logits/rejected": -0.669726550579071,
      "logps/chosen": -0.675976574420929,
      "logps/rejected": -0.976367175579071,
      "loss": 0.8892,
      "nll_loss": 0.8304687738418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06757812201976776,
      "rewards/margins": 0.030059814453125,
      "rewards/rejected": -0.09763183444738388,
      "step": 12810
    },
    {
      "epoch": 0.4804107099361076,
      "grad_norm": 2.015811404748878,
      "learning_rate": 7.065550005475526e-07,
      "log_odds_chosen": 0.3695312440395355,
      "log_odds_ratio": -0.631542980670929,
      "logits/chosen": -0.748242199420929,
      "logits/rejected": -0.6368163824081421,
      "logps/chosen": -0.700488269329071,
      "logps/rejected": -0.9486328363418579,
      "loss": 0.9054,
      "nll_loss": 0.836718738079071,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07010497897863388,
      "rewards/margins": 0.024640655145049095,
      "rewards/rejected": -0.09477539360523224,
      "step": 12820
    },
    {
      "epoch": 0.48078544527927153,
      "grad_norm": 2.1454576548197837,
      "learning_rate": 7.062795941833388e-07,
      "log_odds_chosen": 0.35737305879592896,
      "log_odds_ratio": -0.6400390863418579,
      "logits/chosen": -0.747265636920929,
      "logits/rejected": -0.653027355670929,
      "logps/chosen": -0.69921875,
      "logps/rejected": -0.9468749761581421,
      "loss": 0.9168,
      "nll_loss": 0.8716796636581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06993408501148224,
      "rewards/margins": 0.02473907545208931,
      "rewards/rejected": -0.09475097805261612,
      "step": 12830
    },
    {
      "epoch": 0.48116018062243543,
      "grad_norm": 1.7351728201462913,
      "learning_rate": 7.060045096182077e-07,
      "log_odds_chosen": 0.35980224609375,
      "log_odds_ratio": -0.625292956829071,
      "logits/chosen": -0.7318359613418579,
      "logits/rejected": -0.66162109375,
      "logps/chosen": -0.6888672113418579,
      "logps/rejected": -0.916015625,
      "loss": 0.9099,
      "nll_loss": 0.868359386920929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06877441704273224,
      "rewards/margins": 0.02279968187212944,
      "rewards/rejected": -0.09163818508386612,
      "step": 12840
    },
    {
      "epoch": 0.4815349159655993,
      "grad_norm": 1.7151325689106676,
      "learning_rate": 7.057297462259693e-07,
      "log_odds_chosen": 0.45592039823532104,
      "log_odds_ratio": -0.6001952886581421,
      "logits/chosen": -0.7085937261581421,
      "logits/rejected": -0.623242199420929,
      "logps/chosen": -0.6865234375,
      "logps/rejected": -0.981640636920929,
      "loss": 0.8814,
      "nll_loss": 0.808398425579071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06867675483226776,
      "rewards/margins": 0.029546355828642845,
      "rewards/rejected": -0.09822998195886612,
      "step": 12850
    },
    {
      "epoch": 0.4819096513087632,
      "grad_norm": 1.8112133294430903,
      "learning_rate": 7.05455303382138e-07,
      "log_odds_chosen": 0.47980958223342896,
      "log_odds_ratio": -0.6048828363418579,
      "logits/chosen": -0.6778320074081421,
      "logits/rejected": -0.619140625,
      "logps/chosen": -0.69482421875,
      "logps/rejected": -0.995898425579071,
      "loss": 0.8917,
      "nll_loss": 0.818359375,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06944580376148224,
      "rewards/margins": 0.03017120435833931,
      "rewards/rejected": -0.09968261420726776,
      "step": 12860
    },
    {
      "epoch": 0.4822843866519271,
      "grad_norm": 1.8305724645067685,
      "learning_rate": 7.051811804639268e-07,
      "log_odds_chosen": 0.50958251953125,
      "log_odds_ratio": -0.579296886920929,
      "logits/chosen": -0.7022460699081421,
      "logits/rejected": -0.6435546875,
      "logps/chosen": -0.679980456829071,
      "logps/rejected": -1.0078125,
      "loss": 0.887,
      "nll_loss": 0.819531261920929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06791992485523224,
      "rewards/margins": 0.0328521728515625,
      "rewards/rejected": -0.10075683891773224,
      "step": 12870
    },
    {
      "epoch": 0.482659121995091,
      "grad_norm": 1.8579500549982026,
      "learning_rate": 7.049073768502414e-07,
      "log_odds_chosen": 0.5672363042831421,
      "log_odds_ratio": -0.559863269329071,
      "logits/chosen": -0.694531261920929,
      "logits/rejected": -0.602343738079071,
      "logps/chosen": -0.706738293170929,
      "logps/rejected": -1.069726586341858,
      "loss": 0.9184,
      "nll_loss": 0.889843761920929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07072754204273224,
      "rewards/margins": 0.03622589260339737,
      "rewards/rejected": -0.10698242485523224,
      "step": 12880
    },
    {
      "epoch": 0.4830338573382549,
      "grad_norm": 1.8274850139793526,
      "learning_rate": 7.046338919216742e-07,
      "log_odds_chosen": 0.5078125,
      "log_odds_ratio": -0.5767577886581421,
      "logits/chosen": -0.732226550579071,
      "logits/rejected": -0.667675793170929,
      "logps/chosen": -0.66845703125,
      "logps/rejected": -0.985156238079071,
      "loss": 0.8959,
      "nll_loss": 0.845898449420929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.0667724609375,
      "rewards/margins": 0.031702421605587006,
      "rewards/rejected": -0.09855957329273224,
      "step": 12890
    },
    {
      "epoch": 0.4834085926814187,
      "grad_norm": 1.7364601053826887,
      "learning_rate": 7.04360725060499e-07,
      "log_odds_chosen": 0.49519044160842896,
      "log_odds_ratio": -0.5946289300918579,
      "logits/chosen": -0.7398437261581421,
      "logits/rejected": -0.624316394329071,
      "logps/chosen": -0.748828113079071,
      "logps/rejected": -1.0769531726837158,
      "loss": 0.8858,
      "nll_loss": 0.8333984613418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.074951171875,
      "rewards/margins": 0.03276214748620987,
      "rewards/rejected": -0.10764160007238388,
      "step": 12900
    },
    {
      "epoch": 0.4837833280245826,
      "grad_norm": 1.8506299740685432,
      "learning_rate": 7.040878756506639e-07,
      "log_odds_chosen": 0.5266357660293579,
      "log_odds_ratio": -0.5775390863418579,
      "logits/chosen": -0.708984375,
      "logits/rejected": -0.605175793170929,
      "logps/chosen": -0.697460949420929,
      "logps/rejected": -1.033203125,
      "loss": 0.9141,
      "nll_loss": 0.859570324420929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06976318359375,
      "rewards/margins": 0.03356323391199112,
      "rewards/rejected": -0.10325928032398224,
      "step": 12910
    },
    {
      "epoch": 0.4841580633677465,
      "grad_norm": 2.095646879442153,
      "learning_rate": 7.038153430777867e-07,
      "log_odds_chosen": 0.3190551698207855,
      "log_odds_ratio": -0.6512695550918579,
      "logits/chosen": -0.7459961175918579,
      "logits/rejected": -0.6279296875,
      "logps/chosen": -0.709179699420929,
      "logps/rejected": -0.898242175579071,
      "loss": 0.8806,
      "nll_loss": 0.8160156011581421,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07093505561351776,
      "rewards/margins": 0.01887054368853569,
      "rewards/rejected": -0.08981933444738388,
      "step": 12920
    },
    {
      "epoch": 0.48453279871091043,
      "grad_norm": 1.9213654688519408,
      "learning_rate": 7.035431267291484e-07,
      "log_odds_chosen": 0.5093628168106079,
      "log_odds_ratio": -0.6102539300918579,
      "logits/chosen": -0.695117175579071,
      "logits/rejected": -0.62109375,
      "logps/chosen": -0.68115234375,
      "logps/rejected": -0.991015613079071,
      "loss": 0.9111,
      "nll_loss": 0.8345702886581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06812743842601776,
      "rewards/margins": 0.030913542956113815,
      "rewards/rejected": -0.09919433295726776,
      "step": 12930
    },
    {
      "epoch": 0.48490753405407433,
      "grad_norm": 1.8543584282169954,
      "learning_rate": 7.032712259936877e-07,
      "log_odds_chosen": 0.4820800721645355,
      "log_odds_ratio": -0.5874999761581421,
      "logits/chosen": -0.7132812738418579,
      "logits/rejected": -0.60302734375,
      "logps/chosen": -0.664257824420929,
      "logps/rejected": -0.9544922113418579,
      "loss": 0.896,
      "nll_loss": 0.806445300579071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06645508110523224,
      "rewards/margins": 0.02906188927590847,
      "rewards/rejected": -0.09555663913488388,
      "step": 12940
    },
    {
      "epoch": 0.4852822693972382,
      "grad_norm": 1.8755337285885716,
      "learning_rate": 7.029996402619949e-07,
      "log_odds_chosen": 0.4221435487270355,
      "log_odds_ratio": -0.6131836175918579,
      "logits/chosen": -0.777148425579071,
      "logits/rejected": -0.649707019329071,
      "logps/chosen": -0.7505859136581421,
      "logps/rejected": -1.026757836341858,
      "loss": 0.917,
      "nll_loss": 0.875195324420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07500000298023224,
      "rewards/margins": 0.02760620042681694,
      "rewards/rejected": -0.10267333686351776,
      "step": 12950
    },
    {
      "epoch": 0.4856570047404021,
      "grad_norm": 1.7713758545627023,
      "learning_rate": 7.027283689263065e-07,
      "log_odds_chosen": 0.6556030511856079,
      "log_odds_ratio": -0.560546875,
      "logits/chosen": -0.718066394329071,
      "logits/rejected": -0.5775390863418579,
      "logps/chosen": -0.693359375,
      "logps/rejected": -1.134179711341858,
      "loss": 0.8969,
      "nll_loss": 0.867968738079071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.0693359375,
      "rewards/margins": 0.04407806321978569,
      "rewards/rejected": -0.11347655951976776,
      "step": 12960
    },
    {
      "epoch": 0.486031740083566,
      "grad_norm": 1.8546788806097063,
      "learning_rate": 7.024574113804996e-07,
      "log_odds_chosen": 0.509814441204071,
      "log_odds_ratio": -0.6073242425918579,
      "logits/chosen": -0.7447265386581421,
      "logits/rejected": -0.6475585699081421,
      "logps/chosen": -0.6689453125,
      "logps/rejected": -1.0031249523162842,
      "loss": 0.8999,
      "nll_loss": 0.832226574420929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06683349609375,
      "rewards/margins": 0.03350677341222763,
      "rewards/rejected": -0.10039062798023224,
      "step": 12970
    },
    {
      "epoch": 0.4864064754267299,
      "grad_norm": 1.8925880629980885,
      "learning_rate": 7.021867670200857e-07,
      "log_odds_chosen": 0.46314698457717896,
      "log_odds_ratio": -0.596972644329071,
      "logits/chosen": -0.7818359136581421,
      "logits/rejected": -0.6385742425918579,
      "logps/chosen": -0.662304699420929,
      "logps/rejected": -0.9677734375,
      "loss": 0.9158,
      "nll_loss": 0.8462890386581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06621094048023224,
      "rewards/margins": 0.030541229993104935,
      "rewards/rejected": -0.0968017578125,
      "step": 12980
    },
    {
      "epoch": 0.4867812107698938,
      "grad_norm": 1.9668689250978988,
      "learning_rate": 7.019164352422057e-07,
      "log_odds_chosen": 0.48851317167282104,
      "log_odds_ratio": -0.5965820550918579,
      "logits/chosen": -0.7095702886581421,
      "logits/rejected": -0.6185547113418579,
      "logps/chosen": -0.6463867425918579,
      "logps/rejected": -0.9332031011581421,
      "loss": 0.8999,
      "nll_loss": 0.783007800579071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06462402641773224,
      "rewards/margins": 0.02856750413775444,
      "rewards/rejected": -0.09326171875,
      "step": 12990
    },
    {
      "epoch": 0.4871559461130577,
      "grad_norm": 2.0147080534536976,
      "learning_rate": 7.016464154456234e-07,
      "log_odds_chosen": 0.47187501192092896,
      "log_odds_ratio": -0.6131836175918579,
      "logits/chosen": -0.7344726324081421,
      "logits/rejected": -0.661328136920929,
      "logps/chosen": -0.7533203363418579,
      "logps/rejected": -1.054101586341858,
      "loss": 0.9053,
      "nll_loss": 0.8480468988418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07526855170726776,
      "rewards/margins": 0.03012695349752903,
      "rewards/rejected": -0.10544433444738388,
      "step": 13000
    },
    {
      "epoch": 0.4875306814562215,
      "grad_norm": 1.926442745332714,
      "learning_rate": 7.013767070307207e-07,
      "log_odds_chosen": 0.4592651426792145,
      "log_odds_ratio": -0.6197265386581421,
      "logits/chosen": -0.7412109375,
      "logits/rejected": -0.6439453363418579,
      "logps/chosen": -0.6954101324081421,
      "logps/rejected": -0.992968738079071,
      "loss": 0.9146,
      "nll_loss": 0.796093761920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06953124701976776,
      "rewards/margins": 0.02982635423541069,
      "rewards/rejected": -0.09938964992761612,
      "step": 13010
    },
    {
      "epoch": 0.4879054167993854,
      "grad_norm": 1.8171572927947668,
      "learning_rate": 7.011073093994919e-07,
      "log_odds_chosen": 0.5606445074081421,
      "log_odds_ratio": -0.56298828125,
      "logits/chosen": -0.7188476324081421,
      "logits/rejected": -0.639941394329071,
      "logps/chosen": -0.667187511920929,
      "logps/rejected": -1.015234351158142,
      "loss": 0.9015,
      "nll_loss": 0.8343750238418579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06673584133386612,
      "rewards/margins": 0.03486023098230362,
      "rewards/rejected": -0.10162353515625,
      "step": 13020
    },
    {
      "epoch": 0.48828015214254933,
      "grad_norm": 1.7678462491343077,
      "learning_rate": 7.008382219555372e-07,
      "log_odds_chosen": 0.44329833984375,
      "log_odds_ratio": -0.616015613079071,
      "logits/chosen": -0.698437511920929,
      "logits/rejected": -0.625195324420929,
      "logps/chosen": -0.7120116949081421,
      "logps/rejected": -0.9703124761581421,
      "loss": 0.9018,
      "nll_loss": 0.9091796875,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07114257663488388,
      "rewards/margins": 0.02586975134909153,
      "rewards/rejected": -0.09709472954273224,
      "step": 13030
    },
    {
      "epoch": 0.48865488748571323,
      "grad_norm": 1.8644728427094552,
      "learning_rate": 7.005694441040588e-07,
      "log_odds_chosen": 0.540771484375,
      "log_odds_ratio": -0.583203136920929,
      "logits/chosen": -0.751757800579071,
      "logits/rejected": -0.6625000238418579,
      "logps/chosen": -0.68798828125,
      "logps/rejected": -1.027734398841858,
      "loss": 0.8896,
      "nll_loss": 0.80859375,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06878662109375,
      "rewards/margins": 0.034038543701171875,
      "rewards/rejected": -0.10284423828125,
      "step": 13040
    },
    {
      "epoch": 0.48902962282887713,
      "grad_norm": 1.7952893121491005,
      "learning_rate": 7.003009752518536e-07,
      "log_odds_chosen": 0.42335206270217896,
      "log_odds_ratio": -0.609667956829071,
      "logits/chosen": -0.7378906011581421,
      "logits/rejected": -0.6669921875,
      "logps/chosen": -0.662792980670929,
      "logps/rejected": -0.917773425579071,
      "loss": 0.8846,
      "nll_loss": 0.804492175579071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06624756008386612,
      "rewards/margins": 0.0254669189453125,
      "rewards/rejected": -0.09174804389476776,
      "step": 13050
    },
    {
      "epoch": 0.489404358172041,
      "grad_norm": 2.0673831348294556,
      "learning_rate": 7.000328148073091e-07,
      "log_odds_chosen": 0.43120115995407104,
      "log_odds_ratio": -0.6141601800918579,
      "logits/chosen": -0.72265625,
      "logits/rejected": -0.65234375,
      "logps/chosen": -0.648632824420929,
      "logps/rejected": -0.8843749761581421,
      "loss": 0.9022,
      "nll_loss": 0.8462890386581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06484375149011612,
      "rewards/margins": 0.02352600172162056,
      "rewards/rejected": -0.08845214545726776,
      "step": 13060
    },
    {
      "epoch": 0.4897790935152049,
      "grad_norm": 1.9325981220521704,
      "learning_rate": 6.997649621803973e-07,
      "log_odds_chosen": 0.40330809354782104,
      "log_odds_ratio": -0.6250976324081421,
      "logits/chosen": -0.740527331829071,
      "logits/rejected": -0.658007800579071,
      "logps/chosen": -0.655468761920929,
      "logps/rejected": -0.8853515386581421,
      "loss": 0.8844,
      "nll_loss": 0.8353515863418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06550292670726776,
      "rewards/margins": 0.023062895983457565,
      "rewards/rejected": -0.08851318061351776,
      "step": 13070
    },
    {
      "epoch": 0.4901538288583688,
      "grad_norm": 1.9219019705794902,
      "learning_rate": 6.994974167826689e-07,
      "log_odds_chosen": 0.3576416075229645,
      "log_odds_ratio": -0.6220703125,
      "logits/chosen": -0.7490234375,
      "logits/rejected": -0.629687488079071,
      "logps/chosen": -0.6904296875,
      "logps/rejected": -0.9134765863418579,
      "loss": 0.8951,
      "nll_loss": 0.8119140863418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06895752251148224,
      "rewards/margins": 0.022306060418486595,
      "rewards/rejected": -0.09133300930261612,
      "step": 13080
    },
    {
      "epoch": 0.4905285642015327,
      "grad_norm": 1.753602138593595,
      "learning_rate": 6.99230178027249e-07,
      "log_odds_chosen": 0.558337390422821,
      "log_odds_ratio": -0.580859363079071,
      "logits/chosen": -0.778124988079071,
      "logits/rejected": -0.6670898199081421,
      "logps/chosen": -0.704296886920929,
      "logps/rejected": -1.0554687976837158,
      "loss": 0.9011,
      "nll_loss": 0.8353515863418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07039795070886612,
      "rewards/margins": 0.03516845777630806,
      "rewards/rejected": -0.10552978515625,
      "step": 13090
    },
    {
      "epoch": 0.4909032995446966,
      "grad_norm": 1.8645020394627774,
      "learning_rate": 6.989632453288303e-07,
      "log_odds_chosen": 0.3937011659145355,
      "log_odds_ratio": -0.633593738079071,
      "logits/chosen": -0.7100585699081421,
      "logits/rejected": -0.6117187738418579,
      "logps/chosen": -0.708203136920929,
      "logps/rejected": -0.9681640863418579,
      "loss": 0.9156,
      "nll_loss": 0.8603515625,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07087402045726776,
      "rewards/margins": 0.02594604529440403,
      "rewards/rejected": -0.09677734225988388,
      "step": 13100
    },
    {
      "epoch": 0.4912780348878604,
      "grad_norm": 1.7448278603346328,
      "learning_rate": 6.98696618103669e-07,
      "log_odds_chosen": 0.4095458984375,
      "log_odds_ratio": -0.618945300579071,
      "logits/chosen": -0.7457031011581421,
      "logits/rejected": -0.63671875,
      "logps/chosen": -0.689648449420929,
      "logps/rejected": -0.944140613079071,
      "loss": 0.9034,
      "nll_loss": 0.836621105670929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06898193061351776,
      "rewards/margins": 0.02538757398724556,
      "rewards/rejected": -0.09440918266773224,
      "step": 13110
    },
    {
      "epoch": 0.4916527702310243,
      "grad_norm": 1.8695459252355275,
      "learning_rate": 6.984302957695782e-07,
      "log_odds_chosen": 0.3878417909145355,
      "log_odds_ratio": -0.649218738079071,
      "logits/chosen": -0.737500011920929,
      "logits/rejected": -0.64599609375,
      "logps/chosen": -0.6993163824081421,
      "logps/rejected": -0.9576171636581421,
      "loss": 0.8962,
      "nll_loss": 0.8707031011581421,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06992187350988388,
      "rewards/margins": 0.02589111402630806,
      "rewards/rejected": -0.0958251953125,
      "step": 13120
    },
    {
      "epoch": 0.49202750557418823,
      "grad_norm": 1.8238247692852743,
      "learning_rate": 6.981642777459237e-07,
      "log_odds_chosen": 0.4026855528354645,
      "log_odds_ratio": -0.62744140625,
      "logits/chosen": -0.7972656488418579,
      "logits/rejected": -0.657910168170929,
      "logps/chosen": -0.7328125238418579,
      "logps/rejected": -1.00390625,
      "loss": 0.8808,
      "nll_loss": 0.8570312261581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07329101860523224,
      "rewards/margins": 0.02704162523150444,
      "rewards/rejected": -0.10039062798023224,
      "step": 13130
    },
    {
      "epoch": 0.49240224091735213,
      "grad_norm": 2.1258037145107314,
      "learning_rate": 6.978985634536182e-07,
      "log_odds_chosen": 0.59783935546875,
      "log_odds_ratio": -0.5716797113418579,
      "logits/chosen": -0.68896484375,
      "logits/rejected": -0.5811523199081421,
      "logps/chosen": -0.711230456829071,
      "logps/rejected": -1.098046898841858,
      "loss": 0.9186,
      "nll_loss": 0.8609374761581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07111816108226776,
      "rewards/margins": 0.038756560534238815,
      "rewards/rejected": -0.10991211235523224,
      "step": 13140
    },
    {
      "epoch": 0.49277697626051603,
      "grad_norm": 1.8856551714415226,
      "learning_rate": 6.976331523151157e-07,
      "log_odds_chosen": 0.4369140565395355,
      "log_odds_ratio": -0.6031249761581421,
      "logits/chosen": -0.705761730670929,
      "logits/rejected": -0.640917956829071,
      "logps/chosen": -0.682421863079071,
      "logps/rejected": -0.9537109136581421,
      "loss": 0.9184,
      "nll_loss": 0.8658202886581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.0682373046875,
      "rewards/margins": 0.02716827392578125,
      "rewards/rejected": -0.09531249850988388,
      "step": 13150
    },
    {
      "epoch": 0.4931517116036799,
      "grad_norm": 2.227016826005924,
      "learning_rate": 6.973680437544066e-07,
      "log_odds_chosen": 0.3785644471645355,
      "log_odds_ratio": -0.619824230670929,
      "logits/chosen": -0.7281249761581421,
      "logits/rejected": -0.6167968511581421,
      "logps/chosen": -0.6456054449081421,
      "logps/rejected": -0.8939453363418579,
      "loss": 0.9053,
      "nll_loss": 0.8583984375,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.0645751953125,
      "rewards/margins": 0.024845123291015625,
      "rewards/rejected": -0.08945312350988388,
      "step": 13160
    },
    {
      "epoch": 0.4935264469468438,
      "grad_norm": 1.8144733740752144,
      "learning_rate": 6.971032371970126e-07,
      "log_odds_chosen": 0.40028077363967896,
      "log_odds_ratio": -0.6429687738418579,
      "logits/chosen": -0.785351574420929,
      "logits/rejected": -0.7093750238418579,
      "logps/chosen": -0.70703125,
      "logps/rejected": -0.9912109375,
      "loss": 0.896,
      "nll_loss": 0.8203125,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07064209133386612,
      "rewards/margins": 0.02839813195168972,
      "rewards/rejected": -0.09906005859375,
      "step": 13170
    },
    {
      "epoch": 0.4939011822900077,
      "grad_norm": 1.9612961960197743,
      "learning_rate": 6.968387320699806e-07,
      "log_odds_chosen": 0.34852296113967896,
      "log_odds_ratio": -0.659960925579071,
      "logits/chosen": -0.7197265625,
      "logits/rejected": -0.65625,
      "logps/chosen": -0.708984375,
      "logps/rejected": -0.9261718988418579,
      "loss": 0.9002,
      "nll_loss": 0.8558593988418579,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07097168266773224,
      "rewards/margins": 0.02172393724322319,
      "rewards/rejected": -0.09262695163488388,
      "step": 13180
    },
    {
      "epoch": 0.4942759176331716,
      "grad_norm": 1.8549408478702485,
      "learning_rate": 6.965745278018791e-07,
      "log_odds_chosen": 0.550677478313446,
      "log_odds_ratio": -0.5712890625,
      "logits/chosen": -0.7007812261581421,
      "logits/rejected": -0.600292980670929,
      "logps/chosen": -0.64453125,
      "logps/rejected": -0.999218761920929,
      "loss": 0.9115,
      "nll_loss": 0.8720703125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06442870944738388,
      "rewards/margins": 0.03541717678308487,
      "rewards/rejected": -0.09982910007238388,
      "step": 13190
    },
    {
      "epoch": 0.4946506529763355,
      "grad_norm": 1.801113102245271,
      "learning_rate": 6.963106238227914e-07,
      "log_odds_chosen": 0.41156005859375,
      "log_odds_ratio": -0.624707043170929,
      "logits/chosen": -0.6947265863418579,
      "logits/rejected": -0.592480480670929,
      "logps/chosen": -0.654492199420929,
      "logps/rejected": -0.9247070550918579,
      "loss": 0.8975,
      "nll_loss": 0.8343750238418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0654296875,
      "rewards/margins": 0.027069855481386185,
      "rewards/rejected": -0.09245605766773224,
      "step": 13200
    },
    {
      "epoch": 0.4950253883194993,
      "grad_norm": 1.7666341779703227,
      "learning_rate": 6.96047019564311e-07,
      "log_odds_chosen": 0.39161378145217896,
      "log_odds_ratio": -0.640429675579071,
      "logits/chosen": -0.671093761920929,
      "logits/rejected": -0.603222668170929,
      "logps/chosen": -0.694140613079071,
      "logps/rejected": -0.9439452886581421,
      "loss": 0.9166,
      "nll_loss": 0.9292968511581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06943359225988388,
      "rewards/margins": 0.02492828294634819,
      "rewards/rejected": -0.09440918266773224,
      "step": 13210
    },
    {
      "epoch": 0.4954001236626632,
      "grad_norm": 3.5739536808709493,
      "learning_rate": 6.957837144595368e-07,
      "log_odds_chosen": 0.456787109375,
      "log_odds_ratio": -0.602246105670929,
      "logits/chosen": -0.7442382574081421,
      "logits/rejected": -0.6211913824081421,
      "logps/chosen": -0.6943359375,
      "logps/rejected": -0.9906250238418579,
      "loss": 0.8873,
      "nll_loss": 0.8587890863418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06942138820886612,
      "rewards/margins": 0.02964019775390625,
      "rewards/rejected": -0.09909667819738388,
      "step": 13220
    },
    {
      "epoch": 0.49577485900582713,
      "grad_norm": 1.8216940938195583,
      "learning_rate": 6.955207079430681e-07,
      "log_odds_chosen": 0.5235840082168579,
      "log_odds_ratio": -0.588085949420929,
      "logits/chosen": -0.6908203363418579,
      "logits/rejected": -0.5882812738418579,
      "logps/chosen": -0.6644531488418579,
      "logps/rejected": -1.000585913658142,
      "loss": 0.9079,
      "nll_loss": 0.8580077886581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06646728515625,
      "rewards/margins": 0.03362121433019638,
      "rewards/rejected": -0.10002441704273224,
      "step": 13230
    },
    {
      "epoch": 0.49614959434899103,
      "grad_norm": 1.9747997042653749,
      "learning_rate": 6.952579994509982e-07,
      "log_odds_chosen": 0.47459715604782104,
      "log_odds_ratio": -0.58349609375,
      "logits/chosen": -0.716015636920929,
      "logits/rejected": -0.612988293170929,
      "logps/chosen": -0.708203136920929,
      "logps/rejected": -1.0078125,
      "loss": 0.9161,
      "nll_loss": 0.841015636920929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07086181640625,
      "rewards/margins": 0.03000793419778347,
      "rewards/rejected": -0.10076904296875,
      "step": 13240
    },
    {
      "epoch": 0.49652432969215493,
      "grad_norm": 1.881760840474921,
      "learning_rate": 6.94995588420911e-07,
      "log_odds_chosen": 0.36903685331344604,
      "log_odds_ratio": -0.628222644329071,
      "logits/chosen": -0.754687488079071,
      "logits/rejected": -0.65869140625,
      "logps/chosen": -0.7442382574081421,
      "logps/rejected": -0.985156238079071,
      "loss": 0.8996,
      "nll_loss": 0.8154296875,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07440185546875,
      "rewards/margins": 0.02414245530962944,
      "rewards/rejected": -0.09855957329273224,
      "step": 13250
    },
    {
      "epoch": 0.49689906503531883,
      "grad_norm": 1.7390429269500305,
      "learning_rate": 6.947334742918749e-07,
      "log_odds_chosen": 0.58013916015625,
      "log_odds_ratio": -0.5738281011581421,
      "logits/chosen": -0.69921875,
      "logits/rejected": -0.5838867425918579,
      "logps/chosen": -0.6875976324081421,
      "logps/rejected": -1.077539086341858,
      "loss": 0.9302,
      "nll_loss": 0.932812511920929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06876220554113388,
      "rewards/margins": 0.03906402736902237,
      "rewards/rejected": -0.10773925483226776,
      "step": 13260
    },
    {
      "epoch": 0.4972738003784827,
      "grad_norm": 1.7891960409980436,
      "learning_rate": 6.94471656504438e-07,
      "log_odds_chosen": 0.5835937261581421,
      "log_odds_ratio": -0.594921886920929,
      "logits/chosen": -0.681933581829071,
      "logits/rejected": -0.586230456829071,
      "logps/chosen": -0.65283203125,
      "logps/rejected": -1.0246093273162842,
      "loss": 0.8926,
      "nll_loss": 0.8251953125,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.0653076171875,
      "rewards/margins": 0.03714447095990181,
      "rewards/rejected": -0.10244140774011612,
      "step": 13270
    },
    {
      "epoch": 0.4976485357216466,
      "grad_norm": 1.8387933548098694,
      "learning_rate": 6.942101345006232e-07,
      "log_odds_chosen": 0.5000244379043579,
      "log_odds_ratio": -0.599316418170929,
      "logits/chosen": -0.727734386920929,
      "logits/rejected": -0.630566418170929,
      "logps/chosen": -0.70654296875,
      "logps/rejected": -1.02587890625,
      "loss": 0.9088,
      "nll_loss": 0.861132800579071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.0706787109375,
      "rewards/margins": 0.031894683837890625,
      "rewards/rejected": -0.1025390625,
      "step": 13280
    },
    {
      "epoch": 0.4980232710648105,
      "grad_norm": 1.961453358750603,
      "learning_rate": 6.939489077239235e-07,
      "log_odds_chosen": 0.306884765625,
      "log_odds_ratio": -0.6502929925918579,
      "logits/chosen": -0.7240234613418579,
      "logits/rejected": -0.645214855670929,
      "logps/chosen": -0.7129882574081421,
      "logps/rejected": -0.882617175579071,
      "loss": 0.8726,
      "nll_loss": 0.798828125,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07132568210363388,
      "rewards/margins": 0.016938019543886185,
      "rewards/rejected": -0.0882568359375,
      "step": 13290
    },
    {
      "epoch": 0.4983980064079744,
      "grad_norm": 2.2298430274349585,
      "learning_rate": 6.936879756192959e-07,
      "log_odds_chosen": 0.5433105230331421,
      "log_odds_ratio": -0.5721679925918579,
      "logits/chosen": -0.7779296636581421,
      "logits/rejected": -0.6650390625,
      "logps/chosen": -0.6851562261581421,
      "logps/rejected": -1.0427734851837158,
      "loss": 0.894,
      "nll_loss": 0.881054699420929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06853027641773224,
      "rewards/margins": 0.03578796237707138,
      "rewards/rejected": -0.104248046875,
      "step": 13300
    },
    {
      "epoch": 0.4987727417511383,
      "grad_norm": 1.934172272586143,
      "learning_rate": 6.934273376331579e-07,
      "log_odds_chosen": 0.2745361328125,
      "log_odds_ratio": -0.6639648675918579,
      "logits/chosen": -0.719921886920929,
      "logits/rejected": -0.6708984375,
      "logps/chosen": -0.7342773675918579,
      "logps/rejected": -0.9046875238418579,
      "loss": 0.9022,
      "nll_loss": 0.891406238079071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07342529296875,
      "rewards/margins": 0.017005156725645065,
      "rewards/rejected": -0.09040527045726776,
      "step": 13310
    },
    {
      "epoch": 0.4991474770943021,
      "grad_norm": 1.9880220496766248,
      "learning_rate": 6.931669932133818e-07,
      "log_odds_chosen": 0.4676757752895355,
      "log_odds_ratio": -0.61376953125,
      "logits/chosen": -0.7490234375,
      "logits/rejected": -0.6631835699081421,
      "logps/chosen": -0.6947265863418579,
      "logps/rejected": -0.989062488079071,
      "loss": 0.901,
      "nll_loss": 0.856640636920929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06950683891773224,
      "rewards/margins": 0.029383087530732155,
      "rewards/rejected": -0.09882812201976776,
      "step": 13320
    },
    {
      "epoch": 0.49952221243746603,
      "grad_norm": 1.8845674286745426,
      "learning_rate": 6.929069418092892e-07,
      "log_odds_chosen": 0.42652589082717896,
      "log_odds_ratio": -0.6371093988418579,
      "logits/chosen": -0.751953125,
      "logits/rejected": -0.65234375,
      "logps/chosen": -0.7109375,
      "logps/rejected": -0.981640636920929,
      "loss": 0.9109,
      "nll_loss": 0.8310546875,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07109375298023224,
      "rewards/margins": 0.02710266038775444,
      "rewards/rejected": -0.09820556640625,
      "step": 13330
    },
    {
      "epoch": 0.49989694778062993,
      "grad_norm": 1.8653174847606127,
      "learning_rate": 6.926471828716478e-07,
      "log_odds_chosen": 0.3433471620082855,
      "log_odds_ratio": -0.6390625238418579,
      "logits/chosen": -0.7333984375,
      "logits/rejected": -0.638671875,
      "logps/chosen": -0.691210925579071,
      "logps/rejected": -0.913867175579071,
      "loss": 0.9048,
      "nll_loss": 0.8662109375,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06903076171875,
      "rewards/margins": 0.022319793701171875,
      "rewards/rejected": -0.09140624850988388,
      "step": 13340
    },
    {
      "epoch": 0.5002716831237938,
      "grad_norm": 1.7496022175652681,
      "learning_rate": 6.923877158526646e-07,
      "log_odds_chosen": 0.46586912870407104,
      "log_odds_ratio": -0.6158202886581421,
      "logits/chosen": -0.7822265625,
      "logits/rejected": -0.6322265863418579,
      "logps/chosen": -0.7103515863418579,
      "logps/rejected": -1.0246093273162842,
      "loss": 0.9157,
      "nll_loss": 0.844921886920929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.071044921875,
      "rewards/margins": 0.03135070949792862,
      "rewards/rejected": -0.10244140774011612,
      "step": 13350
    },
    {
      "epoch": 0.5006464184669577,
      "grad_norm": 1.8889388090230916,
      "learning_rate": 6.921285402059827e-07,
      "log_odds_chosen": 0.4144287109375,
      "log_odds_ratio": -0.613574206829071,
      "logits/chosen": -0.756054699420929,
      "logits/rejected": -0.642773449420929,
      "logps/chosen": -0.662890613079071,
      "logps/rejected": -0.9261718988418579,
      "loss": 0.9052,
      "nll_loss": 0.83203125,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06635741889476776,
      "rewards/margins": 0.026258086785674095,
      "rewards/rejected": -0.09263916313648224,
      "step": 13360
    },
    {
      "epoch": 0.5010211538101216,
      "grad_norm": 1.8612693205095479,
      "learning_rate": 6.918696553866751e-07,
      "log_odds_chosen": 0.556469738483429,
      "log_odds_ratio": -0.5830078125,
      "logits/chosen": -0.6668945550918579,
      "logits/rejected": -0.5606445074081421,
      "logps/chosen": -0.6629883050918579,
      "logps/rejected": -0.9857422113418579,
      "loss": 0.8938,
      "nll_loss": 0.841601550579071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0662841796875,
      "rewards/margins": 0.03220367431640625,
      "rewards/rejected": -0.0985107421875,
      "step": 13370
    },
    {
      "epoch": 0.5013958891532855,
      "grad_norm": 1.8056434882220773,
      "learning_rate": 6.916110608512408e-07,
      "log_odds_chosen": 0.48486328125,
      "log_odds_ratio": -0.5966796875,
      "logits/chosen": -0.735546886920929,
      "logits/rejected": -0.625195324420929,
      "logps/chosen": -0.7108398675918579,
      "logps/rejected": -1.028710961341858,
      "loss": 0.8976,
      "nll_loss": 0.8453124761581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07105712592601776,
      "rewards/margins": 0.03179321438074112,
      "rewards/rejected": -0.1029052734375,
      "step": 13380
    },
    {
      "epoch": 0.5017706244964494,
      "grad_norm": 1.7657145106155514,
      "learning_rate": 6.913527560575998e-07,
      "log_odds_chosen": 0.4920654296875,
      "log_odds_ratio": -0.587890625,
      "logits/chosen": -0.726367175579071,
      "logits/rejected": -0.64892578125,
      "logps/chosen": -0.694140613079071,
      "logps/rejected": -1.0,
      "loss": 0.9033,
      "nll_loss": 0.8294922113418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06939697265625,
      "rewards/margins": 0.03061370924115181,
      "rewards/rejected": -0.09995117038488388,
      "step": 13390
    },
    {
      "epoch": 0.5021453598396133,
      "grad_norm": 2.0646096521191613,
      "learning_rate": 6.910947404650881e-07,
      "log_odds_chosen": 0.704272449016571,
      "log_odds_ratio": -0.530957043170929,
      "logits/chosen": -0.7728515863418579,
      "logits/rejected": -0.6533203125,
      "logps/chosen": -0.6298828125,
      "logps/rejected": -1.0695312023162842,
      "loss": 0.8758,
      "nll_loss": 0.7782226800918579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06303711235523224,
      "rewards/margins": 0.043975830078125,
      "rewards/rejected": -0.10688476264476776,
      "step": 13400
    },
    {
      "epoch": 0.5025200951827772,
      "grad_norm": 1.9018647622680165,
      "learning_rate": 6.90837013534453e-07,
      "log_odds_chosen": 0.4906005859375,
      "log_odds_ratio": -0.6021484136581421,
      "logits/chosen": -0.729199230670929,
      "logits/rejected": -0.615527331829071,
      "logps/chosen": -0.689257800579071,
      "logps/rejected": -1.027734398841858,
      "loss": 0.9029,
      "nll_loss": 0.822265625,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06888427585363388,
      "rewards/margins": 0.033910371363162994,
      "rewards/rejected": -0.102783203125,
      "step": 13410
    },
    {
      "epoch": 0.5028948305259411,
      "grad_norm": 1.779284748434201,
      "learning_rate": 6.905795747278488e-07,
      "log_odds_chosen": 0.28608399629592896,
      "log_odds_ratio": -0.68017578125,
      "logits/chosen": -0.7037109136581421,
      "logits/rejected": -0.6513671875,
      "logps/chosen": -0.703320324420929,
      "logps/rejected": -0.9125000238418579,
      "loss": 0.8949,
      "nll_loss": 0.8626953363418579,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07033691555261612,
      "rewards/margins": 0.020865630358457565,
      "rewards/rejected": -0.0911865234375,
      "step": 13420
    },
    {
      "epoch": 0.503269565869105,
      "grad_norm": 1.9419016315582331,
      "learning_rate": 6.903224235088314e-07,
      "log_odds_chosen": 0.572644054889679,
      "log_odds_ratio": -0.5711914300918579,
      "logits/chosen": -0.733203113079071,
      "logits/rejected": -0.632617175579071,
      "logps/chosen": -0.6494140625,
      "logps/rejected": -1.0207030773162842,
      "loss": 0.898,
      "nll_loss": 0.8421875238418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06495361030101776,
      "rewards/margins": 0.037078857421875,
      "rewards/rejected": -0.10209961235523224,
      "step": 13430
    },
    {
      "epoch": 0.5036443012122689,
      "grad_norm": 1.7137541838693624,
      "learning_rate": 6.900655593423541e-07,
      "log_odds_chosen": 0.47581785917282104,
      "log_odds_ratio": -0.61083984375,
      "logits/chosen": -0.6791015863418579,
      "logits/rejected": -0.5674804449081421,
      "logps/chosen": -0.7051757574081421,
      "logps/rejected": -1.020898461341858,
      "loss": 0.8955,
      "nll_loss": 0.853515625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07049560546875,
      "rewards/margins": 0.03157348558306694,
      "rewards/rejected": -0.10198974609375,
      "step": 13440
    },
    {
      "epoch": 0.5040190365554327,
      "grad_norm": 1.7848107570751357,
      "learning_rate": 6.89808981694763e-07,
      "log_odds_chosen": 0.5166015625,
      "log_odds_ratio": -0.5921875238418579,
      "logits/chosen": -0.7015625238418579,
      "logits/rejected": -0.61474609375,
      "logps/chosen": -0.703320324420929,
      "logps/rejected": -1.024023413658142,
      "loss": 0.8827,
      "nll_loss": 0.813671886920929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07037353515625,
      "rewards/margins": 0.03214416652917862,
      "rewards/rejected": -0.10242919623851776,
      "step": 13450
    },
    {
      "epoch": 0.5043937718985966,
      "grad_norm": 1.8905338653148835,
      "learning_rate": 6.895526900337915e-07,
      "log_odds_chosen": 0.5633300542831421,
      "log_odds_ratio": -0.5723632574081421,
      "logits/chosen": -0.7203124761581421,
      "logits/rejected": -0.625683605670929,
      "logps/chosen": -0.7256835699081421,
      "logps/rejected": -1.073632836341858,
      "loss": 0.908,
      "nll_loss": 0.875781238079071,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07253418117761612,
      "rewards/margins": 0.03486175462603569,
      "rewards/rejected": -0.107421875,
      "step": 13460
    },
    {
      "epoch": 0.5047685072417605,
      "grad_norm": 2.5178094336666725,
      "learning_rate": 6.892966838285567e-07,
      "log_odds_chosen": 0.3377929627895355,
      "log_odds_ratio": -0.6376953125,
      "logits/chosen": -0.751953125,
      "logits/rejected": -0.6436523199081421,
      "logps/chosen": -0.688281238079071,
      "logps/rejected": -0.9273437261581421,
      "loss": 0.8919,
      "nll_loss": 0.7939453125,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.06879882514476776,
      "rewards/margins": 0.024010086432099342,
      "rewards/rejected": -0.09274902194738388,
      "step": 13470
    },
    {
      "epoch": 0.5051432425849244,
      "grad_norm": 2.020880651471387,
      "learning_rate": 6.890409625495545e-07,
      "log_odds_chosen": 0.4114746153354645,
      "log_odds_ratio": -0.6214843988418579,
      "logits/chosen": -0.724316418170929,
      "logits/rejected": -0.6312500238418579,
      "logps/chosen": -0.6962890625,
      "logps/rejected": -0.955078125,
      "loss": 0.9076,
      "nll_loss": 0.813281238079071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06962890923023224,
      "rewards/margins": 0.025949478149414062,
      "rewards/rejected": -0.09556885063648224,
      "step": 13480
    },
    {
      "epoch": 0.5055179779280883,
      "grad_norm": 1.8133619975309463,
      "learning_rate": 6.887855256686546e-07,
      "log_odds_chosen": 0.5124267339706421,
      "log_odds_ratio": -0.5908203125,
      "logits/chosen": -0.7118164300918579,
      "logits/rejected": -0.6180664300918579,
      "logps/chosen": -0.6802734136581421,
      "logps/rejected": -1.001953125,
      "loss": 0.8887,
      "nll_loss": 0.863476574420929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06805419921875,
      "rewards/margins": 0.032257080078125,
      "rewards/rejected": -0.10024414211511612,
      "step": 13490
    },
    {
      "epoch": 0.5058927132712522,
      "grad_norm": 1.9436322792031335,
      "learning_rate": 6.885303726590963e-07,
      "log_odds_chosen": 0.513354480266571,
      "log_odds_ratio": -0.600390613079071,
      "logits/chosen": -0.7120116949081421,
      "logits/rejected": -0.608691394329071,
      "logps/chosen": -0.720507800579071,
      "logps/rejected": -1.0478515625,
      "loss": 0.9085,
      "nll_loss": 0.870898425579071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07211913913488388,
      "rewards/margins": 0.03275604173541069,
      "rewards/rejected": -0.10478515923023224,
      "step": 13500
    },
    {
      "epoch": 0.5062674486144161,
      "grad_norm": 1.8622878036657096,
      "learning_rate": 6.882755029954837e-07,
      "log_odds_chosen": 0.4842285215854645,
      "log_odds_ratio": -0.576953113079071,
      "logits/chosen": -0.723828136920929,
      "logits/rejected": -0.5970703363418579,
      "logps/chosen": -0.6815429925918579,
      "logps/rejected": -0.998828113079071,
      "loss": 0.9182,
      "nll_loss": 0.885546863079071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06810303032398224,
      "rewards/margins": 0.03178558498620987,
      "rewards/rejected": -0.09995117038488388,
      "step": 13510
    },
    {
      "epoch": 0.50664218395758,
      "grad_norm": 1.84116038030232,
      "learning_rate": 6.880209161537815e-07,
      "log_odds_chosen": 0.37739259004592896,
      "log_odds_ratio": -0.638476550579071,
      "logits/chosen": -0.736523449420929,
      "logits/rejected": -0.642285168170929,
      "logps/chosen": -0.6451171636581421,
      "logps/rejected": -0.8804687261581421,
      "loss": 0.8916,
      "nll_loss": 0.8433593511581421,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06453857570886612,
      "rewards/margins": 0.02350311353802681,
      "rewards/rejected": -0.08796386420726776,
      "step": 13520
    },
    {
      "epoch": 0.5070169193007439,
      "grad_norm": 1.9070104679692954,
      "learning_rate": 6.877666116113097e-07,
      "log_odds_chosen": 0.4784912168979645,
      "log_odds_ratio": -0.6109374761581421,
      "logits/chosen": -0.7557617425918579,
      "logits/rejected": -0.6705077886581421,
      "logps/chosen": -0.6693359613418579,
      "logps/rejected": -0.9722656011581421,
      "loss": 0.904,
      "nll_loss": 0.8154296875,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06695556640625,
      "rewards/margins": 0.03019103966653347,
      "rewards/rejected": -0.09714355319738388,
      "step": 13530
    },
    {
      "epoch": 0.5073916546439078,
      "grad_norm": 1.6980142788034294,
      "learning_rate": 6.875125888467405e-07,
      "log_odds_chosen": 0.46333009004592896,
      "log_odds_ratio": -0.5936523675918579,
      "logits/chosen": -0.720507800579071,
      "logits/rejected": -0.6348632574081421,
      "logps/chosen": -0.708789050579071,
      "logps/rejected": -1.0066406726837158,
      "loss": 0.9248,
      "nll_loss": 0.846875011920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07088623195886612,
      "rewards/margins": 0.02979888953268528,
      "rewards/rejected": -0.10063476860523224,
      "step": 13540
    },
    {
      "epoch": 0.5077663899870716,
      "grad_norm": 1.7834438018238135,
      "learning_rate": 6.872588473400923e-07,
      "log_odds_chosen": 0.5238037109375,
      "log_odds_ratio": -0.600781261920929,
      "logits/chosen": -0.7037109136581421,
      "logits/rejected": -0.6214843988418579,
      "logps/chosen": -0.676953136920929,
      "logps/rejected": -1.018945336341858,
      "loss": 0.9158,
      "nll_loss": 0.819140613079071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06766357272863388,
      "rewards/margins": 0.034238435328006744,
      "rewards/rejected": -0.10191650688648224,
      "step": 13550
    },
    {
      "epoch": 0.5081411253302355,
      "grad_norm": 2.073389008685775,
      "learning_rate": 6.870053865727262e-07,
      "log_odds_chosen": 0.525390625,
      "log_odds_ratio": -0.5967773199081421,
      "logits/chosen": -0.7349609136581421,
      "logits/rejected": -0.616894543170929,
      "logps/chosen": -0.718066394329071,
      "logps/rejected": -1.087499976158142,
      "loss": 0.8939,
      "nll_loss": 0.911328136920929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07183837890625,
      "rewards/margins": 0.03688659518957138,
      "rewards/rejected": -0.10875244438648224,
      "step": 13560
    },
    {
      "epoch": 0.5085158606733994,
      "grad_norm": 1.9210529769678417,
      "learning_rate": 6.867522060273408e-07,
      "log_odds_chosen": 0.512072741985321,
      "log_odds_ratio": -0.584277331829071,
      "logits/chosen": -0.7251952886581421,
      "logits/rejected": -0.645214855670929,
      "logps/chosen": -0.6836913824081421,
      "logps/rejected": -1.0078125,
      "loss": 0.869,
      "nll_loss": 0.7740234136581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06837157905101776,
      "rewards/margins": 0.03241424635052681,
      "rewards/rejected": -0.10087890923023224,
      "step": 13570
    },
    {
      "epoch": 0.5088905960165633,
      "grad_norm": 1.940459000072061,
      "learning_rate": 6.864993051879688e-07,
      "log_odds_chosen": 0.589648425579071,
      "log_odds_ratio": -0.566699206829071,
      "logits/chosen": -0.722949206829071,
      "logits/rejected": -0.627246081829071,
      "logps/chosen": -0.6756836175918579,
      "logps/rejected": -1.049414038658142,
      "loss": 0.8901,
      "nll_loss": 0.8177734613418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06759033352136612,
      "rewards/margins": 0.03732147067785263,
      "rewards/rejected": -0.10493163764476776,
      "step": 13580
    },
    {
      "epoch": 0.5092653313597272,
      "grad_norm": 1.7271974008510462,
      "learning_rate": 6.862466835399716e-07,
      "log_odds_chosen": 0.564135730266571,
      "log_odds_ratio": -0.5810546875,
      "logits/chosen": -0.7333984375,
      "logits/rejected": -0.660937488079071,
      "logps/chosen": -0.6832031011581421,
      "logps/rejected": -1.0460937023162842,
      "loss": 0.8805,
      "nll_loss": 0.7802734375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06832275539636612,
      "rewards/margins": 0.036234281957149506,
      "rewards/rejected": -0.10465087741613388,
      "step": 13590
    },
    {
      "epoch": 0.5096400667028911,
      "grad_norm": 1.8001792893582427,
      "learning_rate": 6.859943405700353e-07,
      "log_odds_chosen": 0.4710693359375,
      "log_odds_ratio": -0.601757824420929,
      "logits/chosen": -0.7505859136581421,
      "logits/rejected": -0.6434570550918579,
      "logps/chosen": -0.675488293170929,
      "logps/rejected": -0.969921886920929,
      "loss": 0.9042,
      "nll_loss": 0.833203136920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06755371391773224,
      "rewards/margins": 0.02938384935259819,
      "rewards/rejected": -0.09698486328125,
      "step": 13600
    },
    {
      "epoch": 0.510014802046055,
      "grad_norm": 1.8574283159973788,
      "learning_rate": 6.857422757661664e-07,
      "log_odds_chosen": 0.398406982421875,
      "log_odds_ratio": -0.6332031488418579,
      "logits/chosen": -0.754101574420929,
      "logits/rejected": -0.663867175579071,
      "logps/chosen": -0.69140625,
      "logps/rejected": -0.9619140625,
      "loss": 0.8796,
      "nll_loss": 0.828320324420929,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06916503608226776,
      "rewards/margins": 0.027027130126953125,
      "rewards/rejected": -0.09617920219898224,
      "step": 13610
    },
    {
      "epoch": 0.5103895373892189,
      "grad_norm": 2.187736123094349,
      "learning_rate": 6.854904886176873e-07,
      "log_odds_chosen": 0.47004395723342896,
      "log_odds_ratio": -0.5967773199081421,
      "logits/chosen": -0.707812488079071,
      "logits/rejected": -0.621386706829071,
      "logps/chosen": -0.655957043170929,
      "logps/rejected": -0.9437500238418579,
      "loss": 0.8647,
      "nll_loss": 0.7718750238418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06562499701976776,
      "rewards/margins": 0.02865905687212944,
      "rewards/rejected": -0.09434814751148224,
      "step": 13620
    },
    {
      "epoch": 0.5107642727323828,
      "grad_norm": 1.9695948621400503,
      "learning_rate": 6.85238978615232e-07,
      "log_odds_chosen": 0.572338879108429,
      "log_odds_ratio": -0.576464831829071,
      "logits/chosen": -0.7457031011581421,
      "logits/rejected": -0.6133788824081421,
      "logps/chosen": -0.7041991949081421,
      "logps/rejected": -1.0851562023162842,
      "loss": 0.901,
      "nll_loss": 0.8101562261581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07042236626148224,
      "rewards/margins": 0.03812713548541069,
      "rewards/rejected": -0.10854492336511612,
      "step": 13630
    },
    {
      "epoch": 0.5111390080755467,
      "grad_norm": 1.6874735769880636,
      "learning_rate": 6.849877452507417e-07,
      "log_odds_chosen": 0.43250733613967896,
      "log_odds_ratio": -0.60595703125,
      "logits/chosen": -0.7291015386581421,
      "logits/rejected": -0.6499999761581421,
      "logps/chosen": -0.712109386920929,
      "logps/rejected": -0.977734386920929,
      "loss": 0.8986,
      "nll_loss": 0.8984375,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.0711669921875,
      "rewards/margins": 0.026650238782167435,
      "rewards/rejected": -0.09792480617761612,
      "step": 13640
    },
    {
      "epoch": 0.5115137434187106,
      "grad_norm": 1.8697362585306763,
      "learning_rate": 6.847367880174605e-07,
      "log_odds_chosen": 0.508313000202179,
      "log_odds_ratio": -0.585742175579071,
      "logits/chosen": -0.72265625,
      "logits/rejected": -0.621289074420929,
      "logps/chosen": -0.704785168170929,
      "logps/rejected": -1.044921875,
      "loss": 0.9006,
      "nll_loss": 0.874218761920929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07048340141773224,
      "rewards/margins": 0.03400878980755806,
      "rewards/rejected": -0.10452880710363388,
      "step": 13650
    },
    {
      "epoch": 0.5118884787618744,
      "grad_norm": 1.7779187782319394,
      "learning_rate": 6.844861064099317e-07,
      "log_odds_chosen": 0.46477049589157104,
      "log_odds_ratio": -0.5928710699081421,
      "logits/chosen": -0.7779296636581421,
      "logits/rejected": -0.6768554449081421,
      "logps/chosen": -0.6986328363418579,
      "logps/rejected": -0.98046875,
      "loss": 0.892,
      "nll_loss": 0.8365234136581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06986083835363388,
      "rewards/margins": 0.02819061279296875,
      "rewards/rejected": -0.09805908054113388,
      "step": 13660
    },
    {
      "epoch": 0.5122632141050383,
      "grad_norm": 1.7599809842757894,
      "learning_rate": 6.842356999239922e-07,
      "log_odds_chosen": 0.523571789264679,
      "log_odds_ratio": -0.591992199420929,
      "logits/chosen": -0.763964831829071,
      "logits/rejected": -0.647216796875,
      "logps/chosen": -0.6675781011581421,
      "logps/rejected": -0.999804675579071,
      "loss": 0.8961,
      "nll_loss": 0.814257800579071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06668701022863388,
      "rewards/margins": 0.03330840915441513,
      "rewards/rejected": -0.10008545219898224,
      "step": 13670
    },
    {
      "epoch": 0.5126379494482022,
      "grad_norm": 1.9740417174901146,
      "learning_rate": 6.839855680567694e-07,
      "log_odds_chosen": 0.45097655057907104,
      "log_odds_ratio": -0.606738269329071,
      "logits/chosen": -0.823925793170929,
      "logits/rejected": -0.6783202886581421,
      "logps/chosen": -0.6498047113418579,
      "logps/rejected": -0.908984363079071,
      "loss": 0.9014,
      "nll_loss": 0.7544921636581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06499023735523224,
      "rewards/margins": 0.02588195726275444,
      "rewards/rejected": -0.09088134765625,
      "step": 13680
    },
    {
      "epoch": 0.5130126847913661,
      "grad_norm": 2.0459976553442867,
      "learning_rate": 6.837357103066766e-07,
      "log_odds_chosen": 0.4773803651332855,
      "log_odds_ratio": -0.6024414300918579,
      "logits/chosen": -0.756054699420929,
      "logits/rejected": -0.69384765625,
      "logps/chosen": -0.7564452886581421,
      "logps/rejected": -1.048242211341858,
      "loss": 0.9047,
      "nll_loss": 0.8568359613418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07557372748851776,
      "rewards/margins": 0.029158782213926315,
      "rewards/rejected": -0.10466308891773224,
      "step": 13690
    },
    {
      "epoch": 0.51338742013453,
      "grad_norm": 1.8999680259922738,
      "learning_rate": 6.834861261734087e-07,
      "log_odds_chosen": 0.4099975526332855,
      "log_odds_ratio": -0.621874988079071,
      "logits/chosen": -0.7298828363418579,
      "logits/rejected": -0.61767578125,
      "logps/chosen": -0.704882800579071,
      "logps/rejected": -0.96875,
      "loss": 0.9113,
      "nll_loss": 0.8550781011581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07047118991613388,
      "rewards/margins": 0.026325225830078125,
      "rewards/rejected": -0.09687499701976776,
      "step": 13700
    },
    {
      "epoch": 0.5137621554776939,
      "grad_norm": 1.947611048798988,
      "learning_rate": 6.832368151579382e-07,
      "log_odds_chosen": 0.4212646484375,
      "log_odds_ratio": -0.58984375,
      "logits/chosen": -0.781054675579071,
      "logits/rejected": -0.655957043170929,
      "logps/chosen": -0.666796863079071,
      "logps/rejected": -0.9185546636581421,
      "loss": 0.9012,
      "nll_loss": 0.8017578125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06671142578125,
      "rewards/margins": 0.02514190599322319,
      "rewards/rejected": -0.09185791015625,
      "step": 13710
    },
    {
      "epoch": 0.5141368908208578,
      "grad_norm": 2.0042390078770302,
      "learning_rate": 6.829877767625106e-07,
      "log_odds_chosen": 0.38134765625,
      "log_odds_ratio": -0.6114257574081421,
      "logits/chosen": -0.714062511920929,
      "logits/rejected": -0.6117187738418579,
      "logps/chosen": -0.6573241949081421,
      "logps/rejected": -0.8734375238418579,
      "loss": 0.9063,
      "nll_loss": 0.821484386920929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06572265923023224,
      "rewards/margins": 0.021631622686982155,
      "rewards/rejected": -0.08734130859375,
      "step": 13720
    },
    {
      "epoch": 0.5145116261640217,
      "grad_norm": 2.050134585427926,
      "learning_rate": 6.827390104906408e-07,
      "log_odds_chosen": 0.5143676996231079,
      "log_odds_ratio": -0.5810546875,
      "logits/chosen": -0.764941394329071,
      "logits/rejected": -0.648730456829071,
      "logps/chosen": -0.679882824420929,
      "logps/rejected": -1.0115234851837158,
      "loss": 0.8878,
      "nll_loss": 0.7945312261581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06798096001148224,
      "rewards/margins": 0.033111572265625,
      "rewards/rejected": -0.10109863430261612,
      "step": 13730
    },
    {
      "epoch": 0.5148863615071856,
      "grad_norm": 2.0832379419517837,
      "learning_rate": 6.824905158471082e-07,
      "log_odds_chosen": 0.549877941608429,
      "log_odds_ratio": -0.5794922113418579,
      "logits/chosen": -0.721484363079071,
      "logits/rejected": -0.636523425579071,
      "logps/chosen": -0.7005859613418579,
      "logps/rejected": -1.0324218273162842,
      "loss": 0.8779,
      "nll_loss": 0.9097656011581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07000732421875,
      "rewards/margins": 0.033271025866270065,
      "rewards/rejected": -0.10324706882238388,
      "step": 13740
    },
    {
      "epoch": 0.5152610968503495,
      "grad_norm": 1.913987905889767,
      "learning_rate": 6.822422923379532e-07,
      "log_odds_chosen": 0.561206042766571,
      "log_odds_ratio": -0.5694335699081421,
      "logits/chosen": -0.7835937738418579,
      "logits/rejected": -0.681835949420929,
      "logps/chosen": -0.6513671875,
      "logps/rejected": -1.0095703601837158,
      "loss": 0.911,
      "nll_loss": 0.8335937261581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06514892727136612,
      "rewards/margins": 0.03574676439166069,
      "rewards/rejected": -0.10087890923023224,
      "step": 13750
    },
    {
      "epoch": 0.5156358321935133,
      "grad_norm": 1.8927024375145685,
      "learning_rate": 6.819943394704734e-07,
      "log_odds_chosen": 0.41997069120407104,
      "log_odds_ratio": -0.62255859375,
      "logits/chosen": -0.76171875,
      "logits/rejected": -0.6865234375,
      "logps/chosen": -0.6700195074081421,
      "logps/rejected": -0.925976574420929,
      "loss": 0.897,
      "nll_loss": 0.8355468511581421,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06696777045726776,
      "rewards/margins": 0.02567596361041069,
      "rewards/rejected": -0.09272460639476776,
      "step": 13760
    },
    {
      "epoch": 0.5160105675366772,
      "grad_norm": 1.8895795324261289,
      "learning_rate": 6.817466567532181e-07,
      "log_odds_chosen": 0.521282970905304,
      "log_odds_ratio": -0.6045898199081421,
      "logits/chosen": -0.7837890386581421,
      "logits/rejected": -0.6646484136581421,
      "logps/chosen": -0.6573241949081421,
      "logps/rejected": -0.9769531488418579,
      "loss": 0.887,
      "nll_loss": 0.8232421875,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06577148288488388,
      "rewards/margins": 0.03185577318072319,
      "rewards/rejected": -0.09760741889476776,
      "step": 13770
    },
    {
      "epoch": 0.5163853028798411,
      "grad_norm": 1.8732693072795303,
      "learning_rate": 6.814992436959855e-07,
      "log_odds_chosen": 0.4423584043979645,
      "log_odds_ratio": -0.603808581829071,
      "logits/chosen": -0.7509765625,
      "logits/rejected": -0.6273437738418579,
      "logps/chosen": -0.680859386920929,
      "logps/rejected": -0.958984375,
      "loss": 0.8878,
      "nll_loss": 0.828125,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06810303032398224,
      "rewards/margins": 0.027881432324647903,
      "rewards/rejected": -0.09603271633386612,
      "step": 13780
    },
    {
      "epoch": 0.516760038223005,
      "grad_norm": 1.7836276367380546,
      "learning_rate": 6.812520998098182e-07,
      "log_odds_chosen": 0.5430968999862671,
      "log_odds_ratio": -0.586132824420929,
      "logits/chosen": -0.765820324420929,
      "logits/rejected": -0.639941394329071,
      "logps/chosen": -0.7054687738418579,
      "logps/rejected": -1.0480468273162842,
      "loss": 0.8921,
      "nll_loss": 0.8431640863418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.070556640625,
      "rewards/margins": 0.03425407409667969,
      "rewards/rejected": -0.10477294772863388,
      "step": 13790
    },
    {
      "epoch": 0.5171347735661689,
      "grad_norm": 1.7222681006995555,
      "learning_rate": 6.810052246069989e-07,
      "log_odds_chosen": 0.3953002989292145,
      "log_odds_ratio": -0.63916015625,
      "logits/chosen": -0.764453113079071,
      "logits/rejected": -0.643750011920929,
      "logps/chosen": -0.73876953125,
      "logps/rejected": -0.998046875,
      "loss": 0.904,
      "nll_loss": 0.8833984136581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07391357421875,
      "rewards/margins": 0.02596130408346653,
      "rewards/rejected": -0.099853515625,
      "step": 13800
    },
    {
      "epoch": 0.5175095089093328,
      "grad_norm": 1.8001650431685141,
      "learning_rate": 6.807586176010469e-07,
      "log_odds_chosen": 0.4617065489292145,
      "log_odds_ratio": -0.6083984375,
      "logits/chosen": -0.7730468511581421,
      "logits/rejected": -0.6356445550918579,
      "logps/chosen": -0.654589831829071,
      "logps/rejected": -0.9654296636581421,
      "loss": 0.8964,
      "nll_loss": 0.775390625,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06546630710363388,
      "rewards/margins": 0.03107910230755806,
      "rewards/rejected": -0.09658203274011612,
      "step": 13810
    },
    {
      "epoch": 0.5178842442524967,
      "grad_norm": 2.006176005663609,
      "learning_rate": 6.805122783067135e-07,
      "log_odds_chosen": 0.503430187702179,
      "log_odds_ratio": -0.59716796875,
      "logits/chosen": -0.77734375,
      "logits/rejected": -0.6910156011581421,
      "logps/chosen": -0.706835925579071,
      "logps/rejected": -1.045800805091858,
      "loss": 0.9067,
      "nll_loss": 0.8267577886581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07064209133386612,
      "rewards/margins": 0.033889006823301315,
      "rewards/rejected": -0.10454101860523224,
      "step": 13820
    },
    {
      "epoch": 0.5182589795956606,
      "grad_norm": 1.8951827250829612,
      "learning_rate": 6.802662062399785e-07,
      "log_odds_chosen": 0.5112670660018921,
      "log_odds_ratio": -0.595898449420929,
      "logits/chosen": -0.737500011920929,
      "logits/rejected": -0.6556640863418579,
      "logps/chosen": -0.6556640863418579,
      "logps/rejected": -0.9921875,
      "loss": 0.9217,
      "nll_loss": 0.8564453125,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06557617336511612,
      "rewards/margins": 0.033670805394649506,
      "rewards/rejected": -0.09929199516773224,
      "step": 13830
    },
    {
      "epoch": 0.5186337149388245,
      "grad_norm": 1.8628425155185748,
      "learning_rate": 6.800204009180459e-07,
      "log_odds_chosen": 0.34907227754592896,
      "log_odds_ratio": -0.63623046875,
      "logits/chosen": -0.7699218988418579,
      "logits/rejected": -0.657421886920929,
      "logps/chosen": -0.6768554449081421,
      "logps/rejected": -0.910351574420929,
      "loss": 0.8705,
      "nll_loss": 0.7724609375,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.06763915717601776,
      "rewards/margins": 0.023395538330078125,
      "rewards/rejected": -0.09112548828125,
      "step": 13840
    },
    {
      "epoch": 0.5190084502819884,
      "grad_norm": 2.3527882567755665,
      "learning_rate": 6.797748618593397e-07,
      "log_odds_chosen": 0.601025402545929,
      "log_odds_ratio": -0.571484386920929,
      "logits/chosen": -0.774609386920929,
      "logits/rejected": -0.667285144329071,
      "logps/chosen": -0.645800769329071,
      "logps/rejected": -1.042382836341858,
      "loss": 0.8841,
      "nll_loss": 0.7777343988418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06455077975988388,
      "rewards/margins": 0.03973083570599556,
      "rewards/rejected": -0.10419921576976776,
      "step": 13850
    },
    {
      "epoch": 0.5193831856251523,
      "grad_norm": 1.7826032287778923,
      "learning_rate": 6.795295885835008e-07,
      "log_odds_chosen": 0.46821290254592896,
      "log_odds_ratio": -0.607226550579071,
      "logits/chosen": -0.718554675579071,
      "logits/rejected": -0.607421875,
      "logps/chosen": -0.65625,
      "logps/rejected": -0.939648449420929,
      "loss": 0.8927,
      "nll_loss": 0.7925781011581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06556396186351776,
      "rewards/margins": 0.02835388109087944,
      "rewards/rejected": -0.09396972507238388,
      "step": 13860
    },
    {
      "epoch": 0.5197579209683161,
      "grad_norm": 1.769549407597115,
      "learning_rate": 6.792845806113816e-07,
      "log_odds_chosen": 0.6019042730331421,
      "log_odds_ratio": -0.5536133050918579,
      "logits/chosen": -0.7705078125,
      "logits/rejected": -0.6488281488418579,
      "logps/chosen": -0.682421863079071,
      "logps/rejected": -1.0652344226837158,
      "loss": 0.8858,
      "nll_loss": 0.8121093511581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06824950873851776,
      "rewards/margins": 0.03826294094324112,
      "rewards/rejected": -0.10651855170726776,
      "step": 13870
    },
    {
      "epoch": 0.52013265631148,
      "grad_norm": 1.945936007660605,
      "learning_rate": 6.790398374650439e-07,
      "log_odds_chosen": 0.514514148235321,
      "log_odds_ratio": -0.586230456829071,
      "logits/chosen": -0.758007824420929,
      "logits/rejected": -0.6673828363418579,
      "logps/chosen": -0.6600586175918579,
      "logps/rejected": -0.973828136920929,
      "loss": 0.8963,
      "nll_loss": 0.849804699420929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06601562350988388,
      "rewards/margins": 0.03139038011431694,
      "rewards/rejected": -0.09739990532398224,
      "step": 13880
    },
    {
      "epoch": 0.5205073916546439,
      "grad_norm": 1.9315506572355357,
      "learning_rate": 6.787953586677534e-07,
      "log_odds_chosen": 0.5420166254043579,
      "log_odds_ratio": -0.5816406011581421,
      "logits/chosen": -0.71484375,
      "logits/rejected": -0.584765613079071,
      "logps/chosen": -0.6953125,
      "logps/rejected": -1.0439453125,
      "loss": 0.8867,
      "nll_loss": 0.815234363079071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06950683891773224,
      "rewards/margins": 0.03498268127441406,
      "rewards/rejected": -0.10446777194738388,
      "step": 13890
    },
    {
      "epoch": 0.5208821269978078,
      "grad_norm": 2.040351230951985,
      "learning_rate": 6.785511437439767e-07,
      "log_odds_chosen": 0.56231689453125,
      "log_odds_ratio": -0.568554699420929,
      "logits/chosen": -0.711718738079071,
      "logits/rejected": -0.614550769329071,
      "logps/chosen": -0.699414074420929,
      "logps/rejected": -1.0437500476837158,
      "loss": 0.8871,
      "nll_loss": 0.874218761920929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0699462890625,
      "rewards/margins": 0.03441619873046875,
      "rewards/rejected": -0.1043701171875,
      "step": 13900
    },
    {
      "epoch": 0.5212568623409717,
      "grad_norm": 1.8685160917311046,
      "learning_rate": 6.78307192219377e-07,
      "log_odds_chosen": 0.546337902545929,
      "log_odds_ratio": -0.5829101800918579,
      "logits/chosen": -0.7738281488418579,
      "logits/rejected": -0.644726574420929,
      "logps/chosen": -0.67822265625,
      "logps/rejected": -1.016210913658142,
      "loss": 0.8953,
      "nll_loss": 0.8330078125,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06779785454273224,
      "rewards/margins": 0.03384704515337944,
      "rewards/rejected": -0.10161133110523224,
      "step": 13910
    },
    {
      "epoch": 0.5216315976841356,
      "grad_norm": 1.956083966468999,
      "learning_rate": 6.780635036208104e-07,
      "log_odds_chosen": 0.517120361328125,
      "log_odds_ratio": -0.569140613079071,
      "logits/chosen": -0.714648425579071,
      "logits/rejected": -0.62646484375,
      "logps/chosen": -0.6664062738418579,
      "logps/rejected": -0.970703125,
      "loss": 0.8833,
      "nll_loss": 0.812695324420929,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06661377102136612,
      "rewards/margins": 0.03045959398150444,
      "rewards/rejected": -0.09710693359375,
      "step": 13920
    },
    {
      "epoch": 0.5220063330272995,
      "grad_norm": 2.167917869158354,
      "learning_rate": 6.77820077476322e-07,
      "log_odds_chosen": 0.523364245891571,
      "log_odds_ratio": -0.595507800579071,
      "logits/chosen": -0.761523425579071,
      "logits/rejected": -0.64453125,
      "logps/chosen": -0.6583007574081421,
      "logps/rejected": -0.988085925579071,
      "loss": 0.8978,
      "nll_loss": 0.8173828125,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06584472954273224,
      "rewards/margins": 0.033032990992069244,
      "rewards/rejected": -0.09880371391773224,
      "step": 13930
    },
    {
      "epoch": 0.5223810683704634,
      "grad_norm": 1.8864542053189186,
      "learning_rate": 6.775769133151421e-07,
      "log_odds_chosen": 0.4425292909145355,
      "log_odds_ratio": -0.5933593511581421,
      "logits/chosen": -0.728320300579071,
      "logits/rejected": -0.6348632574081421,
      "logps/chosen": -0.6673828363418579,
      "logps/rejected": -0.92333984375,
      "loss": 0.8975,
      "nll_loss": 0.849414050579071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06673584133386612,
      "rewards/margins": 0.02560882642865181,
      "rewards/rejected": -0.09238281100988388,
      "step": 13940
    },
    {
      "epoch": 0.5227558037136273,
      "grad_norm": 1.9531875661957128,
      "learning_rate": 6.773340106676827e-07,
      "log_odds_chosen": 0.527252197265625,
      "log_odds_ratio": -0.5882812738418579,
      "logits/chosen": -0.7470703125,
      "logits/rejected": -0.6612304449081421,
      "logps/chosen": -0.6737304925918579,
      "logps/rejected": -1.014257788658142,
      "loss": 0.8923,
      "nll_loss": 0.863476574420929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06730957329273224,
      "rewards/margins": 0.034082792699337006,
      "rewards/rejected": -0.10148926079273224,
      "step": 13950
    },
    {
      "epoch": 0.5231305390567912,
      "grad_norm": 2.0545579257244198,
      "learning_rate": 6.770913690655326e-07,
      "log_odds_chosen": 0.4415039122104645,
      "log_odds_ratio": -0.604296863079071,
      "logits/chosen": -0.7406250238418579,
      "logits/rejected": -0.6353515386581421,
      "logps/chosen": -0.605273425579071,
      "logps/rejected": -0.867382824420929,
      "loss": 0.9089,
      "nll_loss": 0.788281261920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06055908277630806,
      "rewards/margins": 0.02622222900390625,
      "rewards/rejected": -0.08673095703125,
      "step": 13960
    },
    {
      "epoch": 0.523505274399955,
      "grad_norm": 1.9213148189306142,
      "learning_rate": 6.768489880414549e-07,
      "log_odds_chosen": 0.5657714605331421,
      "log_odds_ratio": -0.573925793170929,
      "logits/chosen": -0.7203124761581421,
      "logits/rejected": -0.6162109375,
      "logps/chosen": -0.7269531488418579,
      "logps/rejected": -1.0998046398162842,
      "loss": 0.8801,
      "nll_loss": 0.8150390386581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07266845554113388,
      "rewards/margins": 0.03730926662683487,
      "rewards/rejected": -0.10997314751148224,
      "step": 13970
    },
    {
      "epoch": 0.5238800097431189,
      "grad_norm": 1.8890523939300317,
      "learning_rate": 6.766068671293825e-07,
      "log_odds_chosen": 0.546069324016571,
      "log_odds_ratio": -0.5760742425918579,
      "logits/chosen": -0.7330077886581421,
      "logits/rejected": -0.6507812738418579,
      "logps/chosen": -0.7139648199081421,
      "logps/rejected": -1.064453125,
      "loss": 0.8941,
      "nll_loss": 0.8876953125,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07142333686351776,
      "rewards/margins": 0.03518066555261612,
      "rewards/rejected": -0.10648193210363388,
      "step": 13980
    },
    {
      "epoch": 0.5242547450862828,
      "grad_norm": 1.8904785043464039,
      "learning_rate": 6.763650058644149e-07,
      "log_odds_chosen": 0.5111083984375,
      "log_odds_ratio": -0.5699218511581421,
      "logits/chosen": -0.741894543170929,
      "logits/rejected": -0.6532226800918579,
      "logps/chosen": -0.6654297113418579,
      "logps/rejected": -0.974414050579071,
      "loss": 0.8676,
      "nll_loss": 0.76171875,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06655273586511612,
      "rewards/margins": 0.03086242638528347,
      "rewards/rejected": -0.097412109375,
      "step": 13990
    },
    {
      "epoch": 0.5246294804294467,
      "grad_norm": 1.8283576482051545,
      "learning_rate": 6.761234037828132e-07,
      "log_odds_chosen": 0.27301025390625,
      "log_odds_ratio": -0.6724609136581421,
      "logits/chosen": -0.7538086175918579,
      "logits/rejected": -0.6675781011581421,
      "logps/chosen": -0.713183581829071,
      "logps/rejected": -0.9009765386581421,
      "loss": 0.8826,
      "nll_loss": 0.8541015386581421,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07132568210363388,
      "rewards/margins": 0.01870880089700222,
      "rewards/rejected": -0.09002685546875,
      "step": 14000
    },
    {
      "epoch": 0.5250042157726106,
      "grad_norm": 2.0266836865794575,
      "learning_rate": 6.758820604219981e-07,
      "log_odds_chosen": 0.40614014863967896,
      "log_odds_ratio": -0.64453125,
      "logits/chosen": -0.736523449420929,
      "logits/rejected": -0.632031261920929,
      "logps/chosen": -0.711718738079071,
      "logps/rejected": -0.964648425579071,
      "loss": 0.8862,
      "nll_loss": 0.8216797113418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07117919623851776,
      "rewards/margins": 0.025252532213926315,
      "rewards/rejected": -0.09641113132238388,
      "step": 14010
    },
    {
      "epoch": 0.5253789511157745,
      "grad_norm": 2.909898089262287,
      "learning_rate": 6.756409753205447e-07,
      "log_odds_chosen": 0.527417004108429,
      "log_odds_ratio": -0.5799804925918579,
      "logits/chosen": -0.7353515625,
      "logits/rejected": -0.629199206829071,
      "logps/chosen": -0.6845703125,
      "logps/rejected": -1.027929663658142,
      "loss": 0.8934,
      "nll_loss": 0.8388671875,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06838379055261612,
      "rewards/margins": 0.03428840637207031,
      "rewards/rejected": -0.102783203125,
      "step": 14020
    },
    {
      "epoch": 0.5257536864589384,
      "grad_norm": 1.8916759705411275,
      "learning_rate": 6.754001480181798e-07,
      "log_odds_chosen": 0.4525390565395355,
      "log_odds_ratio": -0.5887695550918579,
      "logits/chosen": -0.8089843988418579,
      "logits/rejected": -0.660839855670929,
      "logps/chosen": -0.6834961175918579,
      "logps/rejected": -0.963671863079071,
      "loss": 0.874,
      "nll_loss": 0.8197265863418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06831054389476776,
      "rewards/margins": 0.02799224853515625,
      "rewards/rejected": -0.09633789211511612,
      "step": 14030
    },
    {
      "epoch": 0.5261284218021023,
      "grad_norm": 1.943123884636666,
      "learning_rate": 6.751595780557777e-07,
      "log_odds_chosen": 0.6471923589706421,
      "log_odds_ratio": -0.557324230670929,
      "logits/chosen": -0.740039050579071,
      "logits/rejected": -0.626953125,
      "logps/chosen": -0.7152343988418579,
      "logps/rejected": -1.1259765625,
      "loss": 0.9004,
      "nll_loss": 0.8080078363418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07149658352136612,
      "rewards/margins": 0.04105529934167862,
      "rewards/rejected": -0.11258544772863388,
      "step": 14040
    },
    {
      "epoch": 0.5265031571452662,
      "grad_norm": 1.9392637883971706,
      "learning_rate": 6.749192649753564e-07,
      "log_odds_chosen": 0.4749511778354645,
      "log_odds_ratio": -0.59375,
      "logits/chosen": -0.718554675579071,
      "logits/rejected": -0.646679699420929,
      "logps/chosen": -0.664843738079071,
      "logps/rejected": -0.9681640863418579,
      "loss": 0.8653,
      "nll_loss": 0.8134765625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06651611626148224,
      "rewards/margins": 0.03025360032916069,
      "rewards/rejected": -0.0968017578125,
      "step": 14050
    },
    {
      "epoch": 0.5268778924884301,
      "grad_norm": 1.7079133477793558,
      "learning_rate": 6.746792083200745e-07,
      "log_odds_chosen": 0.4922119081020355,
      "log_odds_ratio": -0.608691394329071,
      "logits/chosen": -0.7476562261581421,
      "logits/rejected": -0.6537109613418579,
      "logps/chosen": -0.67138671875,
      "logps/rejected": -1.0001952648162842,
      "loss": 0.9009,
      "nll_loss": 0.829882800579071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06715087592601776,
      "rewards/margins": 0.032886505126953125,
      "rewards/rejected": -0.10004882514476776,
      "step": 14060
    },
    {
      "epoch": 0.527252627831594,
      "grad_norm": 1.9866591534645053,
      "learning_rate": 6.74439407634227e-07,
      "log_odds_chosen": 0.4680542051792145,
      "log_odds_ratio": -0.591992199420929,
      "logits/chosen": -0.7339843511581421,
      "logits/rejected": -0.62451171875,
      "logps/chosen": -0.6802734136581421,
      "logps/rejected": -0.968554675579071,
      "loss": 0.9017,
      "nll_loss": 0.851367175579071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06804199516773224,
      "rewards/margins": 0.02873840369284153,
      "rewards/rejected": -0.09672851860523224,
      "step": 14070
    },
    {
      "epoch": 0.5276273631747578,
      "grad_norm": 2.113187278200605,
      "learning_rate": 6.74199862463242e-07,
      "log_odds_chosen": 0.4922851622104645,
      "log_odds_ratio": -0.6011718511581421,
      "logits/chosen": -0.7685546875,
      "logits/rejected": -0.6546875238418579,
      "logps/chosen": -0.664843738079071,
      "logps/rejected": -0.98974609375,
      "loss": 0.8875,
      "nll_loss": 0.836132824420929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06647948920726776,
      "rewards/margins": 0.03245697170495987,
      "rewards/rejected": -0.098876953125,
      "step": 14080
    },
    {
      "epoch": 0.5280020985179217,
      "grad_norm": 2.3216763132557428,
      "learning_rate": 6.739605723536771e-07,
      "log_odds_chosen": 0.4786376953125,
      "log_odds_ratio": -0.598925769329071,
      "logits/chosen": -0.744140625,
      "logits/rejected": -0.6456054449081421,
      "logps/chosen": -0.7007812261581421,
      "logps/rejected": -1.029296875,
      "loss": 0.8694,
      "nll_loss": 0.832226574420929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07000732421875,
      "rewards/margins": 0.032814789563417435,
      "rewards/rejected": -0.10283203423023224,
      "step": 14090
    },
    {
      "epoch": 0.5283768338610856,
      "grad_norm": 2.002962135243527,
      "learning_rate": 6.737215368532152e-07,
      "log_odds_chosen": 0.5777343511581421,
      "log_odds_ratio": -0.579296886920929,
      "logits/chosen": -0.749707043170929,
      "logits/rejected": -0.624707043170929,
      "logps/chosen": -0.67041015625,
      "logps/rejected": -1.053125023841858,
      "loss": 0.8836,
      "nll_loss": 0.8296874761581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06706543266773224,
      "rewards/margins": 0.038285065442323685,
      "rewards/rejected": -0.10535888373851776,
      "step": 14100
    },
    {
      "epoch": 0.5287515692042495,
      "grad_norm": 2.0140249015583103,
      "learning_rate": 6.734827555106618e-07,
      "log_odds_chosen": 0.627368152141571,
      "log_odds_ratio": -0.57470703125,
      "logits/chosen": -0.7245117425918579,
      "logits/rejected": -0.6192382574081421,
      "logps/chosen": -0.6795898675918579,
      "logps/rejected": -1.098046898841858,
      "loss": 0.8913,
      "nll_loss": 0.8232421875,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06790771335363388,
      "rewards/margins": 0.04191894456744194,
      "rewards/rejected": -0.10991211235523224,
      "step": 14110
    },
    {
      "epoch": 0.5291263045474134,
      "grad_norm": 2.1668414326856587,
      "learning_rate": 6.73244227875941e-07,
      "log_odds_chosen": 0.4648071229457855,
      "log_odds_ratio": -0.615234375,
      "logits/chosen": -0.761523425579071,
      "logits/rejected": -0.640625,
      "logps/chosen": -0.653515636920929,
      "logps/rejected": -0.96484375,
      "loss": 0.864,
      "nll_loss": 0.791210949420929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06536865234375,
      "rewards/margins": 0.03113098070025444,
      "rewards/rejected": -0.09652099758386612,
      "step": 14120
    },
    {
      "epoch": 0.5295010398905773,
      "grad_norm": 2.050674149174158,
      "learning_rate": 6.730059535000916e-07,
      "log_odds_chosen": 0.46479493379592896,
      "log_odds_ratio": -0.5986328125,
      "logits/chosen": -0.7181640863418579,
      "logits/rejected": -0.607714831829071,
      "logps/chosen": -0.679980456829071,
      "logps/rejected": -0.970507800579071,
      "loss": 0.8868,
      "nll_loss": 0.8619140386581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06805419921875,
      "rewards/margins": 0.029003143310546875,
      "rewards/rejected": -0.0970458984375,
      "step": 14130
    },
    {
      "epoch": 0.5298757752337412,
      "grad_norm": 2.0571523873743476,
      "learning_rate": 6.727679319352644e-07,
      "log_odds_chosen": 0.44926756620407104,
      "log_odds_ratio": -0.6102539300918579,
      "logits/chosen": -0.7373046875,
      "logits/rejected": -0.6231445074081421,
      "logps/chosen": -0.7220703363418579,
      "logps/rejected": -1.006445288658142,
      "loss": 0.8849,
      "nll_loss": 0.9013671875,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07221679389476776,
      "rewards/margins": 0.028466034680604935,
      "rewards/rejected": -0.10063476860523224,
      "step": 14140
    },
    {
      "epoch": 0.5302505105769051,
      "grad_norm": 1.9734174210788193,
      "learning_rate": 6.725301627347177e-07,
      "log_odds_chosen": 0.6408935785293579,
      "log_odds_ratio": -0.533203125,
      "logits/chosen": -0.7142578363418579,
      "logits/rejected": -0.626660168170929,
      "logps/chosen": -0.675585925579071,
      "logps/rejected": -1.0759766101837158,
      "loss": 0.8917,
      "nll_loss": 0.7904297113418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.0675048828125,
      "rewards/margins": 0.040126800537109375,
      "rewards/rejected": -0.10759277641773224,
      "step": 14150
    },
    {
      "epoch": 0.530625245920069,
      "grad_norm": 2.1177199193437772,
      "learning_rate": 6.722926454528143e-07,
      "log_odds_chosen": 0.46635740995407104,
      "log_odds_ratio": -0.6031249761581421,
      "logits/chosen": -0.740429699420929,
      "logits/rejected": -0.6333984136581421,
      "logps/chosen": -0.655957043170929,
      "logps/rejected": -0.954296886920929,
      "loss": 0.8652,
      "nll_loss": 0.825390636920929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06560058891773224,
      "rewards/margins": 0.02985382080078125,
      "rewards/rejected": -0.09544678032398224,
      "step": 14160
    },
    {
      "epoch": 0.5309999812632329,
      "grad_norm": 1.9077246339558143,
      "learning_rate": 6.720553796450181e-07,
      "log_odds_chosen": 0.3513549864292145,
      "log_odds_ratio": -0.65576171875,
      "logits/chosen": -0.66650390625,
      "logits/rejected": -0.60009765625,
      "logps/chosen": -0.667773425579071,
      "logps/rejected": -0.9144531488418579,
      "loss": 0.8935,
      "nll_loss": 0.900195300579071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06684570014476776,
      "rewards/margins": 0.02463836595416069,
      "rewards/rejected": -0.09146728366613388,
      "step": 14170
    },
    {
      "epoch": 0.5313747166063967,
      "grad_norm": 2.151585667622358,
      "learning_rate": 6.718183648678903e-07,
      "log_odds_chosen": 0.543774425983429,
      "log_odds_ratio": -0.57373046875,
      "logits/chosen": -0.7119140625,
      "logits/rejected": -0.5785156488418579,
      "logps/chosen": -0.6826171875,
      "logps/rejected": -1.0275390148162842,
      "loss": 0.8689,
      "nll_loss": 0.8221679925918579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06813964992761612,
      "rewards/margins": 0.0345306396484375,
      "rewards/rejected": -0.10274658352136612,
      "step": 14180
    },
    {
      "epoch": 0.5317494519495606,
      "grad_norm": 2.0319797994449122,
      "learning_rate": 6.715816006790862e-07,
      "log_odds_chosen": 0.5687011480331421,
      "log_odds_ratio": -0.5814453363418579,
      "logits/chosen": -0.7681640386581421,
      "logits/rejected": -0.667773425579071,
      "logps/chosen": -0.66357421875,
      "logps/rejected": -1.0402343273162842,
      "loss": 0.8962,
      "nll_loss": 0.7958984375,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06633301079273224,
      "rewards/margins": 0.03765564039349556,
      "rewards/rejected": -0.10395507514476776,
      "step": 14190
    },
    {
      "epoch": 0.5321241872927245,
      "grad_norm": 1.847879043360253,
      "learning_rate": 6.713450866373512e-07,
      "log_odds_chosen": 0.5830078125,
      "log_odds_ratio": -0.582324206829071,
      "logits/chosen": -0.744921863079071,
      "logits/rejected": -0.643847644329071,
      "logps/chosen": -0.672070324420929,
      "logps/rejected": -1.06640625,
      "loss": 0.8845,
      "nll_loss": 0.8173828125,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06716308742761612,
      "rewards/margins": 0.03946533054113388,
      "rewards/rejected": -0.10664062201976776,
      "step": 14200
    },
    {
      "epoch": 0.5324989226358884,
      "grad_norm": 2.069025704205118,
      "learning_rate": 6.711088223025183e-07,
      "log_odds_chosen": 0.3682495057582855,
      "log_odds_ratio": -0.6250976324081421,
      "logits/chosen": -0.70166015625,
      "logits/rejected": -0.641406238079071,
      "logps/chosen": -0.675585925579071,
      "logps/rejected": -0.909375011920929,
      "loss": 0.8867,
      "nll_loss": 0.7798827886581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06752929836511612,
      "rewards/margins": 0.023381805047392845,
      "rewards/rejected": -0.09099121391773224,
      "step": 14210
    },
    {
      "epoch": 0.5328736579790523,
      "grad_norm": 1.9600393452161304,
      "learning_rate": 6.708728072355036e-07,
      "log_odds_chosen": 0.38043212890625,
      "log_odds_ratio": -0.6351562738418579,
      "logits/chosen": -0.7611328363418579,
      "logits/rejected": -0.687207043170929,
      "logps/chosen": -0.757031261920929,
      "logps/rejected": -1.006445288658142,
      "loss": 0.9039,
      "nll_loss": 0.869921863079071,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07570800930261612,
      "rewards/margins": 0.024952316656708717,
      "rewards/rejected": -0.10063476860523224,
      "step": 14220
    },
    {
      "epoch": 0.5332483933222162,
      "grad_norm": 1.854351810039464,
      "learning_rate": 6.706370409983032e-07,
      "log_odds_chosen": 0.36309814453125,
      "log_odds_ratio": -0.6436523199081421,
      "logits/chosen": -0.7230468988418579,
      "logits/rejected": -0.686718761920929,
      "logps/chosen": -0.6880859136581421,
      "logps/rejected": -0.924023449420929,
      "loss": 0.8906,
      "nll_loss": 0.7710937261581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06879882514476776,
      "rewards/margins": 0.0236053466796875,
      "rewards/rejected": -0.09230957180261612,
      "step": 14230
    },
    {
      "epoch": 0.5336231286653801,
      "grad_norm": 2.502643548695064,
      "learning_rate": 6.704015231539909e-07,
      "log_odds_chosen": 0.5226074457168579,
      "log_odds_ratio": -0.5912109613418579,
      "logits/chosen": -0.7992187738418579,
      "logits/rejected": -0.7074218988418579,
      "logps/chosen": -0.704785168170929,
      "logps/rejected": -1.036035180091858,
      "loss": 0.8777,
      "nll_loss": 0.826367199420929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07044677436351776,
      "rewards/margins": 0.033092498779296875,
      "rewards/rejected": -0.10361327975988388,
      "step": 14240
    },
    {
      "epoch": 0.533997864008544,
      "grad_norm": 1.9189333314776214,
      "learning_rate": 6.701662532667127e-07,
      "log_odds_chosen": 0.38671875,
      "log_odds_ratio": -0.629199206829071,
      "logits/chosen": -0.7798827886581421,
      "logits/rejected": -0.6773437261581421,
      "logps/chosen": -0.7040039300918579,
      "logps/rejected": -0.9403320550918579,
      "loss": 0.8976,
      "nll_loss": 0.8861328363418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07033691555261612,
      "rewards/margins": 0.02365417405962944,
      "rewards/rejected": -0.09405517578125,
      "step": 14250
    },
    {
      "epoch": 0.5343725993517079,
      "grad_norm": 1.8262868148769296,
      "learning_rate": 6.699312309016855e-07,
      "log_odds_chosen": 0.4037841856479645,
      "log_odds_ratio": -0.616894543170929,
      "logits/chosen": -0.76171875,
      "logits/rejected": -0.6202148199081421,
      "logps/chosen": -0.6884765625,
      "logps/rejected": -0.961718738079071,
      "loss": 0.8912,
      "nll_loss": 0.8414062261581421,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06889648735523224,
      "rewards/margins": 0.02730255201458931,
      "rewards/rejected": -0.09614257514476776,
      "step": 14260
    },
    {
      "epoch": 0.5347473346948718,
      "grad_norm": 2.16632010779235,
      "learning_rate": 6.696964556251918e-07,
      "log_odds_chosen": 0.47637939453125,
      "log_odds_ratio": -0.594531238079071,
      "logits/chosen": -0.716796875,
      "logits/rejected": -0.646191418170929,
      "logps/chosen": -0.6859375238418579,
      "logps/rejected": -0.9867187738418579,
      "loss": 0.8823,
      "nll_loss": 0.8207031488418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06857910007238388,
      "rewards/margins": 0.0301055908203125,
      "rewards/rejected": -0.09860839694738388,
      "step": 14270
    },
    {
      "epoch": 0.5351220700380357,
      "grad_norm": 1.879966896145246,
      "learning_rate": 6.694619270045784e-07,
      "log_odds_chosen": 0.6253296136856079,
      "log_odds_ratio": -0.5396484136581421,
      "logits/chosen": -0.771484375,
      "logits/rejected": -0.667773425579071,
      "logps/chosen": -0.6592773199081421,
      "logps/rejected": -1.0447266101837158,
      "loss": 0.8762,
      "nll_loss": 0.776660144329071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06591796875,
      "rewards/margins": 0.03858489915728569,
      "rewards/rejected": -0.10456542670726776,
      "step": 14280
    },
    {
      "epoch": 0.5354968053811995,
      "grad_norm": 2.0185349241217305,
      "learning_rate": 6.69227644608251e-07,
      "log_odds_chosen": 0.52935791015625,
      "log_odds_ratio": -0.5835937261581421,
      "logits/chosen": -0.712695300579071,
      "logits/rejected": -0.6220703125,
      "logps/chosen": -0.694531261920929,
      "logps/rejected": -1.031640648841858,
      "loss": 0.8821,
      "nll_loss": 0.8773437738418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06942138820886612,
      "rewards/margins": 0.03367462009191513,
      "rewards/rejected": -0.10308837890625,
      "step": 14290
    },
    {
      "epoch": 0.5358715407243634,
      "grad_norm": 2.1287066815049824,
      "learning_rate": 6.689936080056727e-07,
      "log_odds_chosen": 0.5583862066268921,
      "log_odds_ratio": -0.57861328125,
      "logits/chosen": -0.7298828363418579,
      "logits/rejected": -0.613085925579071,
      "logps/chosen": -0.685253918170929,
      "logps/rejected": -1.0519530773162842,
      "loss": 0.8648,
      "nll_loss": 0.856640636920929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06854248046875,
      "rewards/margins": 0.03671874850988388,
      "rewards/rejected": -0.105224609375,
      "step": 14300
    },
    {
      "epoch": 0.5362462760675273,
      "grad_norm": 1.896282841287982,
      "learning_rate": 6.687598167673588e-07,
      "log_odds_chosen": 0.560412585735321,
      "log_odds_ratio": -0.568554699420929,
      "logits/chosen": -0.7269531488418579,
      "logits/rejected": -0.631054699420929,
      "logps/chosen": -0.6597656011581421,
      "logps/rejected": -1.006250023841858,
      "loss": 0.8672,
      "nll_loss": 0.8026367425918579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06596679985523224,
      "rewards/margins": 0.03464660793542862,
      "rewards/rejected": -0.10063476860523224,
      "step": 14310
    },
    {
      "epoch": 0.5366210114106912,
      "grad_norm": 1.8965309291128536,
      "learning_rate": 6.685262704648755e-07,
      "log_odds_chosen": 0.40089112520217896,
      "log_odds_ratio": -0.647167980670929,
      "logits/chosen": -0.7509765625,
      "logits/rejected": -0.6513671875,
      "logps/chosen": -0.6908203363418579,
      "logps/rejected": -0.98388671875,
      "loss": 0.8878,
      "nll_loss": 0.8150390386581421,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06905517727136612,
      "rewards/margins": 0.029393767938017845,
      "rewards/rejected": -0.09842529147863388,
      "step": 14320
    },
    {
      "epoch": 0.5369957467538551,
      "grad_norm": 1.9924095866165534,
      "learning_rate": 6.682929686708352e-07,
      "log_odds_chosen": 0.4814086854457855,
      "log_odds_ratio": -0.6014648675918579,
      "logits/chosen": -0.757031261920929,
      "logits/rejected": -0.6513671875,
      "logps/chosen": -0.654101550579071,
      "logps/rejected": -0.974609375,
      "loss": 0.8996,
      "nll_loss": 0.829882800579071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06544189155101776,
      "rewards/margins": 0.03206787258386612,
      "rewards/rejected": -0.09750976413488388,
      "step": 14330
    },
    {
      "epoch": 0.537370482097019,
      "grad_norm": 2.301725723442159,
      "learning_rate": 6.680599109588932e-07,
      "log_odds_chosen": 0.49864500761032104,
      "log_odds_ratio": -0.6050781011581421,
      "logits/chosen": -0.7408202886581421,
      "logits/rejected": -0.656054675579071,
      "logps/chosen": -0.7152343988418579,
      "logps/rejected": -1.0353515148162842,
      "loss": 0.8807,
      "nll_loss": 0.779101550579071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07155761867761612,
      "rewards/margins": 0.032018281519412994,
      "rewards/rejected": -0.10364989936351776,
      "step": 14340
    },
    {
      "epoch": 0.5377452174401829,
      "grad_norm": 1.8483961424590907,
      "learning_rate": 6.678270969037457e-07,
      "log_odds_chosen": 0.581530749797821,
      "log_odds_ratio": -0.5555664300918579,
      "logits/chosen": -0.779492199420929,
      "logits/rejected": -0.6498047113418579,
      "logps/chosen": -0.6509765386581421,
      "logps/rejected": -1.011132836341858,
      "loss": 0.8842,
      "nll_loss": 0.848828136920929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06510009616613388,
      "rewards/margins": 0.03602142259478569,
      "rewards/rejected": -0.10113525390625,
      "step": 14350
    },
    {
      "epoch": 0.5381199527833468,
      "grad_norm": 1.8878324915461322,
      "learning_rate": 6.67594526081125e-07,
      "log_odds_chosen": 0.3864807188510895,
      "log_odds_ratio": -0.6322265863418579,
      "logits/chosen": -0.7867187261581421,
      "logits/rejected": -0.7076171636581421,
      "logps/chosen": -0.714550793170929,
      "logps/rejected": -0.9517577886581421,
      "loss": 0.8938,
      "nll_loss": 0.8412109613418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07147216796875,
      "rewards/margins": 0.02364196814596653,
      "rewards/rejected": -0.09523925930261612,
      "step": 14360
    },
    {
      "epoch": 0.5384946881265107,
      "grad_norm": 1.8785301425669687,
      "learning_rate": 6.673621980677971e-07,
      "log_odds_chosen": 0.40260010957717896,
      "log_odds_ratio": -0.625195324420929,
      "logits/chosen": -0.749218761920929,
      "logits/rejected": -0.6617187261581421,
      "logps/chosen": -0.6802734136581421,
      "logps/rejected": -0.9501953125,
      "loss": 0.8839,
      "nll_loss": 0.821484386920929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06801757961511612,
      "rewards/margins": 0.02702789381146431,
      "rewards/rejected": -0.09503173828125,
      "step": 14370
    },
    {
      "epoch": 0.5388694234696746,
      "grad_norm": 1.8097288687550648,
      "learning_rate": 6.671301124415585e-07,
      "log_odds_chosen": 0.604663074016571,
      "log_odds_ratio": -0.568164050579071,
      "logits/chosen": -0.7710937261581421,
      "logits/rejected": -0.62841796875,
      "logps/chosen": -0.6800781488418579,
      "logps/rejected": -1.0859375,
      "loss": 0.8782,
      "nll_loss": 0.84375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0679931640625,
      "rewards/margins": 0.04064331203699112,
      "rewards/rejected": -0.10865478217601776,
      "step": 14380
    },
    {
      "epoch": 0.5392441588128384,
      "grad_norm": 1.8442194507409593,
      "learning_rate": 6.668982687812326e-07,
      "log_odds_chosen": 0.4459472596645355,
      "log_odds_ratio": -0.6202148199081421,
      "logits/chosen": -0.773242175579071,
      "logits/rejected": -0.6732422113418579,
      "logps/chosen": -0.7232421636581421,
      "logps/rejected": -1.0164062976837158,
      "loss": 0.8883,
      "nll_loss": 0.8451172113418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07227782905101776,
      "rewards/margins": 0.02936096116900444,
      "rewards/rejected": -0.10163573920726776,
      "step": 14390
    },
    {
      "epoch": 0.5396188941560023,
      "grad_norm": 2.180466333186099,
      "learning_rate": 6.666666666666666e-07,
      "log_odds_chosen": 0.763964831829071,
      "log_odds_ratio": -0.498046875,
      "logits/chosen": -0.7894531488418579,
      "logits/rejected": -0.653027355670929,
      "logps/chosen": -0.679394543170929,
      "logps/rejected": -1.183203101158142,
      "loss": 0.8988,
      "nll_loss": 0.8062499761581421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06793212890625,
      "rewards/margins": 0.05042724683880806,
      "rewards/rejected": -0.11835937201976776,
      "step": 14400
    },
    {
      "epoch": 0.5399936294991662,
      "grad_norm": 1.873446208767733,
      "learning_rate": 6.664353056787287e-07,
      "log_odds_chosen": 0.5550781488418579,
      "log_odds_ratio": -0.5575195550918579,
      "logits/chosen": -0.831835925579071,
      "logits/rejected": -0.6763671636581421,
      "logps/chosen": -0.6617187261581421,
      "logps/rejected": -0.984570324420929,
      "loss": 0.8876,
      "nll_loss": 0.8125,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.066162109375,
      "rewards/margins": 0.03229980543255806,
      "rewards/rejected": -0.09843750298023224,
      "step": 14410
    },
    {
      "epoch": 0.5403683648423301,
      "grad_norm": 1.9879666501708462,
      "learning_rate": 6.662041853993043e-07,
      "log_odds_chosen": 0.603991687297821,
      "log_odds_ratio": -0.591113269329071,
      "logits/chosen": -0.7621093988418579,
      "logits/rejected": -0.6494140625,
      "logps/chosen": -0.691210925579071,
      "logps/rejected": -1.099609375,
      "loss": 0.8924,
      "nll_loss": 0.8814452886581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06910400092601776,
      "rewards/margins": 0.04093017429113388,
      "rewards/rejected": -0.11003418266773224,
      "step": 14420
    },
    {
      "epoch": 0.540743100185494,
      "grad_norm": 1.900946042536634,
      "learning_rate": 6.659733054112932e-07,
      "log_odds_chosen": 0.6115967035293579,
      "log_odds_ratio": -0.5478515625,
      "logits/chosen": -0.715624988079071,
      "logits/rejected": -0.6373046636581421,
      "logps/chosen": -0.62109375,
      "logps/rejected": -0.9798828363418579,
      "loss": 0.8845,
      "nll_loss": 0.765429675579071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06208496168255806,
      "rewards/margins": 0.03591613844037056,
      "rewards/rejected": -0.09799804538488388,
      "step": 14430
    },
    {
      "epoch": 0.5411178355286579,
      "grad_norm": 1.9651635188197611,
      "learning_rate": 6.657426652986061e-07,
      "log_odds_chosen": 0.5567382574081421,
      "log_odds_ratio": -0.59423828125,
      "logits/chosen": -0.7613281011581421,
      "logits/rejected": -0.6625000238418579,
      "logps/chosen": -0.722460925579071,
      "logps/rejected": -1.0822265148162842,
      "loss": 0.882,
      "nll_loss": 0.8021484613418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07230224460363388,
      "rewards/margins": 0.03594665601849556,
      "rewards/rejected": -0.10817871242761612,
      "step": 14440
    },
    {
      "epoch": 0.5414925708718218,
      "grad_norm": 2.019502411835844,
      "learning_rate": 6.655122646461624e-07,
      "log_odds_chosen": 0.4471801817417145,
      "log_odds_ratio": -0.626269519329071,
      "logits/chosen": -0.719531238079071,
      "logits/rejected": -0.6439453363418579,
      "logps/chosen": -0.6649414300918579,
      "logps/rejected": -0.953125,
      "loss": 0.8977,
      "nll_loss": 0.800000011920929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06645508110523224,
      "rewards/margins": 0.02880554273724556,
      "rewards/rejected": -0.09523925930261612,
      "step": 14450
    },
    {
      "epoch": 0.5418673062149857,
      "grad_norm": 1.691735381333674,
      "learning_rate": 6.652821030398855e-07,
      "log_odds_chosen": 0.520764172077179,
      "log_odds_ratio": -0.569140613079071,
      "logits/chosen": -0.7841796875,
      "logits/rejected": -0.683398425579071,
      "logps/chosen": -0.6841796636581421,
      "logps/rejected": -1.0187499523162842,
      "loss": 0.8975,
      "nll_loss": 0.8251953125,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.0684814453125,
      "rewards/margins": 0.03342285007238388,
      "rewards/rejected": -0.10185547173023224,
      "step": 14460
    },
    {
      "epoch": 0.5422420415581496,
      "grad_norm": 1.9499579803335292,
      "learning_rate": 6.650521800667013e-07,
      "log_odds_chosen": 0.36284178495407104,
      "log_odds_ratio": -0.6343749761581421,
      "logits/chosen": -0.757617175579071,
      "logits/rejected": -0.6581054925918579,
      "logps/chosen": -0.6875,
      "logps/rejected": -0.9208984375,
      "loss": 0.9113,
      "nll_loss": 0.8392578363418579,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06871338188648224,
      "rewards/margins": 0.02333831787109375,
      "rewards/rejected": -0.09201660007238388,
      "step": 14470
    },
    {
      "epoch": 0.5426167769013135,
      "grad_norm": 1.99646901228458,
      "learning_rate": 6.648224953145336e-07,
      "log_odds_chosen": 0.509136974811554,
      "log_odds_ratio": -0.5794922113418579,
      "logits/chosen": -0.7857421636581421,
      "logits/rejected": -0.673632800579071,
      "logps/chosen": -0.682421863079071,
      "logps/rejected": -1.013085961341858,
      "loss": 0.8937,
      "nll_loss": 0.802929699420929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.0682373046875,
      "rewards/margins": 0.033011626452207565,
      "rewards/rejected": -0.10124512016773224,
      "step": 14480
    },
    {
      "epoch": 0.5429915122444774,
      "grad_norm": 1.9965957165491162,
      "learning_rate": 6.645930483723025e-07,
      "log_odds_chosen": 0.3838867247104645,
      "log_odds_ratio": -0.623828113079071,
      "logits/chosen": -0.750781238079071,
      "logits/rejected": -0.6465820074081421,
      "logps/chosen": -0.730761706829071,
      "logps/rejected": -0.99609375,
      "loss": 0.8643,
      "nll_loss": 0.852734386920929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07308349758386612,
      "rewards/margins": 0.026514435186982155,
      "rewards/rejected": -0.09963379055261612,
      "step": 14490
    },
    {
      "epoch": 0.5433662475876412,
      "grad_norm": 1.9496667803859378,
      "learning_rate": 6.643638388299198e-07,
      "log_odds_chosen": 0.5928589105606079,
      "log_odds_ratio": -0.554003894329071,
      "logits/chosen": -0.766308605670929,
      "logits/rejected": -0.616015613079071,
      "logps/chosen": -0.6792968511581421,
      "logps/rejected": -1.052343726158142,
      "loss": 0.8841,
      "nll_loss": 0.778515636920929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06793212890625,
      "rewards/margins": 0.03737793117761612,
      "rewards/rejected": -0.10524902492761612,
      "step": 14500
    },
    {
      "epoch": 0.5437409829308051,
      "grad_norm": 2.2643334233385537,
      "learning_rate": 6.64134866278287e-07,
      "log_odds_chosen": 0.4837097227573395,
      "log_odds_ratio": -0.6133788824081421,
      "logits/chosen": -0.7451171875,
      "logits/rejected": -0.6465820074081421,
      "logps/chosen": -0.6670898199081421,
      "logps/rejected": -0.9613281488418579,
      "loss": 0.8925,
      "nll_loss": 0.904101550579071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.066650390625,
      "rewards/margins": 0.02941284142434597,
      "rewards/rejected": -0.09616699069738388,
      "step": 14510
    },
    {
      "epoch": 0.544115718273969,
      "grad_norm": 1.9241146971273424,
      "learning_rate": 6.639061303092922e-07,
      "log_odds_chosen": 0.4903564453125,
      "log_odds_ratio": -0.6234375238418579,
      "logits/chosen": -0.742871105670929,
      "logits/rejected": -0.646777331829071,
      "logps/chosen": -0.70703125,
      "logps/rejected": -1.021093726158142,
      "loss": 0.8861,
      "nll_loss": 0.847851574420929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07062987983226776,
      "rewards/margins": 0.03146476671099663,
      "rewards/rejected": -0.10208740085363388,
      "step": 14520
    },
    {
      "epoch": 0.5444904536171329,
      "grad_norm": 1.9141531550928848,
      "learning_rate": 6.636776305158062e-07,
      "log_odds_chosen": 0.5707153081893921,
      "log_odds_ratio": -0.566601574420929,
      "logits/chosen": -0.766406238079071,
      "logits/rejected": -0.681640625,
      "logps/chosen": -0.6849609613418579,
      "logps/rejected": -1.0460937023162842,
      "loss": 0.8818,
      "nll_loss": 0.8238281011581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06855468451976776,
      "rewards/margins": 0.03610076755285263,
      "rewards/rejected": -0.10463867336511612,
      "step": 14530
    },
    {
      "epoch": 0.5448651889602968,
      "grad_norm": 1.8912742140665204,
      "learning_rate": 6.634493664916802e-07,
      "log_odds_chosen": 0.52581787109375,
      "log_odds_ratio": -0.563769519329071,
      "logits/chosen": -0.8150390386581421,
      "logits/rejected": -0.708789050579071,
      "logps/chosen": -0.66796875,
      "logps/rejected": -1.001562476158142,
      "loss": 0.877,
      "nll_loss": 0.7466796636581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06682129204273224,
      "rewards/margins": 0.03333740308880806,
      "rewards/rejected": -0.10017089545726776,
      "step": 14540
    },
    {
      "epoch": 0.5452399243034607,
      "grad_norm": 2.0272340754632867,
      "learning_rate": 6.632213378317426e-07,
      "log_odds_chosen": 0.4513183534145355,
      "log_odds_ratio": -0.623046875,
      "logits/chosen": -0.7890625,
      "logits/rejected": -0.6766601800918579,
      "logps/chosen": -0.71923828125,
      "logps/rejected": -1.0197265148162842,
      "loss": 0.8799,
      "nll_loss": 0.8597656488418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07188721001148224,
      "rewards/margins": 0.03007354773581028,
      "rewards/rejected": -0.10202636569738388,
      "step": 14550
    },
    {
      "epoch": 0.5456146596466246,
      "grad_norm": 2.001828730110836,
      "learning_rate": 6.629935441317959e-07,
      "log_odds_chosen": 0.527099609375,
      "log_odds_ratio": -0.5712890625,
      "logits/chosen": -0.802929699420929,
      "logits/rejected": -0.7056640386581421,
      "logps/chosen": -0.703125,
      "logps/rejected": -1.0380859375,
      "loss": 0.8789,
      "nll_loss": 0.8423827886581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07025146484375,
      "rewards/margins": 0.03356323391199112,
      "rewards/rejected": -0.103759765625,
      "step": 14560
    },
    {
      "epoch": 0.5459893949897885,
      "grad_norm": 2.0242066373352867,
      "learning_rate": 6.627659849886136e-07,
      "log_odds_chosen": 0.49317628145217896,
      "log_odds_ratio": -0.5907226800918579,
      "logits/chosen": -0.764843761920929,
      "logits/rejected": -0.6507812738418579,
      "logps/chosen": -0.687304675579071,
      "logps/rejected": -0.9970703125,
      "loss": 0.8882,
      "nll_loss": 0.814160168170929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06870117038488388,
      "rewards/margins": 0.03092498704791069,
      "rewards/rejected": -0.09968261420726776,
      "step": 14570
    },
    {
      "epoch": 0.5463641303329524,
      "grad_norm": 1.85199316385712,
      "learning_rate": 6.625386599999376e-07,
      "log_odds_chosen": 0.44807130098342896,
      "log_odds_ratio": -0.6190429925918579,
      "logits/chosen": -0.7652343511581421,
      "logits/rejected": -0.704785168170929,
      "logps/chosen": -0.661328136920929,
      "logps/rejected": -0.9615234136581421,
      "loss": 0.8905,
      "nll_loss": 0.832812488079071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06614990532398224,
      "rewards/margins": 0.02997741661965847,
      "rewards/rejected": -0.09616699069738388,
      "step": 14580
    },
    {
      "epoch": 0.5467388656761163,
      "grad_norm": 2.11978770105477,
      "learning_rate": 6.623115687644749e-07,
      "log_odds_chosen": 0.52203369140625,
      "log_odds_ratio": -0.586230456829071,
      "logits/chosen": -0.762011706829071,
      "logits/rejected": -0.6705077886581421,
      "logps/chosen": -0.670605480670929,
      "logps/rejected": -0.9794921875,
      "loss": 0.8753,
      "nll_loss": 0.8070312738418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06712646782398224,
      "rewards/margins": 0.03085632249712944,
      "rewards/rejected": -0.09793701022863388,
      "step": 14590
    },
    {
      "epoch": 0.5471136010192801,
      "grad_norm": 1.9203064022402072,
      "learning_rate": 6.620847108818943e-07,
      "log_odds_chosen": 0.29792481660842896,
      "log_odds_ratio": -0.657031238079071,
      "logits/chosen": -0.720898449420929,
      "logits/rejected": -0.668164074420929,
      "logps/chosen": -0.6923828125,
      "logps/rejected": -0.8697265386581421,
      "loss": 0.923,
      "nll_loss": 0.8384765386581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06917724758386612,
      "rewards/margins": 0.017764663323760033,
      "rewards/rejected": -0.08693847805261612,
      "step": 14600
    },
    {
      "epoch": 0.547488336362444,
      "grad_norm": 1.9401193893322863,
      "learning_rate": 6.618580859528244e-07,
      "log_odds_chosen": 0.39453125,
      "log_odds_ratio": -0.6253906488418579,
      "logits/chosen": -0.695507824420929,
      "logits/rejected": -0.6219726800918579,
      "logps/chosen": -0.726757824420929,
      "logps/rejected": -0.989453136920929,
      "loss": 0.894,
      "nll_loss": 0.819140613079071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07261963188648224,
      "rewards/margins": 0.02631073072552681,
      "rewards/rejected": -0.0989990234375,
      "step": 14610
    },
    {
      "epoch": 0.5478630717056079,
      "grad_norm": 1.9579331659449326,
      "learning_rate": 6.616316935788494e-07,
      "log_odds_chosen": 0.5741516351699829,
      "log_odds_ratio": -0.583691418170929,
      "logits/chosen": -0.722460925579071,
      "logits/rejected": -0.6375976800918579,
      "logps/chosen": -0.6552734375,
      "logps/rejected": -1.0148437023162842,
      "loss": 0.8951,
      "nll_loss": 0.853808581829071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06547851860523224,
      "rewards/margins": 0.03602447360754013,
      "rewards/rejected": -0.10151366889476776,
      "step": 14620
    },
    {
      "epoch": 0.5482378070487718,
      "grad_norm": 1.8459779721680556,
      "learning_rate": 6.614055333625071e-07,
      "log_odds_chosen": 0.42247313261032104,
      "log_odds_ratio": -0.6124023199081421,
      "logits/chosen": -0.7357422113418579,
      "logits/rejected": -0.637499988079071,
      "logps/chosen": -0.6927734613418579,
      "logps/rejected": -0.965527355670929,
      "loss": 0.8785,
      "nll_loss": 0.8525390625,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06928710639476776,
      "rewards/margins": 0.02729492262005806,
      "rewards/rejected": -0.09658203274011612,
      "step": 14630
    },
    {
      "epoch": 0.5486125423919357,
      "grad_norm": 1.972311284028819,
      "learning_rate": 6.611796049072861e-07,
      "log_odds_chosen": 0.567431628704071,
      "log_odds_ratio": -0.561328113079071,
      "logits/chosen": -0.788281261920929,
      "logits/rejected": -0.697460949420929,
      "logps/chosen": -0.6348632574081421,
      "logps/rejected": -1.0006835460662842,
      "loss": 0.8759,
      "nll_loss": 0.749316394329071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.0634765625,
      "rewards/margins": 0.036547087132930756,
      "rewards/rejected": -0.10003662109375,
      "step": 14640
    },
    {
      "epoch": 0.5489872777350996,
      "grad_norm": 1.9828184091027623,
      "learning_rate": 6.60953907817622e-07,
      "log_odds_chosen": 0.589221179485321,
      "log_odds_ratio": -0.57080078125,
      "logits/chosen": -0.769726574420929,
      "logits/rejected": -0.65576171875,
      "logps/chosen": -0.6611328125,
      "logps/rejected": -1.0412108898162842,
      "loss": 0.8747,
      "nll_loss": 0.856249988079071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06611327826976776,
      "rewards/margins": 0.03803711012005806,
      "rewards/rejected": -0.10410156100988388,
      "step": 14650
    },
    {
      "epoch": 0.5493620130782635,
      "grad_norm": 2.545943676597517,
      "learning_rate": 6.607284416988948e-07,
      "log_odds_chosen": 0.46271973848342896,
      "log_odds_ratio": -0.607714831829071,
      "logits/chosen": -0.745898425579071,
      "logits/rejected": -0.669921875,
      "logps/chosen": -0.647265613079071,
      "logps/rejected": -0.9326171875,
      "loss": 0.8726,
      "nll_loss": 0.7749999761581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06475830078125,
      "rewards/margins": 0.028604889288544655,
      "rewards/rejected": -0.09327392280101776,
      "step": 14660
    },
    {
      "epoch": 0.5497367484214274,
      "grad_norm": 1.9565943760196747,
      "learning_rate": 6.605032061574266e-07,
      "log_odds_chosen": 0.447265625,
      "log_odds_ratio": -0.6170898675918579,
      "logits/chosen": -0.802929699420929,
      "logits/rejected": -0.6708984375,
      "logps/chosen": -0.66259765625,
      "logps/rejected": -0.9267578125,
      "loss": 0.8751,
      "nll_loss": 0.8203125,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06622314453125,
      "rewards/margins": 0.02648467943072319,
      "rewards/rejected": -0.09267578274011612,
      "step": 14670
    },
    {
      "epoch": 0.5501114837645913,
      "grad_norm": 2.1736043239470835,
      "learning_rate": 6.602782008004778e-07,
      "log_odds_chosen": 0.3592895567417145,
      "log_odds_ratio": -0.637988269329071,
      "logits/chosen": -0.792187511920929,
      "logits/rejected": -0.7074218988418579,
      "logps/chosen": -0.7103515863418579,
      "logps/rejected": -0.9583984613418579,
      "loss": 0.8784,
      "nll_loss": 0.846875011920929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07102050632238388,
      "rewards/margins": 0.02477722242474556,
      "rewards/rejected": -0.09580077975988388,
      "step": 14680
    },
    {
      "epoch": 0.5504862191077552,
      "grad_norm": 2.039604973349929,
      "learning_rate": 6.600534252362451e-07,
      "log_odds_chosen": 0.5513671636581421,
      "log_odds_ratio": -0.5752929449081421,
      "logits/chosen": -0.7632812261581421,
      "logits/rejected": -0.66162109375,
      "logps/chosen": -0.695117175579071,
      "logps/rejected": -1.0525391101837158,
      "loss": 0.8788,
      "nll_loss": 0.8193359375,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.069580078125,
      "rewards/margins": 0.03564758226275444,
      "rewards/rejected": -0.10517577826976776,
      "step": 14690
    },
    {
      "epoch": 0.5508609544509191,
      "grad_norm": 1.8952393985884226,
      "learning_rate": 6.59828879073858e-07,
      "log_odds_chosen": 0.5672607421875,
      "log_odds_ratio": -0.5850585699081421,
      "logits/chosen": -0.7025390863418579,
      "logits/rejected": -0.591503918170929,
      "logps/chosen": -0.68115234375,
      "logps/rejected": -1.0480468273162842,
      "loss": 0.8832,
      "nll_loss": 0.8460937738418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06815185397863388,
      "rewards/margins": 0.03659515455365181,
      "rewards/rejected": -0.10476074367761612,
      "step": 14700
    },
    {
      "epoch": 0.5512356897940829,
      "grad_norm": 2.0199678719124345,
      "learning_rate": 6.596045619233764e-07,
      "log_odds_chosen": 0.5965576171875,
      "log_odds_ratio": -0.5653320550918579,
      "logits/chosen": -0.7935546636581421,
      "logits/rejected": -0.6470702886581421,
      "logps/chosen": -0.6625000238418579,
      "logps/rejected": -1.0478515625,
      "loss": 0.8673,
      "nll_loss": 0.774609386920929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06624756008386612,
      "rewards/margins": 0.038677215576171875,
      "rewards/rejected": -0.10484619438648224,
      "step": 14710
    },
    {
      "epoch": 0.5516104251372468,
      "grad_norm": 2.1753389024045773,
      "learning_rate": 6.59380473395787e-07,
      "log_odds_chosen": 0.3912597596645355,
      "log_odds_ratio": -0.633496105670929,
      "logits/chosen": -0.7007812261581421,
      "logits/rejected": -0.644824206829071,
      "logps/chosen": -0.671679675579071,
      "logps/rejected": -0.9183593988418579,
      "loss": 0.9037,
      "nll_loss": 0.7998046875,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06707763671875,
      "rewards/margins": 0.02471160888671875,
      "rewards/rejected": -0.09182129055261612,
      "step": 14720
    },
    {
      "epoch": 0.5519851604804107,
      "grad_norm": 1.8547180991243366,
      "learning_rate": 6.591566131030017e-07,
      "log_odds_chosen": 0.4614501893520355,
      "log_odds_ratio": -0.595703125,
      "logits/chosen": -0.7314453125,
      "logits/rejected": -0.6333984136581421,
      "logps/chosen": -0.6412109136581421,
      "logps/rejected": -0.9156249761581421,
      "loss": 0.8959,
      "nll_loss": 0.805859386920929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06413574516773224,
      "rewards/margins": 0.027448272332549095,
      "rewards/rejected": -0.09154053032398224,
      "step": 14730
    },
    {
      "epoch": 0.5523598958235746,
      "grad_norm": 1.840694089067691,
      "learning_rate": 6.589329806578535e-07,
      "log_odds_chosen": 0.42857664823532104,
      "log_odds_ratio": -0.6278320550918579,
      "logits/chosen": -0.745312511920929,
      "logits/rejected": -0.6558593511581421,
      "logps/chosen": -0.6722656488418579,
      "logps/rejected": -0.9306640625,
      "loss": 0.8841,
      "nll_loss": 0.776171863079071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06717529147863388,
      "rewards/margins": 0.025835419073700905,
      "rewards/rejected": -0.09304199367761612,
      "step": 14740
    },
    {
      "epoch": 0.5527346311667385,
      "grad_norm": 2.2278511392572913,
      "learning_rate": 6.587095756740946e-07,
      "log_odds_chosen": 0.512341320514679,
      "log_odds_ratio": -0.589550793170929,
      "logits/chosen": -0.733105480670929,
      "logits/rejected": -0.6465820074081421,
      "logps/chosen": -0.6826171875,
      "logps/rejected": -1.0080077648162842,
      "loss": 0.899,
      "nll_loss": 0.8804687261581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06824950873851776,
      "rewards/margins": 0.03250732272863388,
      "rewards/rejected": -0.10075683891773224,
      "step": 14750
    },
    {
      "epoch": 0.5531093665099024,
      "grad_norm": 1.9385212067186248,
      "learning_rate": 6.58486397766393e-07,
      "log_odds_chosen": 0.5478515625,
      "log_odds_ratio": -0.580273449420929,
      "logits/chosen": -0.669140636920929,
      "logits/rejected": -0.595996081829071,
      "logps/chosen": -0.654296875,
      "logps/rejected": -0.985546886920929,
      "loss": 0.8909,
      "nll_loss": 0.8241211175918579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06547851860523224,
      "rewards/margins": 0.03305511549115181,
      "rewards/rejected": -0.09855957329273224,
      "step": 14760
    },
    {
      "epoch": 0.5534841018530663,
      "grad_norm": 2.1492452911240405,
      "learning_rate": 6.582634465503303e-07,
      "log_odds_chosen": 0.4854980409145355,
      "log_odds_ratio": -0.592480480670929,
      "logits/chosen": -0.747851550579071,
      "logits/rejected": -0.62109375,
      "logps/chosen": -0.659472644329071,
      "logps/rejected": -0.9673827886581421,
      "loss": 0.8905,
      "nll_loss": 0.8392578363418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06597900390625,
      "rewards/margins": 0.03066101111471653,
      "rewards/rejected": -0.09663085639476776,
      "step": 14770
    },
    {
      "epoch": 0.5538588371962302,
      "grad_norm": 1.9063860245877928,
      "learning_rate": 6.580407216423983e-07,
      "log_odds_chosen": 0.5179198980331421,
      "log_odds_ratio": -0.582812488079071,
      "logits/chosen": -0.728808581829071,
      "logits/rejected": -0.62158203125,
      "logps/chosen": -0.6962890625,
      "logps/rejected": -1.030859351158142,
      "loss": 0.8826,
      "nll_loss": 0.825390636920929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.069580078125,
      "rewards/margins": 0.03354644775390625,
      "rewards/rejected": -0.10307617485523224,
      "step": 14780
    },
    {
      "epoch": 0.5542335725393941,
      "grad_norm": 2.023938858410499,
      "learning_rate": 6.578182226599962e-07,
      "log_odds_chosen": 0.4525512754917145,
      "log_odds_ratio": -0.59619140625,
      "logits/chosen": -0.7222656011581421,
      "logits/rejected": -0.626269519329071,
      "logps/chosen": -0.6689453125,
      "logps/rejected": -0.9564453363418579,
      "loss": 0.8881,
      "nll_loss": 0.834765613079071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06685791164636612,
      "rewards/margins": 0.028774261474609375,
      "rewards/rejected": -0.09567870944738388,
      "step": 14790
    },
    {
      "epoch": 0.554608307882558,
      "grad_norm": 1.8811167455462927,
      "learning_rate": 6.575959492214291e-07,
      "log_odds_chosen": 0.46776121854782104,
      "log_odds_ratio": -0.593554675579071,
      "logits/chosen": -0.7802734375,
      "logits/rejected": -0.674023449420929,
      "logps/chosen": -0.677734375,
      "logps/rejected": -0.9632812738418579,
      "loss": 0.9034,
      "nll_loss": 0.858593761920929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06776122748851776,
      "rewards/margins": 0.028557587414979935,
      "rewards/rejected": -0.09628906100988388,
      "step": 14800
    },
    {
      "epoch": 0.5549830432257218,
      "grad_norm": 1.90958153619906,
      "learning_rate": 6.573739009459035e-07,
      "log_odds_chosen": 0.5022338628768921,
      "log_odds_ratio": -0.5892578363418579,
      "logits/chosen": -0.6953125,
      "logits/rejected": -0.5938476324081421,
      "logps/chosen": -0.6640625,
      "logps/rejected": -0.967968761920929,
      "loss": 0.8869,
      "nll_loss": 0.808398425579071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06646728515625,
      "rewards/margins": 0.03039245679974556,
      "rewards/rejected": -0.09683837741613388,
      "step": 14810
    },
    {
      "epoch": 0.5553577785688857,
      "grad_norm": 2.105045324604539,
      "learning_rate": 6.571520774535256e-07,
      "log_odds_chosen": 0.44001466035842896,
      "log_odds_ratio": -0.617480456829071,
      "logits/chosen": -0.7256835699081421,
      "logits/rejected": -0.6302734613418579,
      "logps/chosen": -0.703125,
      "logps/rejected": -0.984179675579071,
      "loss": 0.8852,
      "nll_loss": 0.8314453363418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07028808444738388,
      "rewards/margins": 0.02815094031393528,
      "rewards/rejected": -0.0985107421875,
      "step": 14820
    },
    {
      "epoch": 0.5557325139120496,
      "grad_norm": 1.9727493018119413,
      "learning_rate": 6.569304783652981e-07,
      "log_odds_chosen": 0.42030030488967896,
      "log_odds_ratio": -0.6172851324081421,
      "logits/chosen": -0.741015613079071,
      "logits/rejected": -0.638964831829071,
      "logps/chosen": -0.698535144329071,
      "logps/rejected": -0.943554699420929,
      "loss": 0.9023,
      "nll_loss": 0.8394531011581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06989745795726776,
      "rewards/margins": 0.02450714074075222,
      "rewards/rejected": -0.0943603515625,
      "step": 14830
    },
    {
      "epoch": 0.5561072492552135,
      "grad_norm": 1.7299895243560062,
      "learning_rate": 6.567091033031185e-07,
      "log_odds_chosen": 0.542431652545929,
      "log_odds_ratio": -0.59228515625,
      "logits/chosen": -0.6898437738418579,
      "logits/rejected": -0.6004883050918579,
      "logps/chosen": -0.67724609375,
      "logps/rejected": -1.0244140625,
      "loss": 0.8998,
      "nll_loss": 0.835156261920929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06771240383386612,
      "rewards/margins": 0.03465423732995987,
      "rewards/rejected": -0.10235595703125,
      "step": 14840
    },
    {
      "epoch": 0.5564819845983774,
      "grad_norm": 2.3809149282995565,
      "learning_rate": 6.564879518897744e-07,
      "log_odds_chosen": 0.3590454161167145,
      "log_odds_ratio": -0.643261730670929,
      "logits/chosen": -0.7476562261581421,
      "logits/rejected": -0.6656249761581421,
      "logps/chosen": -0.66650390625,
      "logps/rejected": -0.900585949420929,
      "loss": 0.8652,
      "nll_loss": 0.768359363079071,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06663818657398224,
      "rewards/margins": 0.023400116711854935,
      "rewards/rejected": -0.09003905951976776,
      "step": 14850
    },
    {
      "epoch": 0.5568567199415413,
      "grad_norm": 1.8372098946034858,
      "learning_rate": 6.562670237489432e-07,
      "log_odds_chosen": 0.4483886659145355,
      "log_odds_ratio": -0.6073242425918579,
      "logits/chosen": -0.699999988079071,
      "logits/rejected": -0.6075195074081421,
      "logps/chosen": -0.711718738079071,
      "logps/rejected": -1.0037109851837158,
      "loss": 0.8903,
      "nll_loss": 0.870312511920929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.0711669921875,
      "rewards/margins": 0.0290985107421875,
      "rewards/rejected": -0.10030517727136612,
      "step": 14860
    },
    {
      "epoch": 0.5572314552847052,
      "grad_norm": 2.1262888175617864,
      "learning_rate": 6.560463185051874e-07,
      "log_odds_chosen": 0.519360363483429,
      "log_odds_ratio": -0.585156261920929,
      "logits/chosen": -0.702343761920929,
      "logits/rejected": -0.592578113079071,
      "logps/chosen": -0.721972644329071,
      "logps/rejected": -1.059960961341858,
      "loss": 0.8846,
      "nll_loss": 0.83154296875,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07220458984375,
      "rewards/margins": 0.03385315090417862,
      "rewards/rejected": -0.10598144680261612,
      "step": 14870
    },
    {
      "epoch": 0.5576061906278691,
      "grad_norm": 2.0882429369755857,
      "learning_rate": 6.558258357839529e-07,
      "log_odds_chosen": 0.4078002870082855,
      "log_odds_ratio": -0.6092773675918579,
      "logits/chosen": -0.74609375,
      "logits/rejected": -0.616894543170929,
      "logps/chosen": -0.7093750238418579,
      "logps/rejected": -0.9574218988418579,
      "loss": 0.9036,
      "nll_loss": 0.844531238079071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07094726711511612,
      "rewards/margins": 0.02483520470559597,
      "rewards/rejected": -0.09580077975988388,
      "step": 14880
    },
    {
      "epoch": 0.557980925971033,
      "grad_norm": 1.939724513592317,
      "learning_rate": 6.556055752115664e-07,
      "log_odds_chosen": 0.44880372285842896,
      "log_odds_ratio": -0.601269543170929,
      "logits/chosen": -0.7334960699081421,
      "logits/rejected": -0.656933605670929,
      "logps/chosen": -0.649609386920929,
      "logps/rejected": -0.953320324420929,
      "loss": 0.8735,
      "nll_loss": 0.7867187261581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06491699069738388,
      "rewards/margins": 0.030364990234375,
      "rewards/rejected": -0.0953369140625,
      "step": 14890
    },
    {
      "epoch": 0.5583556613141969,
      "grad_norm": 1.9397991528767238,
      "learning_rate": 6.553855364152324e-07,
      "log_odds_chosen": 0.5130615234375,
      "log_odds_ratio": -0.5760742425918579,
      "logits/chosen": -0.7689453363418579,
      "logits/rejected": -0.6513671875,
      "logps/chosen": -0.670605480670929,
      "logps/rejected": -1.006250023841858,
      "loss": 0.8858,
      "nll_loss": 0.7891601324081421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06707763671875,
      "rewards/margins": 0.03354186937212944,
      "rewards/rejected": -0.10063476860523224,
      "step": 14900
    },
    {
      "epoch": 0.5587303966573607,
      "grad_norm": 1.9894902100649372,
      "learning_rate": 6.551657190230304e-07,
      "log_odds_chosen": 0.43328857421875,
      "log_odds_ratio": -0.6226562261581421,
      "logits/chosen": -0.723828136920929,
      "logits/rejected": -0.655566394329071,
      "logps/chosen": -0.717089831829071,
      "logps/rejected": -1.002539038658142,
      "loss": 0.8907,
      "nll_loss": 0.830273449420929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07170410454273224,
      "rewards/margins": 0.02852325513958931,
      "rewards/rejected": -0.10019531100988388,
      "step": 14910
    },
    {
      "epoch": 0.5591051320005246,
      "grad_norm": 1.9816349244551872,
      "learning_rate": 6.549461226639129e-07,
      "log_odds_chosen": 0.48139649629592896,
      "log_odds_ratio": -0.612109363079071,
      "logits/chosen": -0.707226574420929,
      "logits/rejected": -0.6329101324081421,
      "logps/chosen": -0.6753906011581421,
      "logps/rejected": -0.988085925579071,
      "loss": 0.8858,
      "nll_loss": 0.813671886920929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.067626953125,
      "rewards/margins": 0.03121643140912056,
      "rewards/rejected": -0.09880371391773224,
      "step": 14920
    },
    {
      "epoch": 0.5594798673436885,
      "grad_norm": 1.9508338481830818,
      "learning_rate": 6.547267469677022e-07,
      "log_odds_chosen": 0.34605711698532104,
      "log_odds_ratio": -0.647265613079071,
      "logits/chosen": -0.733203113079071,
      "logits/rejected": -0.6552734375,
      "logps/chosen": -0.721972644329071,
      "logps/rejected": -0.9673827886581421,
      "loss": 0.8853,
      "nll_loss": 0.820117175579071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07215575873851776,
      "rewards/margins": 0.024506378918886185,
      "rewards/rejected": -0.09671630710363388,
      "step": 14930
    },
    {
      "epoch": 0.5598546026868524,
      "grad_norm": 1.930844371649174,
      "learning_rate": 6.545075915650879e-07,
      "log_odds_chosen": 0.37554931640625,
      "log_odds_ratio": -0.633984386920929,
      "logits/chosen": -0.693359375,
      "logits/rejected": -0.630664050579071,
      "logps/chosen": -0.6875976324081421,
      "logps/rejected": -0.9222656488418579,
      "loss": 0.8728,
      "nll_loss": 0.839062511920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0687255859375,
      "rewards/margins": 0.02349700964987278,
      "rewards/rejected": -0.09223632514476776,
      "step": 14940
    },
    {
      "epoch": 0.5602293380300163,
      "grad_norm": 1.9740877066697355,
      "learning_rate": 6.542886560876246e-07,
      "log_odds_chosen": 0.4530273377895355,
      "log_odds_ratio": -0.597949206829071,
      "logits/chosen": -0.7708984613418579,
      "logits/rejected": -0.6734374761581421,
      "logps/chosen": -0.680957019329071,
      "logps/rejected": -0.953320324420929,
      "loss": 0.9085,
      "nll_loss": 0.837890625,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06815185397863388,
      "rewards/margins": 0.02717285230755806,
      "rewards/rejected": -0.09526367485523224,
      "step": 14950
    },
    {
      "epoch": 0.5606040733731802,
      "grad_norm": 1.931084229247889,
      "learning_rate": 6.540699401677286e-07,
      "log_odds_chosen": 0.494293212890625,
      "log_odds_ratio": -0.5831054449081421,
      "logits/chosen": -0.7164062261581421,
      "logits/rejected": -0.6128906011581421,
      "logps/chosen": -0.6640625,
      "logps/rejected": -0.95703125,
      "loss": 0.8673,
      "nll_loss": 0.755664050579071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06635741889476776,
      "rewards/margins": 0.029291534796357155,
      "rewards/rejected": -0.09565429389476776,
      "step": 14960
    },
    {
      "epoch": 0.5609788087163441,
      "grad_norm": 2.0688741393205725,
      "learning_rate": 6.538514434386762e-07,
      "log_odds_chosen": 0.48394775390625,
      "log_odds_ratio": -0.5923827886581421,
      "logits/chosen": -0.7748047113418579,
      "logits/rejected": -0.6551758050918579,
      "logps/chosen": -0.6717773675918579,
      "logps/rejected": -0.9849609136581421,
      "loss": 0.8867,
      "nll_loss": 0.792285144329071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.067138671875,
      "rewards/margins": 0.031308747828006744,
      "rewards/rejected": -0.09847412258386612,
      "step": 14970
    },
    {
      "epoch": 0.561353544059508,
      "grad_norm": 1.9650581826748803,
      "learning_rate": 6.536331655346004e-07,
      "log_odds_chosen": 0.5532592535018921,
      "log_odds_ratio": -0.5628906488418579,
      "logits/chosen": -0.7708984613418579,
      "logits/rejected": -0.6556640863418579,
      "logps/chosen": -0.655468761920929,
      "logps/rejected": -0.991406261920929,
      "loss": 0.8773,
      "nll_loss": 0.736523449420929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0655517578125,
      "rewards/margins": 0.0335693359375,
      "rewards/rejected": -0.09909667819738388,
      "step": 14980
    },
    {
      "epoch": 0.5617282794026719,
      "grad_norm": 2.390569665388273,
      "learning_rate": 6.534151060904888e-07,
      "log_odds_chosen": 0.7598632574081421,
      "log_odds_ratio": -0.49604493379592896,
      "logits/chosen": -0.7021484375,
      "logits/rejected": -0.58642578125,
      "logps/chosen": -0.674121081829071,
      "logps/rejected": -1.135351538658142,
      "loss": 0.8688,
      "nll_loss": 0.8359375,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06745605170726776,
      "rewards/margins": 0.0461273193359375,
      "rewards/rejected": -0.11359862983226776,
      "step": 14990
    },
    {
      "epoch": 0.5621030147458358,
      "grad_norm": 2.0628781671595577,
      "learning_rate": 6.531972647421808e-07,
      "log_odds_chosen": 0.571728527545929,
      "log_odds_ratio": -0.5838867425918579,
      "logits/chosen": -0.7289062738418579,
      "logits/rejected": -0.647265613079071,
      "logps/chosen": -0.682910144329071,
      "logps/rejected": -1.022851586341858,
      "loss": 0.8793,
      "nll_loss": 0.8316406011581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06828613579273224,
      "rewards/margins": 0.033935546875,
      "rewards/rejected": -0.10222168266773224,
      "step": 15000
    },
    {
      "epoch": 0.5621030147458358,
      "eval_log_odds_chosen": 0.38765624165534973,
      "eval_log_odds_ratio": -0.6348696947097778,
      "eval_logits/chosen": -0.6853703856468201,
      "eval_logits/rejected": -0.589185893535614,
      "eval_logps/chosen": -0.743023157119751,
      "eval_logps/rejected": -1.0043553113937378,
      "eval_loss": 1.0970234870910645,
      "eval_nll_loss": 1.0358529090881348,
      "eval_rewards/accuracies": 0.5886294841766357,
      "eval_rewards/chosen": -0.07430221885442734,
      "eval_rewards/margins": 0.02613230235874653,
      "eval_rewards/rejected": -0.10042962431907654,
      "eval_runtime": 2431.3985,
      "eval_samples_per_second": 78.07,
      "eval_steps_per_second": 1.22,
      "step": 15000
    },
    {
      "epoch": 0.5624777500889997,
      "grad_norm": 1.8428833955529094,
      "learning_rate": 6.529796411263649e-07,
      "log_odds_chosen": 0.42072755098342896,
      "log_odds_ratio": -0.619140625,
      "logits/chosen": -0.7398437261581421,
      "logits/rejected": -0.6585937738418579,
      "logps/chosen": -0.7113281488418579,
      "logps/rejected": -0.964062511920929,
      "loss": 0.891,
      "nll_loss": 0.844531238079071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07110595703125,
      "rewards/margins": 0.025269318372011185,
      "rewards/rejected": -0.09637451171875,
      "step": 15010
    },
    {
      "epoch": 0.5628524854321635,
      "grad_norm": 2.196447393828265,
      "learning_rate": 6.527622348805765e-07,
      "log_odds_chosen": 0.5714355707168579,
      "log_odds_ratio": -0.5654296875,
      "logits/chosen": -0.784375011920929,
      "logits/rejected": -0.65087890625,
      "logps/chosen": -0.630664050579071,
      "logps/rejected": -0.971875011920929,
      "loss": 0.8832,
      "nll_loss": 0.7464843988418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06306152045726776,
      "rewards/margins": 0.03412475436925888,
      "rewards/rejected": -0.09716796875,
      "step": 15020
    },
    {
      "epoch": 0.5632272207753274,
      "grad_norm": 2.4293547040367676,
      "learning_rate": 6.525450456431951e-07,
      "log_odds_chosen": 0.549121081829071,
      "log_odds_ratio": -0.5909179449081421,
      "logits/chosen": -0.7685546875,
      "logits/rejected": -0.6361328363418579,
      "logps/chosen": -0.672558605670929,
      "logps/rejected": -1.025976538658142,
      "loss": 0.8955,
      "nll_loss": 0.8578125238418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0672607421875,
      "rewards/margins": 0.03541412204504013,
      "rewards/rejected": -0.10260009765625,
      "step": 15030
    },
    {
      "epoch": 0.5636019561184913,
      "grad_norm": 1.9987439357868035,
      "learning_rate": 6.523280730534421e-07,
      "log_odds_chosen": 0.6279296875,
      "log_odds_ratio": -0.560839831829071,
      "logits/chosen": -0.7978515625,
      "logits/rejected": -0.6576172113418579,
      "logps/chosen": -0.6527343988418579,
      "logps/rejected": -1.0625,
      "loss": 0.8758,
      "nll_loss": 0.820117175579071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06535644829273224,
      "rewards/margins": 0.041010282933712006,
      "rewards/rejected": -0.1063232421875,
      "step": 15040
    },
    {
      "epoch": 0.5639766914616552,
      "grad_norm": 1.6817430844466437,
      "learning_rate": 6.521113167513779e-07,
      "log_odds_chosen": 0.584301769733429,
      "log_odds_ratio": -0.5850585699081421,
      "logits/chosen": -0.6817382574081421,
      "logits/rejected": -0.627734363079071,
      "logps/chosen": -0.679394543170929,
      "logps/rejected": -1.0398437976837158,
      "loss": 0.8848,
      "nll_loss": 0.836132824420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06796874850988388,
      "rewards/margins": 0.03603210300207138,
      "rewards/rejected": -0.10393066704273224,
      "step": 15050
    },
    {
      "epoch": 0.5643514268048191,
      "grad_norm": 1.9645909966294755,
      "learning_rate": 6.518947763778994e-07,
      "log_odds_chosen": 0.49165040254592896,
      "log_odds_ratio": -0.5780273675918579,
      "logits/chosen": -0.708984375,
      "logits/rejected": -0.597363293170929,
      "logps/chosen": -0.716015636920929,
      "logps/rejected": -1.037695288658142,
      "loss": 0.8835,
      "nll_loss": 0.807812511920929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07155761867761612,
      "rewards/margins": 0.032277680933475494,
      "rewards/rejected": -0.10386963188648224,
      "step": 15060
    },
    {
      "epoch": 0.564726162147983,
      "grad_norm": 2.010603029778389,
      "learning_rate": 6.516784515747379e-07,
      "log_odds_chosen": 0.5499511957168579,
      "log_odds_ratio": -0.5775390863418579,
      "logits/chosen": -0.7552734613418579,
      "logits/rejected": -0.6747070550918579,
      "logps/chosen": -0.722460925579071,
      "logps/rejected": -1.078125,
      "loss": 0.881,
      "nll_loss": 0.890625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07220458984375,
      "rewards/margins": 0.035695649683475494,
      "rewards/rejected": -0.10795898735523224,
      "step": 15070
    },
    {
      "epoch": 0.5651008974911469,
      "grad_norm": 2.011796584994902,
      "learning_rate": 6.514623419844562e-07,
      "log_odds_chosen": 0.47734373807907104,
      "log_odds_ratio": -0.5960937738418579,
      "logits/chosen": -0.6869140863418579,
      "logits/rejected": -0.6234375238418579,
      "logps/chosen": -0.63720703125,
      "logps/rejected": -0.916015625,
      "loss": 0.8915,
      "nll_loss": 0.817578136920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06375732272863388,
      "rewards/margins": 0.02790374681353569,
      "rewards/rejected": -0.09160156548023224,
      "step": 15080
    },
    {
      "epoch": 0.5654756328343108,
      "grad_norm": 1.807881335495859,
      "learning_rate": 6.512464472504465e-07,
      "log_odds_chosen": 0.48908692598342896,
      "log_odds_ratio": -0.616992175579071,
      "logits/chosen": -0.793749988079071,
      "logits/rejected": -0.71484375,
      "logps/chosen": -0.687695324420929,
      "logps/rejected": -0.9951171875,
      "loss": 0.8834,
      "nll_loss": 0.8169921636581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06878662109375,
      "rewards/margins": 0.030727386474609375,
      "rewards/rejected": -0.09953613579273224,
      "step": 15090
    },
    {
      "epoch": 0.5658503681774747,
      "grad_norm": 2.2109630449518223,
      "learning_rate": 6.510307670169275e-07,
      "log_odds_chosen": 0.35816651582717896,
      "log_odds_ratio": -0.6368163824081421,
      "logits/chosen": -0.7093750238418579,
      "logits/rejected": -0.6415039300918579,
      "logps/chosen": -0.716992199420929,
      "logps/rejected": -0.950390636920929,
      "loss": 0.887,
      "nll_loss": 0.9253906011581421,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07167968899011612,
      "rewards/margins": 0.02339782752096653,
      "rewards/rejected": -0.09512939304113388,
      "step": 15100
    },
    {
      "epoch": 0.5662251035206386,
      "grad_norm": 2.0419684871309007,
      "learning_rate": 6.508153009289422e-07,
      "log_odds_chosen": 0.37761229276657104,
      "log_odds_ratio": -0.6255859136581421,
      "logits/chosen": -0.76171875,
      "logits/rejected": -0.6724609136581421,
      "logps/chosen": -0.668652355670929,
      "logps/rejected": -0.917187511920929,
      "loss": 0.8813,
      "nll_loss": 0.8392578363418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06687011569738388,
      "rewards/margins": 0.02480774000287056,
      "rewards/rejected": -0.09165038913488388,
      "step": 15110
    },
    {
      "epoch": 0.5665998388638024,
      "grad_norm": 1.7616147530336506,
      "learning_rate": 6.506000486323554e-07,
      "log_odds_chosen": 0.592517077922821,
      "log_odds_ratio": -0.571484386920929,
      "logits/chosen": -0.7147461175918579,
      "logits/rejected": -0.611621081829071,
      "logps/chosen": -0.6605468988418579,
      "logps/rejected": -1.014257788658142,
      "loss": 0.8795,
      "nll_loss": 0.7740234136581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06610107421875,
      "rewards/margins": 0.03538971021771431,
      "rewards/rejected": -0.10141601413488388,
      "step": 15120
    },
    {
      "epoch": 0.5669745742069663,
      "grad_norm": 2.03013682012621,
      "learning_rate": 6.503850097738512e-07,
      "log_odds_chosen": 0.59381103515625,
      "log_odds_ratio": -0.546093761920929,
      "logits/chosen": -0.758496105670929,
      "logits/rejected": -0.6148437261581421,
      "logps/chosen": -0.6763671636581421,
      "logps/rejected": -1.062890648841858,
      "loss": 0.8781,
      "nll_loss": 0.793164074420929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06759033352136612,
      "rewards/margins": 0.03863525390625,
      "rewards/rejected": -0.10627441108226776,
      "step": 15130
    },
    {
      "epoch": 0.5673493095501302,
      "grad_norm": 1.9598341435099877,
      "learning_rate": 6.50170184000931e-07,
      "log_odds_chosen": 0.46693116426467896,
      "log_odds_ratio": -0.611132800579071,
      "logits/chosen": -0.7689453363418579,
      "logits/rejected": -0.6488281488418579,
      "logps/chosen": -0.6929687261581421,
      "logps/rejected": -1.0138671398162842,
      "loss": 0.8781,
      "nll_loss": 0.822070300579071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06929931789636612,
      "rewards/margins": 0.0320281982421875,
      "rewards/rejected": -0.10134277492761612,
      "step": 15140
    },
    {
      "epoch": 0.5677240448932941,
      "grad_norm": 1.9761033818159592,
      "learning_rate": 6.4995557096191e-07,
      "log_odds_chosen": 0.537792980670929,
      "log_odds_ratio": -0.564160168170929,
      "logits/chosen": -0.761523425579071,
      "logits/rejected": -0.6791015863418579,
      "logps/chosen": -0.6522461175918579,
      "logps/rejected": -0.9546874761581421,
      "loss": 0.8866,
      "nll_loss": 0.807421863079071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06520996242761612,
      "rewards/margins": 0.03029632568359375,
      "rewards/rejected": -0.09549560397863388,
      "step": 15150
    },
    {
      "epoch": 0.568098780236458,
      "grad_norm": 2.1590517869018964,
      "learning_rate": 6.49741170305916e-07,
      "log_odds_chosen": 0.4337524473667145,
      "log_odds_ratio": -0.608105480670929,
      "logits/chosen": -0.71044921875,
      "logits/rejected": -0.612597644329071,
      "logps/chosen": -0.6796875,
      "logps/rejected": -0.9605468511581421,
      "loss": 0.8798,
      "nll_loss": 0.8324218988418579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06804199516773224,
      "rewards/margins": 0.02802734449505806,
      "rewards/rejected": -0.09602050483226776,
      "step": 15160
    },
    {
      "epoch": 0.5684735155796219,
      "grad_norm": 2.1846793183394815,
      "learning_rate": 6.495269816828862e-07,
      "log_odds_chosen": 0.633251965045929,
      "log_odds_ratio": -0.552929699420929,
      "logits/chosen": -0.7466796636581421,
      "logits/rejected": -0.640429675579071,
      "logps/chosen": -0.681835949420929,
      "logps/rejected": -1.064453125,
      "loss": 0.8682,
      "nll_loss": 0.807812511920929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06813964992761612,
      "rewards/margins": 0.03827819973230362,
      "rewards/rejected": -0.10651855170726776,
      "step": 15170
    },
    {
      "epoch": 0.5688482509227858,
      "grad_norm": 2.224240266929831,
      "learning_rate": 6.493130047435654e-07,
      "log_odds_chosen": 0.5973144769668579,
      "log_odds_ratio": -0.5909179449081421,
      "logits/chosen": -0.773242175579071,
      "logits/rejected": -0.6421874761581421,
      "logps/chosen": -0.693652331829071,
      "logps/rejected": -1.079492211341858,
      "loss": 0.8692,
      "nll_loss": 0.8492187261581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06932373344898224,
      "rewards/margins": 0.03862304612994194,
      "rewards/rejected": -0.1080322265625,
      "step": 15180
    },
    {
      "epoch": 0.5692229862659497,
      "grad_norm": 2.1407229119870217,
      "learning_rate": 6.490992391395026e-07,
      "log_odds_chosen": 0.4519287049770355,
      "log_odds_ratio": -0.596484363079071,
      "logits/chosen": -0.787109375,
      "logits/rejected": -0.654296875,
      "logps/chosen": -0.6514648199081421,
      "logps/rejected": -0.923632800579071,
      "loss": 0.8846,
      "nll_loss": 0.7669922113418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06511230766773224,
      "rewards/margins": 0.02723388746380806,
      "rewards/rejected": -0.09237060695886612,
      "step": 15190
    },
    {
      "epoch": 0.5695977216091136,
      "grad_norm": 1.8642226027321926,
      "learning_rate": 6.488856845230501e-07,
      "log_odds_chosen": 0.5368407964706421,
      "log_odds_ratio": -0.5733398199081421,
      "logits/chosen": -0.7876952886581421,
      "logits/rejected": -0.6377929449081421,
      "logps/chosen": -0.6669921875,
      "logps/rejected": -1.01171875,
      "loss": 0.8867,
      "nll_loss": 0.810253918170929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06668701022863388,
      "rewards/margins": 0.03446044772863388,
      "rewards/rejected": -0.1011962890625,
      "step": 15200
    },
    {
      "epoch": 0.5699724569522775,
      "grad_norm": 1.9233098840743637,
      "learning_rate": 6.486723405473599e-07,
      "log_odds_chosen": 0.48823243379592896,
      "log_odds_ratio": -0.602246105670929,
      "logits/chosen": -0.7496093511581421,
      "logits/rejected": -0.655468761920929,
      "logps/chosen": -0.673535168170929,
      "logps/rejected": -0.9505859613418579,
      "loss": 0.8779,
      "nll_loss": 0.8335937261581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06735839694738388,
      "rewards/margins": 0.02769775316119194,
      "rewards/rejected": -0.0950927734375,
      "step": 15210
    },
    {
      "epoch": 0.5703471922954414,
      "grad_norm": 2.036795997401011,
      "learning_rate": 6.484592068663816e-07,
      "log_odds_chosen": 0.42841798067092896,
      "log_odds_ratio": -0.6392577886581421,
      "logits/chosen": -0.7689453363418579,
      "logits/rejected": -0.6625000238418579,
      "logps/chosen": -0.693164050579071,
      "logps/rejected": -0.9662109613418579,
      "loss": 0.8884,
      "nll_loss": 0.8609374761581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06931152194738388,
      "rewards/margins": 0.02734680101275444,
      "rewards/rejected": -0.09660644829273224,
      "step": 15220
    },
    {
      "epoch": 0.5707219276386052,
      "grad_norm": 2.10609806624468,
      "learning_rate": 6.482462831348603e-07,
      "log_odds_chosen": 0.38872069120407104,
      "log_odds_ratio": -0.630664050579071,
      "logits/chosen": -0.758593738079071,
      "logits/rejected": -0.6766601800918579,
      "logps/chosen": -0.6832031011581421,
      "logps/rejected": -0.9486328363418579,
      "loss": 0.8848,
      "nll_loss": 0.806835949420929,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06831054389476776,
      "rewards/margins": 0.02652587927877903,
      "rewards/rejected": -0.0948486328125,
      "step": 15230
    },
    {
      "epoch": 0.5710966629817691,
      "grad_norm": 1.7672351232864927,
      "learning_rate": 6.480335690083345e-07,
      "log_odds_chosen": 0.5618896484375,
      "log_odds_ratio": -0.57568359375,
      "logits/chosen": -0.759960949420929,
      "logits/rejected": -0.6795898675918579,
      "logps/chosen": -0.681933581829071,
      "logps/rejected": -1.040624976158142,
      "loss": 0.8725,
      "nll_loss": 0.798828125,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06820068508386612,
      "rewards/margins": 0.03590087965130806,
      "rewards/rejected": -0.1041259765625,
      "step": 15240
    },
    {
      "epoch": 0.571471398324933,
      "grad_norm": 1.8793074374197312,
      "learning_rate": 6.478210641431327e-07,
      "log_odds_chosen": 0.5955810546875,
      "log_odds_ratio": -0.559863269329071,
      "logits/chosen": -0.7391601800918579,
      "logits/rejected": -0.676464855670929,
      "logps/chosen": -0.708203136920929,
      "logps/rejected": -1.072265625,
      "loss": 0.8694,
      "nll_loss": 0.8531249761581421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07082519680261612,
      "rewards/margins": 0.036370087414979935,
      "rewards/rejected": -0.10725097358226776,
      "step": 15250
    },
    {
      "epoch": 0.5718461336680969,
      "grad_norm": 1.9902118914349856,
      "learning_rate": 6.476087681963725e-07,
      "log_odds_chosen": 0.5298217535018921,
      "log_odds_ratio": -0.5819336175918579,
      "logits/chosen": -0.7642577886581421,
      "logits/rejected": -0.669921875,
      "logps/chosen": -0.6458984613418579,
      "logps/rejected": -0.968945324420929,
      "loss": 0.8735,
      "nll_loss": 0.8490234613418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06462402641773224,
      "rewards/margins": 0.03227081149816513,
      "rewards/rejected": -0.09688720852136612,
      "step": 15260
    },
    {
      "epoch": 0.5722208690112608,
      "grad_norm": 2.055284239179166,
      "learning_rate": 6.473966808259572e-07,
      "log_odds_chosen": 0.638354480266571,
      "log_odds_ratio": -0.561718761920929,
      "logits/chosen": -0.788281261920929,
      "logits/rejected": -0.671679675579071,
      "logps/chosen": -0.6485351324081421,
      "logps/rejected": -1.0546875,
      "loss": 0.8955,
      "nll_loss": 0.7964843511581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06483153998851776,
      "rewards/margins": 0.040604401379823685,
      "rewards/rejected": -0.10549316555261612,
      "step": 15270
    },
    {
      "epoch": 0.5725956043544247,
      "grad_norm": 1.9604835161945768,
      "learning_rate": 6.471848016905735e-07,
      "log_odds_chosen": 0.658740222454071,
      "log_odds_ratio": -0.5498046875,
      "logits/chosen": -0.7308593988418579,
      "logits/rejected": -0.622265636920929,
      "logps/chosen": -0.631542980670929,
      "logps/rejected": -1.033593773841858,
      "loss": 0.8814,
      "nll_loss": 0.797656238079071,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.063201904296875,
      "rewards/margins": 0.04015655443072319,
      "rewards/rejected": -0.10338135063648224,
      "step": 15280
    },
    {
      "epoch": 0.5729703396975886,
      "grad_norm": 1.9745288690951122,
      "learning_rate": 6.469731304496901e-07,
      "log_odds_chosen": 0.5916992425918579,
      "log_odds_ratio": -0.5723632574081421,
      "logits/chosen": -0.754687488079071,
      "logits/rejected": -0.6480468511581421,
      "logps/chosen": -0.6695312261581421,
      "logps/rejected": -1.036523461341858,
      "loss": 0.881,
      "nll_loss": 0.8287109136581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06696777045726776,
      "rewards/margins": 0.03675232082605362,
      "rewards/rejected": -0.10368652641773224,
      "step": 15290
    },
    {
      "epoch": 0.5733450750407525,
      "grad_norm": 1.9177388821965293,
      "learning_rate": 6.467616667635546e-07,
      "log_odds_chosen": 0.39750367403030396,
      "log_odds_ratio": -0.6240234375,
      "logits/chosen": -0.73828125,
      "logits/rejected": -0.661328136920929,
      "logps/chosen": -0.6636718511581421,
      "logps/rejected": -0.9156249761581421,
      "loss": 0.8723,
      "nll_loss": 0.7974609136581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06627197563648224,
      "rewards/margins": 0.025279998779296875,
      "rewards/rejected": -0.09158935397863388,
      "step": 15300
    },
    {
      "epoch": 0.5737198103839164,
      "grad_norm": 1.901887058018343,
      "learning_rate": 6.465504102931912e-07,
      "log_odds_chosen": 0.43854981660842896,
      "log_odds_ratio": -0.617480456829071,
      "logits/chosen": -0.7601562738418579,
      "logits/rejected": -0.6484375,
      "logps/chosen": -0.6615234613418579,
      "logps/rejected": -0.935351550579071,
      "loss": 0.8672,
      "nll_loss": 0.7978515625,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06617431342601776,
      "rewards/margins": 0.02741241455078125,
      "rewards/rejected": -0.09349365532398224,
      "step": 15310
    },
    {
      "epoch": 0.5740945457270803,
      "grad_norm": 1.8684028486804616,
      "learning_rate": 6.463393607003991e-07,
      "log_odds_chosen": 0.3863281309604645,
      "log_odds_ratio": -0.645312488079071,
      "logits/chosen": -0.716992199420929,
      "logits/rejected": -0.689453125,
      "logps/chosen": -0.6957031488418579,
      "logps/rejected": -0.959765613079071,
      "loss": 0.8847,
      "nll_loss": 0.85546875,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.06954345852136612,
      "rewards/margins": 0.02641601487994194,
      "rewards/rejected": -0.09603271633386612,
      "step": 15320
    },
    {
      "epoch": 0.5744692810702441,
      "grad_norm": 1.8572152248790954,
      "learning_rate": 6.461285176477491e-07,
      "log_odds_chosen": 0.527111828327179,
      "log_odds_ratio": -0.593457043170929,
      "logits/chosen": -0.7579101324081421,
      "logits/rejected": -0.6322265863418579,
      "logps/chosen": -0.68212890625,
      "logps/rejected": -1.020898461341858,
      "loss": 0.8757,
      "nll_loss": 0.8148437738418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06817626953125,
      "rewards/margins": 0.03399963304400444,
      "rewards/rejected": -0.1021728515625,
      "step": 15330
    },
    {
      "epoch": 0.574844016413408,
      "grad_norm": 2.0912563040685854,
      "learning_rate": 6.45917880798583e-07,
      "log_odds_chosen": 0.36248779296875,
      "log_odds_ratio": -0.629589855670929,
      "logits/chosen": -0.722851574420929,
      "logits/rejected": -0.6181640625,
      "logps/chosen": -0.71875,
      "logps/rejected": -0.962109386920929,
      "loss": 0.8671,
      "nll_loss": 0.862500011920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07181396335363388,
      "rewards/margins": 0.024460602551698685,
      "rewards/rejected": -0.0963134765625,
      "step": 15340
    },
    {
      "epoch": 0.5752187517565719,
      "grad_norm": 2.0310703499960003,
      "learning_rate": 6.457074498170094e-07,
      "log_odds_chosen": 0.545605480670929,
      "log_odds_ratio": -0.5726562738418579,
      "logits/chosen": -0.791015625,
      "logits/rejected": -0.682910144329071,
      "logps/chosen": -0.705859363079071,
      "logps/rejected": -1.0525391101837158,
      "loss": 0.894,
      "nll_loss": 0.857617199420929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07061767578125,
      "rewards/margins": 0.03455505520105362,
      "rewards/rejected": -0.10524902492761612,
      "step": 15350
    },
    {
      "epoch": 0.5755934870997358,
      "grad_norm": 1.9593698963686867,
      "learning_rate": 6.454972243679028e-07,
      "log_odds_chosen": 0.7340332269668579,
      "log_odds_ratio": -0.523632824420929,
      "logits/chosen": -0.8324218988418579,
      "logits/rejected": -0.690234363079071,
      "logps/chosen": -0.66943359375,
      "logps/rejected": -1.1248047351837158,
      "loss": 0.8864,
      "nll_loss": 0.777539074420929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06694336235523224,
      "rewards/margins": 0.04554290696978569,
      "rewards/rejected": -0.1124267578125,
      "step": 15360
    },
    {
      "epoch": 0.5759682224428997,
      "grad_norm": 2.0635992288940335,
      "learning_rate": 6.452872041169011e-07,
      "log_odds_chosen": 0.4528442323207855,
      "log_odds_ratio": -0.6234375238418579,
      "logits/chosen": -0.797656238079071,
      "logits/rejected": -0.682324230670929,
      "logps/chosen": -0.716992199420929,
      "logps/rejected": -1.013671875,
      "loss": 0.8658,
      "nll_loss": 0.781054675579071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07175292819738388,
      "rewards/margins": 0.02957916259765625,
      "rewards/rejected": -0.101318359375,
      "step": 15370
    },
    {
      "epoch": 0.5763429577860636,
      "grad_norm": 1.7443158538150576,
      "learning_rate": 6.450773887304029e-07,
      "log_odds_chosen": 0.36442869901657104,
      "log_odds_ratio": -0.6371093988418579,
      "logits/chosen": -0.7816406488418579,
      "logits/rejected": -0.699023425579071,
      "logps/chosen": -0.7154296636581421,
      "logps/rejected": -0.940234363079071,
      "loss": 0.8791,
      "nll_loss": 0.835156261920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07155761867761612,
      "rewards/margins": 0.02252502366900444,
      "rewards/rejected": -0.09401855617761612,
      "step": 15380
    },
    {
      "epoch": 0.5767176931292275,
      "grad_norm": 1.9478149290500337,
      "learning_rate": 6.448677778755658e-07,
      "log_odds_chosen": 0.4894042909145355,
      "log_odds_ratio": -0.619140625,
      "logits/chosen": -0.779296875,
      "logits/rejected": -0.673828125,
      "logps/chosen": -0.6763671636581421,
      "logps/rejected": -1.003320336341858,
      "loss": 0.8604,
      "nll_loss": 0.8070312738418579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06761474907398224,
      "rewards/margins": 0.032749176025390625,
      "rewards/rejected": -0.10043945163488388,
      "step": 15390
    },
    {
      "epoch": 0.5770924284723914,
      "grad_norm": 2.2031949959007626,
      "learning_rate": 6.446583712203042e-07,
      "log_odds_chosen": 0.551074206829071,
      "log_odds_ratio": -0.573535144329071,
      "logits/chosen": -0.798632800579071,
      "logits/rejected": -0.705859363079071,
      "logps/chosen": -0.66748046875,
      "logps/rejected": -1.012109398841858,
      "loss": 0.8704,
      "nll_loss": 0.8193359375,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06672362983226776,
      "rewards/margins": 0.03447265550494194,
      "rewards/rejected": -0.10118408501148224,
      "step": 15400
    },
    {
      "epoch": 0.5774671638155553,
      "grad_norm": 1.9896846660910514,
      "learning_rate": 6.444491684332863e-07,
      "log_odds_chosen": 0.42054444551467896,
      "log_odds_ratio": -0.6175781488418579,
      "logits/chosen": -0.717089831829071,
      "logits/rejected": -0.6670898199081421,
      "logps/chosen": -0.7212890386581421,
      "logps/rejected": -0.9691406488418579,
      "loss": 0.8752,
      "nll_loss": 0.8248046636581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07209472358226776,
      "rewards/margins": 0.02476806566119194,
      "rewards/rejected": -0.096923828125,
      "step": 15410
    },
    {
      "epoch": 0.5778418991587192,
      "grad_norm": 1.8688288577476009,
      "learning_rate": 6.442401691839331e-07,
      "log_odds_chosen": 0.542980968952179,
      "log_odds_ratio": -0.5586913824081421,
      "logits/chosen": -0.7392578125,
      "logits/rejected": -0.6141601800918579,
      "logps/chosen": -0.62841796875,
      "logps/rejected": -0.964062511920929,
      "loss": 0.8568,
      "nll_loss": 0.7894531488418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06287841498851776,
      "rewards/margins": 0.03359832614660263,
      "rewards/rejected": -0.096435546875,
      "step": 15420
    },
    {
      "epoch": 0.5782166345018831,
      "grad_norm": 2.0767532465637277,
      "learning_rate": 6.440313731424148e-07,
      "log_odds_chosen": 0.6619507074356079,
      "log_odds_ratio": -0.55078125,
      "logits/chosen": -0.712890625,
      "logits/rejected": -0.619140625,
      "logps/chosen": -0.6552734375,
      "logps/rejected": -1.087304711341858,
      "loss": 0.8729,
      "nll_loss": 0.8316406011581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0655517578125,
      "rewards/margins": 0.04316864162683487,
      "rewards/rejected": -0.10869140923023224,
      "step": 15430
    },
    {
      "epoch": 0.5785913698450469,
      "grad_norm": 2.007252838126507,
      "learning_rate": 6.438227799796504e-07,
      "log_odds_chosen": 0.6005614995956421,
      "log_odds_ratio": -0.534863293170929,
      "logits/chosen": -0.7289062738418579,
      "logits/rejected": -0.63134765625,
      "logps/chosen": -0.6260741949081421,
      "logps/rejected": -0.9632812738418579,
      "loss": 0.8668,
      "nll_loss": 0.7763671875,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06259765475988388,
      "rewards/margins": 0.033765412867069244,
      "rewards/rejected": -0.09630127251148224,
      "step": 15440
    },
    {
      "epoch": 0.5789661051882108,
      "grad_norm": 2.030533095255955,
      "learning_rate": 6.436143893673037e-07,
      "log_odds_chosen": 0.3837890625,
      "log_odds_ratio": -0.621386706829071,
      "logits/chosen": -0.711718738079071,
      "logits/rejected": -0.6327148675918579,
      "logps/chosen": -0.6771484613418579,
      "logps/rejected": -0.9144531488418579,
      "loss": 0.8816,
      "nll_loss": 0.8880859613418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06773681938648224,
      "rewards/margins": 0.02363891527056694,
      "rewards/rejected": -0.09140624850988388,
      "step": 15450
    },
    {
      "epoch": 0.5793408405313747,
      "grad_norm": 2.2418519742626173,
      "learning_rate": 6.434062009777823e-07,
      "log_odds_chosen": 0.5201416015625,
      "log_odds_ratio": -0.58642578125,
      "logits/chosen": -0.776171863079071,
      "logits/rejected": -0.686816394329071,
      "logps/chosen": -0.7021484375,
      "logps/rejected": -1.0490233898162842,
      "loss": 0.849,
      "nll_loss": 0.748730480670929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07022704929113388,
      "rewards/margins": 0.034698486328125,
      "rewards/rejected": -0.1048583984375,
      "step": 15460
    },
    {
      "epoch": 0.5797155758745386,
      "grad_norm": 2.21163517806057,
      "learning_rate": 6.431982144842349e-07,
      "log_odds_chosen": 0.41358643770217896,
      "log_odds_ratio": -0.6094726324081421,
      "logits/chosen": -0.72265625,
      "logits/rejected": -0.62939453125,
      "logps/chosen": -0.705859363079071,
      "logps/rejected": -0.984375,
      "loss": 0.8824,
      "nll_loss": 0.8345702886581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07062987983226776,
      "rewards/margins": 0.0278167724609375,
      "rewards/rejected": -0.09841308742761612,
      "step": 15470
    },
    {
      "epoch": 0.5800903112177025,
      "grad_norm": 2.3135971321061017,
      "learning_rate": 6.429904295605495e-07,
      "log_odds_chosen": 0.6019531488418579,
      "log_odds_ratio": -0.549121081829071,
      "logits/chosen": -0.7496093511581421,
      "logits/rejected": -0.6357421875,
      "logps/chosen": -0.6820312738418579,
      "logps/rejected": -1.0673828125,
      "loss": 0.8728,
      "nll_loss": 0.7862304449081421,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06818847358226776,
      "rewards/margins": 0.03844146803021431,
      "rewards/rejected": -0.10661621391773224,
      "step": 15480
    },
    {
      "epoch": 0.5804650465608664,
      "grad_norm": 1.9855401206908359,
      "learning_rate": 6.42782845881351e-07,
      "log_odds_chosen": 0.43397217988967896,
      "log_odds_ratio": -0.6034179925918579,
      "logits/chosen": -0.6910156011581421,
      "logits/rejected": -0.633105456829071,
      "logps/chosen": -0.668164074420929,
      "logps/rejected": -0.9505859613418579,
      "loss": 0.861,
      "nll_loss": 0.8218749761581421,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.06682129204273224,
      "rewards/margins": 0.028289031237363815,
      "rewards/rejected": -0.09506835788488388,
      "step": 15490
    },
    {
      "epoch": 0.5808397819040303,
      "grad_norm": 2.05955130412697,
      "learning_rate": 6.42575463121999e-07,
      "log_odds_chosen": 0.5911620855331421,
      "log_odds_ratio": -0.572558581829071,
      "logits/chosen": -0.75390625,
      "logits/rejected": -0.6475585699081421,
      "logps/chosen": -0.707324206829071,
      "logps/rejected": -1.0771484375,
      "loss": 0.8792,
      "nll_loss": 0.841601550579071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07072754204273224,
      "rewards/margins": 0.03704528883099556,
      "rewards/rejected": -0.10773925483226776,
      "step": 15500
    },
    {
      "epoch": 0.5812145172471942,
      "grad_norm": 1.9696550405171507,
      "learning_rate": 6.423682809585863e-07,
      "log_odds_chosen": 0.636181652545929,
      "log_odds_ratio": -0.541210949420929,
      "logits/chosen": -0.748339831829071,
      "logits/rejected": -0.6239258050918579,
      "logps/chosen": -0.65625,
      "logps/rejected": -1.034765601158142,
      "loss": 0.8542,
      "nll_loss": 0.7845703363418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06560058891773224,
      "rewards/margins": 0.03785552829504013,
      "rewards/rejected": -0.10341797024011612,
      "step": 15510
    },
    {
      "epoch": 0.5815892525903581,
      "grad_norm": 2.106938492890502,
      "learning_rate": 6.421612990679356e-07,
      "log_odds_chosen": 0.4230590760707855,
      "log_odds_ratio": -0.62744140625,
      "logits/chosen": -0.72265625,
      "logits/rejected": -0.6283203363418579,
      "logps/chosen": -0.6875,
      "logps/rejected": -0.9638671875,
      "loss": 0.8798,
      "nll_loss": 0.8177734613418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06875000149011612,
      "rewards/margins": 0.02769165113568306,
      "rewards/rejected": -0.09645996242761612,
      "step": 15520
    },
    {
      "epoch": 0.581963987933522,
      "grad_norm": 1.9539510466816208,
      "learning_rate": 6.419545171275983e-07,
      "log_odds_chosen": 0.3837890625,
      "log_odds_ratio": -0.619140625,
      "logits/chosen": -0.7137695550918579,
      "logits/rejected": -0.629589855670929,
      "logps/chosen": -0.6543945074081421,
      "logps/rejected": -0.8921874761581421,
      "loss": 0.8635,
      "nll_loss": 0.7857421636581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06551513820886612,
      "rewards/margins": 0.023703765124082565,
      "rewards/rejected": -0.08929443359375,
      "step": 15530
    },
    {
      "epoch": 0.5823387232766858,
      "grad_norm": 1.9808920180826497,
      "learning_rate": 6.417479348158526e-07,
      "log_odds_chosen": 0.515850841999054,
      "log_odds_ratio": -0.5860351324081421,
      "logits/chosen": -0.745898425579071,
      "logits/rejected": -0.6260741949081421,
      "logps/chosen": -0.692578136920929,
      "logps/rejected": -1.0324218273162842,
      "loss": 0.8685,
      "nll_loss": 0.8038085699081421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06925048679113388,
      "rewards/margins": 0.0339813232421875,
      "rewards/rejected": -0.10322265326976776,
      "step": 15540
    },
    {
      "epoch": 0.5827134586198497,
      "grad_norm": 2.0964184473185017,
      "learning_rate": 6.415415518117003e-07,
      "log_odds_chosen": 0.5701659917831421,
      "log_odds_ratio": -0.582324206829071,
      "logits/chosen": -0.7447265386581421,
      "logits/rejected": -0.6537109613418579,
      "logps/chosen": -0.7054687738418579,
      "logps/rejected": -1.0714843273162842,
      "loss": 0.8756,
      "nll_loss": 0.822070300579071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07050780951976776,
      "rewards/margins": 0.03662414476275444,
      "rewards/rejected": -0.10712890326976776,
      "step": 15550
    },
    {
      "epoch": 0.5830881939630136,
      "grad_norm": 1.9632821460287415,
      "learning_rate": 6.413353677948659e-07,
      "log_odds_chosen": 0.5213378667831421,
      "log_odds_ratio": -0.584277331829071,
      "logits/chosen": -0.731249988079071,
      "logits/rejected": -0.6514648199081421,
      "logps/chosen": -0.666699230670929,
      "logps/rejected": -1.0066406726837158,
      "loss": 0.8617,
      "nll_loss": 0.7962890863418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06666259467601776,
      "rewards/margins": 0.03396911546587944,
      "rewards/rejected": -0.10062255710363388,
      "step": 15560
    },
    {
      "epoch": 0.5834629293061775,
      "grad_norm": 2.199091408502574,
      "learning_rate": 6.411293824457933e-07,
      "log_odds_chosen": 0.592730700969696,
      "log_odds_ratio": -0.573437511920929,
      "logits/chosen": -0.7291015386581421,
      "logits/rejected": -0.664257824420929,
      "logps/chosen": -0.67919921875,
      "logps/rejected": -1.087304711341858,
      "loss": 0.88,
      "nll_loss": 0.7808593511581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06789550930261612,
      "rewards/margins": 0.04090576246380806,
      "rewards/rejected": -0.1087646484375,
      "step": 15570
    },
    {
      "epoch": 0.5838376646493414,
      "grad_norm": 1.9575011003572969,
      "learning_rate": 6.409235954456451e-07,
      "log_odds_chosen": 0.5240844488143921,
      "log_odds_ratio": -0.57373046875,
      "logits/chosen": -0.759570300579071,
      "logits/rejected": -0.655078113079071,
      "logps/chosen": -0.678515613079071,
      "logps/rejected": -1.009765625,
      "loss": 0.8793,
      "nll_loss": 0.8208984136581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06782226264476776,
      "rewards/margins": 0.03318939357995987,
      "rewards/rejected": -0.1009521484375,
      "step": 15580
    },
    {
      "epoch": 0.5842123999925053,
      "grad_norm": 2.049027357611801,
      "learning_rate": 6.407180064762995e-07,
      "log_odds_chosen": 0.48613280057907104,
      "log_odds_ratio": -0.5829101800918579,
      "logits/chosen": -0.750781238079071,
      "logits/rejected": -0.6324218511581421,
      "logps/chosen": -0.65771484375,
      "logps/rejected": -0.9390624761581421,
      "loss": 0.8587,
      "nll_loss": 0.7650390863418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06575927883386612,
      "rewards/margins": 0.02817077562212944,
      "rewards/rejected": -0.09394530951976776,
      "step": 15590
    },
    {
      "epoch": 0.5845871353356692,
      "grad_norm": 2.1882950009426976,
      "learning_rate": 6.405126152203485e-07,
      "log_odds_chosen": 0.6062256097793579,
      "log_odds_ratio": -0.550097644329071,
      "logits/chosen": -0.786425769329071,
      "logits/rejected": -0.64599609375,
      "logps/chosen": -0.6629883050918579,
      "logps/rejected": -1.04296875,
      "loss": 0.8544,
      "nll_loss": 0.804492175579071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06633301079273224,
      "rewards/margins": 0.03796996921300888,
      "rewards/rejected": -0.10432128608226776,
      "step": 15600
    },
    {
      "epoch": 0.5849618706788331,
      "grad_norm": 1.8337831904807362,
      "learning_rate": 6.403074213610959e-07,
      "log_odds_chosen": 0.516430675983429,
      "log_odds_ratio": -0.603808581829071,
      "logits/chosen": -0.7109375,
      "logits/rejected": -0.6102539300918579,
      "logps/chosen": -0.65771484375,
      "logps/rejected": -0.9593750238418579,
      "loss": 0.8516,
      "nll_loss": 0.8189452886581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.0657958984375,
      "rewards/margins": 0.03017120435833931,
      "rewards/rejected": -0.095947265625,
      "step": 15610
    },
    {
      "epoch": 0.585336606021997,
      "grad_norm": 1.9156429056187778,
      "learning_rate": 6.401024245825555e-07,
      "log_odds_chosen": 0.46229857206344604,
      "log_odds_ratio": -0.6065429449081421,
      "logits/chosen": -0.7147461175918579,
      "logits/rejected": -0.645703136920929,
      "logps/chosen": -0.680468738079071,
      "logps/rejected": -0.9662109613418579,
      "loss": 0.8593,
      "nll_loss": 0.7939453125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06813964992761612,
      "rewards/margins": 0.02846374548971653,
      "rewards/rejected": -0.09660644829273224,
      "step": 15620
    },
    {
      "epoch": 0.5857113413651609,
      "grad_norm": 3.462610426498461,
      "learning_rate": 6.398976245694481e-07,
      "log_odds_chosen": 0.4691528379917145,
      "log_odds_ratio": -0.593554675579071,
      "logits/chosen": -0.726367175579071,
      "logits/rejected": -0.6498047113418579,
      "logps/chosen": -0.648632824420929,
      "logps/rejected": -0.912304699420929,
      "loss": 0.8626,
      "nll_loss": 0.7728515863418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06488037109375,
      "rewards/margins": 0.02635803259909153,
      "rewards/rejected": -0.09122314304113388,
      "step": 15630
    },
    {
      "epoch": 0.5860860767083248,
      "grad_norm": 1.926272309919125,
      "learning_rate": 6.396930210072012e-07,
      "log_odds_chosen": 0.3134399354457855,
      "log_odds_ratio": -0.6615234613418579,
      "logits/chosen": -0.773242175579071,
      "logits/rejected": -0.690234363079071,
      "logps/chosen": -0.7513672113418579,
      "logps/rejected": -0.9429687261581421,
      "loss": 0.8622,
      "nll_loss": 0.8023437261581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07521972805261612,
      "rewards/margins": 0.01912078820168972,
      "rewards/rejected": -0.09432373195886612,
      "step": 15640
    },
    {
      "epoch": 0.5864608120514886,
      "grad_norm": 1.8544062189535804,
      "learning_rate": 6.394886135819452e-07,
      "log_odds_chosen": 0.4737182557582855,
      "log_odds_ratio": -0.6412109136581421,
      "logits/chosen": -0.7337890863418579,
      "logits/rejected": -0.650390625,
      "logps/chosen": -0.6807616949081421,
      "logps/rejected": -0.983203113079071,
      "loss": 0.8682,
      "nll_loss": 0.8408203125,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06807861477136612,
      "rewards/margins": 0.03021698072552681,
      "rewards/rejected": -0.09835205227136612,
      "step": 15650
    },
    {
      "epoch": 0.5868355473946525,
      "grad_norm": 2.085191141363641,
      "learning_rate": 6.392844019805121e-07,
      "log_odds_chosen": 0.39716798067092896,
      "log_odds_ratio": -0.6273437738418579,
      "logits/chosen": -0.755859375,
      "logits/rejected": -0.673828125,
      "logps/chosen": -0.684277355670929,
      "logps/rejected": -0.9410156011581421,
      "loss": 0.8609,
      "nll_loss": 0.810742199420929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06838379055261612,
      "rewards/margins": 0.025787353515625,
      "rewards/rejected": -0.09420166164636612,
      "step": 15660
    },
    {
      "epoch": 0.5872102827378164,
      "grad_norm": 2.4809341650687484,
      "learning_rate": 6.39080385890434e-07,
      "log_odds_chosen": 0.4626708924770355,
      "log_odds_ratio": -0.612109363079071,
      "logits/chosen": -0.7724609375,
      "logits/rejected": -0.659375011920929,
      "logps/chosen": -0.6884765625,
      "logps/rejected": -0.9775390625,
      "loss": 0.881,
      "nll_loss": 0.850781261920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06887207180261612,
      "rewards/margins": 0.02898559533059597,
      "rewards/rejected": -0.09782715141773224,
      "step": 15670
    },
    {
      "epoch": 0.5875850180809803,
      "grad_norm": 2.220891978019128,
      "learning_rate": 6.388765649999399e-07,
      "log_odds_chosen": 0.5491698980331421,
      "log_odds_ratio": -0.5849609375,
      "logits/chosen": -0.751171886920929,
      "logits/rejected": -0.69140625,
      "logps/chosen": -0.6898437738418579,
      "logps/rejected": -1.0625,
      "loss": 0.8701,
      "nll_loss": 0.8851562738418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06893310695886612,
      "rewards/margins": 0.03728485107421875,
      "rewards/rejected": -0.1063232421875,
      "step": 15680
    },
    {
      "epoch": 0.5879597534241442,
      "grad_norm": 2.0530529064973377,
      "learning_rate": 6.38672938997955e-07,
      "log_odds_chosen": 0.5090087652206421,
      "log_odds_ratio": -0.603710949420929,
      "logits/chosen": -0.7279297113418579,
      "logits/rejected": -0.6231445074081421,
      "logps/chosen": -0.6748046875,
      "logps/rejected": -1.0031249523162842,
      "loss": 0.8891,
      "nll_loss": 0.825976550579071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06743164360523224,
      "rewards/margins": 0.032928466796875,
      "rewards/rejected": -0.10040283203125,
      "step": 15690
    },
    {
      "epoch": 0.5883344887673081,
      "grad_norm": 1.9180451336873032,
      "learning_rate": 6.384695075740977e-07,
      "log_odds_chosen": 0.43541258573532104,
      "log_odds_ratio": -0.643359363079071,
      "logits/chosen": -0.765820324420929,
      "logits/rejected": -0.6734374761581421,
      "logps/chosen": -0.708984375,
      "logps/rejected": -0.996289074420929,
      "loss": 0.8785,
      "nll_loss": 0.8447265625,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07095947116613388,
      "rewards/margins": 0.028697967529296875,
      "rewards/rejected": -0.09962157905101776,
      "step": 15700
    },
    {
      "epoch": 0.588709224110472,
      "grad_norm": 2.004028758787152,
      "learning_rate": 6.382662704186784e-07,
      "log_odds_chosen": 0.571606457233429,
      "log_odds_ratio": -0.598925769329071,
      "logits/chosen": -0.723828136920929,
      "logits/rejected": -0.6416991949081421,
      "logps/chosen": -0.685546875,
      "logps/rejected": -1.041601538658142,
      "loss": 0.884,
      "nll_loss": 0.771484375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06853027641773224,
      "rewards/margins": 0.035651396960020065,
      "rewards/rejected": -0.10416259616613388,
      "step": 15710
    },
    {
      "epoch": 0.5890839594536359,
      "grad_norm": 2.097237896677445,
      "learning_rate": 6.380632272226964e-07,
      "log_odds_chosen": 0.29267579317092896,
      "log_odds_ratio": -0.665332019329071,
      "logits/chosen": -0.7000976800918579,
      "logits/rejected": -0.6373046636581421,
      "logps/chosen": -0.7035156488418579,
      "logps/rejected": -0.894726574420929,
      "loss": 0.8427,
      "nll_loss": 0.8115234375,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07033691555261612,
      "rewards/margins": 0.01907501183450222,
      "rewards/rejected": -0.08941650390625,
      "step": 15720
    },
    {
      "epoch": 0.5894586947967998,
      "grad_norm": 2.047167470107154,
      "learning_rate": 6.378603776778394e-07,
      "log_odds_chosen": 0.4858154356479645,
      "log_odds_ratio": -0.6009765863418579,
      "logits/chosen": -0.7164062261581421,
      "logits/rejected": -0.645703136920929,
      "logps/chosen": -0.727734386920929,
      "logps/rejected": -1.0451171398162842,
      "loss": 0.8648,
      "nll_loss": 0.85546875,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07282714545726776,
      "rewards/margins": 0.03168945387005806,
      "rewards/rejected": -0.10450439155101776,
      "step": 15730
    },
    {
      "epoch": 0.5898334301399637,
      "grad_norm": 2.181601820142221,
      "learning_rate": 6.376577214764806e-07,
      "log_odds_chosen": 0.20122070610523224,
      "log_odds_ratio": -0.68310546875,
      "logits/chosen": -0.7662109136581421,
      "logits/rejected": -0.6773437261581421,
      "logps/chosen": -0.7333984375,
      "logps/rejected": -0.8531249761581421,
      "loss": 0.8877,
      "nll_loss": 0.836718738079071,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07335205376148224,
      "rewards/margins": 0.011989593505859375,
      "rewards/rejected": -0.08537597954273224,
      "step": 15740
    },
    {
      "epoch": 0.5902081654831275,
      "grad_norm": 2.009601969125096,
      "learning_rate": 6.374552583116766e-07,
      "log_odds_chosen": 0.378662109375,
      "log_odds_ratio": -0.6207031011581421,
      "logits/chosen": -0.779980480670929,
      "logits/rejected": -0.667675793170929,
      "logps/chosen": -0.6968749761581421,
      "logps/rejected": -0.9564453363418579,
      "loss": 0.8839,
      "nll_loss": 0.8177734613418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0697021484375,
      "rewards/margins": 0.02591114118695259,
      "rewards/rejected": -0.0955810546875,
      "step": 15750
    },
    {
      "epoch": 0.5905829008262914,
      "grad_norm": 1.9887057361834934,
      "learning_rate": 6.372529878771662e-07,
      "log_odds_chosen": 0.6090332269668579,
      "log_odds_ratio": -0.552441418170929,
      "logits/chosen": -0.732617199420929,
      "logits/rejected": -0.634960949420929,
      "logps/chosen": -0.6744140386581421,
      "logps/rejected": -1.06640625,
      "loss": 0.8809,
      "nll_loss": 0.8291015625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0675048828125,
      "rewards/margins": 0.03916778415441513,
      "rewards/rejected": -0.10665283352136612,
      "step": 15760
    },
    {
      "epoch": 0.5909576361694553,
      "grad_norm": 2.1709448123232353,
      "learning_rate": 6.370509098673676e-07,
      "log_odds_chosen": 0.5130859613418579,
      "log_odds_ratio": -0.5997070074081421,
      "logits/chosen": -0.7552734613418579,
      "logits/rejected": -0.6561523675918579,
      "logps/chosen": -0.6650390625,
      "logps/rejected": -0.9912109375,
      "loss": 0.8557,
      "nll_loss": 0.773242175579071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06647948920726776,
      "rewards/margins": 0.03262939304113388,
      "rewards/rejected": -0.09914550930261612,
      "step": 15770
    },
    {
      "epoch": 0.5913323715126192,
      "grad_norm": 1.8198008157916203,
      "learning_rate": 6.368490239773771e-07,
      "log_odds_chosen": 0.4957275390625,
      "log_odds_ratio": -0.595410168170929,
      "logits/chosen": -0.780468761920929,
      "logits/rejected": -0.6572265625,
      "logps/chosen": -0.703906238079071,
      "logps/rejected": -1.0320312976837158,
      "loss": 0.888,
      "nll_loss": 0.867382824420929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07039795070886612,
      "rewards/margins": 0.03282470628619194,
      "rewards/rejected": -0.10323486477136612,
      "step": 15780
    },
    {
      "epoch": 0.5917071068557831,
      "grad_norm": 2.4022763899747237,
      "learning_rate": 6.366473299029672e-07,
      "log_odds_chosen": 0.532177746295929,
      "log_odds_ratio": -0.5780273675918579,
      "logits/chosen": -0.7939453125,
      "logits/rejected": -0.7095702886581421,
      "logps/chosen": -0.699999988079071,
      "logps/rejected": -1.022851586341858,
      "loss": 0.8692,
      "nll_loss": 0.782421886920929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06999512016773224,
      "rewards/margins": 0.0323333740234375,
      "rewards/rejected": -0.10227050632238388,
      "step": 15790
    },
    {
      "epoch": 0.592081842198947,
      "grad_norm": 1.9892612275082127,
      "learning_rate": 6.36445827340584e-07,
      "log_odds_chosen": 0.5291503667831421,
      "log_odds_ratio": -0.56396484375,
      "logits/chosen": -0.7587890625,
      "logits/rejected": -0.6426757574081421,
      "logps/chosen": -0.7002929449081421,
      "logps/rejected": -1.0285155773162842,
      "loss": 0.8705,
      "nll_loss": 0.7935546636581421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06990966945886612,
      "rewards/margins": 0.03288726881146431,
      "rewards/rejected": -0.10283203423023224,
      "step": 15800
    },
    {
      "epoch": 0.5924565775421109,
      "grad_norm": 1.970143156520368,
      "learning_rate": 6.362445159873459e-07,
      "log_odds_chosen": 0.38582152128219604,
      "log_odds_ratio": -0.618847668170929,
      "logits/chosen": -0.7646484375,
      "logits/rejected": -0.7021484375,
      "logps/chosen": -0.6846679449081421,
      "logps/rejected": -0.902148425579071,
      "loss": 0.8844,
      "nll_loss": 0.8236328363418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06849364936351776,
      "rewards/margins": 0.021707916632294655,
      "rewards/rejected": -0.09023437649011612,
      "step": 15810
    },
    {
      "epoch": 0.5928313128852748,
      "grad_norm": 2.012563350838857,
      "learning_rate": 6.360433955410417e-07,
      "log_odds_chosen": 0.522753894329071,
      "log_odds_ratio": -0.5765625238418579,
      "logits/chosen": -0.7626953125,
      "logits/rejected": -0.6307617425918579,
      "logps/chosen": -0.6666015386581421,
      "logps/rejected": -1.001562476158142,
      "loss": 0.864,
      "nll_loss": 0.828906238079071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06669922173023224,
      "rewards/margins": 0.03348083421587944,
      "rewards/rejected": -0.10017089545726776,
      "step": 15820
    },
    {
      "epoch": 0.5932060482284387,
      "grad_norm": 1.9890833067655245,
      "learning_rate": 6.35842465700128e-07,
      "log_odds_chosen": 0.448486328125,
      "log_odds_ratio": -0.623339831829071,
      "logits/chosen": -0.827343761920929,
      "logits/rejected": -0.696093738079071,
      "logps/chosen": -0.7085937261581421,
      "logps/rejected": -1.005859375,
      "loss": 0.8893,
      "nll_loss": 0.8486328125,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07084961235523224,
      "rewards/margins": 0.02970123291015625,
      "rewards/rejected": -0.10065917670726776,
      "step": 15830
    },
    {
      "epoch": 0.5935807835716026,
      "grad_norm": 1.9890382190244777,
      "learning_rate": 6.356417261637281e-07,
      "log_odds_chosen": 0.549572765827179,
      "log_odds_ratio": -0.6004883050918579,
      "logits/chosen": -0.730273425579071,
      "logits/rejected": -0.6338866949081421,
      "logps/chosen": -0.676953136920929,
      "logps/rejected": -1.059179663658142,
      "loss": 0.8697,
      "nll_loss": 0.7724609375,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06766357272863388,
      "rewards/margins": 0.038196563720703125,
      "rewards/rejected": -0.10592041164636612,
      "step": 15840
    },
    {
      "epoch": 0.5939555189147665,
      "grad_norm": 2.0990618798052307,
      "learning_rate": 6.354411766316301e-07,
      "log_odds_chosen": 0.4456420838832855,
      "log_odds_ratio": -0.6000000238418579,
      "logits/chosen": -0.7632812261581421,
      "logits/rejected": -0.631640613079071,
      "logps/chosen": -0.643847644329071,
      "logps/rejected": -0.927734375,
      "loss": 0.848,
      "nll_loss": 0.7943359613418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06441650539636612,
      "rewards/margins": 0.02832641638815403,
      "rewards/rejected": -0.09272460639476776,
      "step": 15850
    },
    {
      "epoch": 0.5943302542579303,
      "grad_norm": 2.0887657294715725,
      "learning_rate": 6.35240816804284e-07,
      "log_odds_chosen": 0.40919190645217896,
      "log_odds_ratio": -0.6001952886581421,
      "logits/chosen": -0.779101550579071,
      "logits/rejected": -0.663867175579071,
      "logps/chosen": -0.675000011920929,
      "logps/rejected": -0.947460949420929,
      "loss": 0.8597,
      "nll_loss": 0.7662109136581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06745605170726776,
      "rewards/margins": 0.027249908074736595,
      "rewards/rejected": -0.09473876655101776,
      "step": 15860
    },
    {
      "epoch": 0.5947049896010942,
      "grad_norm": 2.038845086051526,
      "learning_rate": 6.350406463828013e-07,
      "log_odds_chosen": 0.45783692598342896,
      "log_odds_ratio": -0.5985351800918579,
      "logits/chosen": -0.7607421875,
      "logits/rejected": -0.6783202886581421,
      "logps/chosen": -0.7217773199081421,
      "logps/rejected": -0.9908202886581421,
      "loss": 0.8738,
      "nll_loss": 0.7870117425918579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07219238579273224,
      "rewards/margins": 0.026889801025390625,
      "rewards/rejected": -0.09904785454273224,
      "step": 15870
    },
    {
      "epoch": 0.5950797249442581,
      "grad_norm": 1.8802909825044418,
      "learning_rate": 6.348406650689516e-07,
      "log_odds_chosen": 0.44938963651657104,
      "log_odds_ratio": -0.613574206829071,
      "logits/chosen": -0.7808593511581421,
      "logits/rejected": -0.7119140625,
      "logps/chosen": -0.7265625,
      "logps/rejected": -1.0343749523162842,
      "loss": 0.8683,
      "nll_loss": 0.8345702886581421,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07266845554113388,
      "rewards/margins": 0.03071136400103569,
      "rewards/rejected": -0.10332031548023224,
      "step": 15880
    },
    {
      "epoch": 0.595454460287422,
      "grad_norm": 2.1720815519785854,
      "learning_rate": 6.346408725651623e-07,
      "log_odds_chosen": 0.5589233636856079,
      "log_odds_ratio": -0.56787109375,
      "logits/chosen": -0.7982422113418579,
      "logits/rejected": -0.705273449420929,
      "logps/chosen": -0.6625000238418579,
      "logps/rejected": -1.017968773841858,
      "loss": 0.8468,
      "nll_loss": 0.74609375,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.0662841796875,
      "rewards/margins": 0.03558044508099556,
      "rewards/rejected": -0.101806640625,
      "step": 15890
    },
    {
      "epoch": 0.5958291956305859,
      "grad_norm": 2.103170052311764,
      "learning_rate": 6.344412685745153e-07,
      "log_odds_chosen": 0.577709972858429,
      "log_odds_ratio": -0.577832043170929,
      "logits/chosen": -0.784375011920929,
      "logits/rejected": -0.6792968511581421,
      "logps/chosen": -0.692187488079071,
      "logps/rejected": -1.0724608898162842,
      "loss": 0.8884,
      "nll_loss": 0.800976574420929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06920166313648224,
      "rewards/margins": 0.03800811618566513,
      "rewards/rejected": -0.107177734375,
      "step": 15900
    },
    {
      "epoch": 0.5962039309737498,
      "grad_norm": 1.9460393440875827,
      "learning_rate": 6.342418528007461e-07,
      "log_odds_chosen": 0.49846190214157104,
      "log_odds_ratio": -0.6255859136581421,
      "logits/chosen": -0.6944335699081421,
      "logits/rejected": -0.6221679449081421,
      "logps/chosen": -0.720898449420929,
      "logps/rejected": -1.047460913658142,
      "loss": 0.8458,
      "nll_loss": 0.8060547113418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07203368842601776,
      "rewards/margins": 0.03270263597369194,
      "rewards/rejected": -0.10482177883386612,
      "step": 15910
    },
    {
      "epoch": 0.5965786663169137,
      "grad_norm": 2.2157533543767705,
      "learning_rate": 6.340426249482414e-07,
      "log_odds_chosen": 0.41790771484375,
      "log_odds_ratio": -0.659375011920929,
      "logits/chosen": -0.7373046875,
      "logits/rejected": -0.6451171636581421,
      "logps/chosen": -0.69677734375,
      "logps/rejected": -0.975781261920929,
      "loss": 0.858,
      "nll_loss": 0.8076171875,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06961669772863388,
      "rewards/margins": 0.02788391150534153,
      "rewards/rejected": -0.09752197563648224,
      "step": 15920
    },
    {
      "epoch": 0.5969534016600776,
      "grad_norm": 2.1578933309819712,
      "learning_rate": 6.338435847220378e-07,
      "log_odds_chosen": 0.4134887754917145,
      "log_odds_ratio": -0.6163085699081421,
      "logits/chosen": -0.740039050579071,
      "logits/rejected": -0.656933605670929,
      "logps/chosen": -0.71337890625,
      "logps/rejected": -0.964648425579071,
      "loss": 0.8844,
      "nll_loss": 0.816210925579071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07124023139476776,
      "rewards/margins": 0.025209808722138405,
      "rewards/rejected": -0.09649658203125,
      "step": 15930
    },
    {
      "epoch": 0.5973281370032415,
      "grad_norm": 2.2638128822330517,
      "learning_rate": 6.336447318278194e-07,
      "log_odds_chosen": 0.43937987089157104,
      "log_odds_ratio": -0.6197265386581421,
      "logits/chosen": -0.8003906011581421,
      "logits/rejected": -0.702343761920929,
      "logps/chosen": -0.660351574420929,
      "logps/rejected": -0.9287109375,
      "loss": 0.8775,
      "nll_loss": 0.8291015625,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06607665866613388,
      "rewards/margins": 0.02686614915728569,
      "rewards/rejected": -0.09294433891773224,
      "step": 15940
    },
    {
      "epoch": 0.5977028723464054,
      "grad_norm": 1.9940392096375035,
      "learning_rate": 6.334460659719166e-07,
      "log_odds_chosen": 0.42542725801467896,
      "log_odds_ratio": -0.623828113079071,
      "logits/chosen": -0.788281261920929,
      "logits/rejected": -0.6773437261581421,
      "logps/chosen": -0.6583007574081421,
      "logps/rejected": -0.9398437738418579,
      "loss": 0.8733,
      "nll_loss": 0.824999988079071,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06577148288488388,
      "rewards/margins": 0.028156280517578125,
      "rewards/rejected": -0.09398193657398224,
      "step": 15950
    },
    {
      "epoch": 0.5980776076895692,
      "grad_norm": 1.78049953871022,
      "learning_rate": 6.332475868613035e-07,
      "log_odds_chosen": 0.3668579161167145,
      "log_odds_ratio": -0.644238293170929,
      "logits/chosen": -0.736523449420929,
      "logits/rejected": -0.6480468511581421,
      "logps/chosen": -0.6884765625,
      "logps/rejected": -0.920117199420929,
      "loss": 0.8951,
      "nll_loss": 0.829882800579071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06885986030101776,
      "rewards/margins": 0.02315063402056694,
      "rewards/rejected": -0.092041015625,
      "step": 15960
    },
    {
      "epoch": 0.5984523430327331,
      "grad_norm": 1.8625689658396358,
      "learning_rate": 6.330492942035967e-07,
      "log_odds_chosen": 0.45454102754592896,
      "log_odds_ratio": -0.611132800579071,
      "logits/chosen": -0.806445300579071,
      "logits/rejected": -0.6962890625,
      "logps/chosen": -0.679882824420929,
      "logps/rejected": -0.979687511920929,
      "loss": 0.87,
      "nll_loss": 0.806835949420929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06802978366613388,
      "rewards/margins": 0.029955292120575905,
      "rewards/rejected": -0.09801025688648224,
      "step": 15970
    },
    {
      "epoch": 0.598827078375897,
      "grad_norm": 1.8413482146629077,
      "learning_rate": 6.32851187707053e-07,
      "log_odds_chosen": 0.4239746034145355,
      "log_odds_ratio": -0.61376953125,
      "logits/chosen": -0.759570300579071,
      "logits/rejected": -0.6591796875,
      "logps/chosen": -0.7310546636581421,
      "logps/rejected": -1.001953125,
      "loss": 0.8909,
      "nll_loss": 0.846484363079071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07308349758386612,
      "rewards/margins": 0.027196502313017845,
      "rewards/rejected": -0.10023193061351776,
      "step": 15980
    },
    {
      "epoch": 0.5992018137190609,
      "grad_norm": 1.9422979193020902,
      "learning_rate": 6.326532670805687e-07,
      "log_odds_chosen": 0.553149402141571,
      "log_odds_ratio": -0.5660156011581421,
      "logits/chosen": -0.685351550579071,
      "logits/rejected": -0.6050781011581421,
      "logps/chosen": -0.6954101324081421,
      "logps/rejected": -1.033593773841858,
      "loss": 0.8757,
      "nll_loss": 0.866015613079071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06951904296875,
      "rewards/margins": 0.03394012525677681,
      "rewards/rejected": -0.10349120944738388,
      "step": 15990
    },
    {
      "epoch": 0.5995765490622248,
      "grad_norm": 1.9007543164055776,
      "learning_rate": 6.324555320336758e-07,
      "log_odds_chosen": 0.386077880859375,
      "log_odds_ratio": -0.613964855670929,
      "logits/chosen": -0.7544921636581421,
      "logits/rejected": -0.669921875,
      "logps/chosen": -0.69140625,
      "logps/rejected": -0.919921875,
      "loss": 0.8521,
      "nll_loss": 0.789257824420929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06915283203125,
      "rewards/margins": 0.022821808233857155,
      "rewards/rejected": -0.09200439602136612,
      "step": 16000
    },
    {
      "epoch": 0.5999512844053887,
      "grad_norm": 2.1985492351146467,
      "learning_rate": 6.322579822765425e-07,
      "log_odds_chosen": 0.668749988079071,
      "log_odds_ratio": -0.5455077886581421,
      "logits/chosen": -0.7652343511581421,
      "logits/rejected": -0.639453113079071,
      "logps/chosen": -0.667285144329071,
      "logps/rejected": -1.099609375,
      "loss": 0.8584,
      "nll_loss": 0.810742199420929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06673584133386612,
      "rewards/margins": 0.04320068284869194,
      "rewards/rejected": -0.10993652045726776,
      "step": 16010
    },
    {
      "epoch": 0.6003260197485526,
      "grad_norm": 2.0818735525569974,
      "learning_rate": 6.320606175199696e-07,
      "log_odds_chosen": 0.3217407166957855,
      "log_odds_ratio": -0.6551758050918579,
      "logits/chosen": -0.721484363079071,
      "logits/rejected": -0.667285144329071,
      "logps/chosen": -0.708203136920929,
      "logps/rejected": -0.915234386920929,
      "loss": 0.8793,
      "nll_loss": 0.81640625,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07084961235523224,
      "rewards/margins": 0.020618248730897903,
      "rewards/rejected": -0.09149169921875,
      "step": 16020
    },
    {
      "epoch": 0.6007007550917165,
      "grad_norm": 2.02596266049436,
      "learning_rate": 6.318634374753898e-07,
      "log_odds_chosen": 0.42841798067092896,
      "log_odds_ratio": -0.5899413824081421,
      "logits/chosen": -0.771484375,
      "logits/rejected": -0.65185546875,
      "logps/chosen": -0.682421863079071,
      "logps/rejected": -0.9632812738418579,
      "loss": 0.8735,
      "nll_loss": 0.8179687261581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.0682373046875,
      "rewards/margins": 0.028146743774414062,
      "rewards/rejected": -0.09641113132238388,
      "step": 16030
    },
    {
      "epoch": 0.6010754904348804,
      "grad_norm": 2.5099057608979374,
      "learning_rate": 6.316664418548654e-07,
      "log_odds_chosen": 0.510632336139679,
      "log_odds_ratio": -0.6058593988418579,
      "logits/chosen": -0.766796886920929,
      "logits/rejected": -0.668652355670929,
      "logps/chosen": -0.650390625,
      "logps/rejected": -0.99609375,
      "loss": 0.872,
      "nll_loss": 0.8154296875,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06501464545726776,
      "rewards/margins": 0.03452606126666069,
      "rewards/rejected": -0.09958495944738388,
      "step": 16040
    },
    {
      "epoch": 0.6014502257780443,
      "grad_norm": 2.2203459563134302,
      "learning_rate": 6.314696303710867e-07,
      "log_odds_chosen": 0.5445922613143921,
      "log_odds_ratio": -0.5777343511581421,
      "logits/chosen": -0.727734386920929,
      "logits/rejected": -0.6285156011581421,
      "logps/chosen": -0.6353515386581421,
      "logps/rejected": -0.9820312261581421,
      "loss": 0.8656,
      "nll_loss": 0.792285144329071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06348876655101776,
      "rewards/margins": 0.03467407077550888,
      "rewards/rejected": -0.09818115085363388,
      "step": 16050
    },
    {
      "epoch": 0.6018249611212082,
      "grad_norm": 2.005195548612276,
      "learning_rate": 6.312730027373703e-07,
      "log_odds_chosen": 0.461883544921875,
      "log_odds_ratio": -0.595410168170929,
      "logits/chosen": -0.7855468988418579,
      "logits/rejected": -0.7113281488418579,
      "logps/chosen": -0.6513671875,
      "logps/rejected": -0.9312499761581421,
      "loss": 0.8609,
      "nll_loss": 0.8021484613418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06517334282398224,
      "rewards/margins": 0.0279998779296875,
      "rewards/rejected": -0.09318847954273224,
      "step": 16060
    },
    {
      "epoch": 0.602199696464372,
      "grad_norm": 2.0965017111302333,
      "learning_rate": 6.310765586676574e-07,
      "log_odds_chosen": 0.3877197206020355,
      "log_odds_ratio": -0.598437488079071,
      "logits/chosen": -0.7837890386581421,
      "logits/rejected": -0.6615234613418579,
      "logps/chosen": -0.676562488079071,
      "logps/rejected": -0.886914074420929,
      "loss": 0.8872,
      "nll_loss": 0.8218749761581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.067626953125,
      "rewards/margins": 0.02109680138528347,
      "rewards/rejected": -0.08872070163488388,
      "step": 16070
    },
    {
      "epoch": 0.6025744318075359,
      "grad_norm": 2.0404116575355884,
      "learning_rate": 6.308802978765117e-07,
      "log_odds_chosen": 0.30755615234375,
      "log_odds_ratio": -0.640625,
      "logits/chosen": -0.747265636920929,
      "logits/rejected": -0.6636718511581421,
      "logps/chosen": -0.7119140625,
      "logps/rejected": -0.90771484375,
      "loss": 0.8781,
      "nll_loss": 0.8824218511581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07119140774011612,
      "rewards/margins": 0.01957244798541069,
      "rewards/rejected": -0.09075927734375,
      "step": 16080
    },
    {
      "epoch": 0.6029491671506998,
      "grad_norm": 2.241665810400965,
      "learning_rate": 6.306842200791181e-07,
      "log_odds_chosen": 0.4262939393520355,
      "log_odds_ratio": -0.6322265863418579,
      "logits/chosen": -0.737109363079071,
      "logits/rejected": -0.679980456829071,
      "logps/chosen": -0.6734374761581421,
      "logps/rejected": -0.9423828125,
      "loss": 0.8676,
      "nll_loss": 0.8091796636581421,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.06740722805261612,
      "rewards/margins": 0.026824187487363815,
      "rewards/rejected": -0.09421386569738388,
      "step": 16090
    },
    {
      "epoch": 0.6033239024938637,
      "grad_norm": 2.1851995299564573,
      "learning_rate": 6.304883249912805e-07,
      "log_odds_chosen": 0.4863647520542145,
      "log_odds_ratio": -0.606249988079071,
      "logits/chosen": -0.738085925579071,
      "logits/rejected": -0.667773425579071,
      "logps/chosen": -0.65869140625,
      "logps/rejected": -0.9365234375,
      "loss": 0.8797,
      "nll_loss": 0.803027331829071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06586913764476776,
      "rewards/margins": 0.02782898023724556,
      "rewards/rejected": -0.09370116889476776,
      "step": 16100
    },
    {
      "epoch": 0.6036986378370276,
      "grad_norm": 2.0944555048238667,
      "learning_rate": 6.302926123294205e-07,
      "log_odds_chosen": 0.5923217535018921,
      "log_odds_ratio": -0.5498046875,
      "logits/chosen": -0.7230468988418579,
      "logits/rejected": -0.646777331829071,
      "logps/chosen": -0.671679675579071,
      "logps/rejected": -1.0388672351837158,
      "loss": 0.8676,
      "nll_loss": 0.8203125,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06724853813648224,
      "rewards/margins": 0.036651611328125,
      "rewards/rejected": -0.10393066704273224,
      "step": 16110
    },
    {
      "epoch": 0.6040733731801915,
      "grad_norm": 2.460022791991083,
      "learning_rate": 6.300970818105758e-07,
      "log_odds_chosen": 0.7201172113418579,
      "log_odds_ratio": -0.53662109375,
      "logits/chosen": -0.748339831829071,
      "logits/rejected": -0.6509765386581421,
      "logps/chosen": -0.6773437261581421,
      "logps/rejected": -1.1486327648162842,
      "loss": 0.8558,
      "nll_loss": 0.8199218511581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06773681938648224,
      "rewards/margins": 0.04720459133386612,
      "rewards/rejected": -0.11495361477136612,
      "step": 16120
    },
    {
      "epoch": 0.6044481085233554,
      "grad_norm": 2.7384331328521636,
      "learning_rate": 6.299017331523976e-07,
      "log_odds_chosen": 0.597705066204071,
      "log_odds_ratio": -0.540722668170929,
      "logits/chosen": -0.803906261920929,
      "logits/rejected": -0.699902355670929,
      "logps/chosen": -0.6551758050918579,
      "logps/rejected": -0.9990234375,
      "loss": 0.8767,
      "nll_loss": 0.7718750238418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.0655517578125,
      "rewards/margins": 0.03436889499425888,
      "rewards/rejected": -0.09984131157398224,
      "step": 16130
    },
    {
      "epoch": 0.6048228438665193,
      "grad_norm": 1.9841293692720245,
      "learning_rate": 6.297065660731497e-07,
      "log_odds_chosen": 0.578625500202179,
      "log_odds_ratio": -0.5689452886581421,
      "logits/chosen": -0.7583984136581421,
      "logits/rejected": -0.6683593988418579,
      "logps/chosen": -0.654980480670929,
      "logps/rejected": -1.0125000476837158,
      "loss": 0.8583,
      "nll_loss": 0.826171875,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.0654296875,
      "rewards/margins": 0.03579101711511612,
      "rewards/rejected": -0.10124512016773224,
      "step": 16140
    },
    {
      "epoch": 0.6051975792096832,
      "grad_norm": 2.4235258123837187,
      "learning_rate": 6.29511580291707e-07,
      "log_odds_chosen": 0.28441160917282104,
      "log_odds_ratio": -0.6509765386581421,
      "logits/chosen": -0.7535156011581421,
      "logits/rejected": -0.671582043170929,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -0.8837890625,
      "loss": 0.8708,
      "nll_loss": 0.7982422113418579,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07070312649011612,
      "rewards/margins": 0.017638396471738815,
      "rewards/rejected": -0.08836670219898224,
      "step": 16150
    },
    {
      "epoch": 0.6055723145528471,
      "grad_norm": 2.1298225062813154,
      "learning_rate": 6.293167755275526e-07,
      "log_odds_chosen": 0.4762206971645355,
      "log_odds_ratio": -0.60205078125,
      "logits/chosen": -0.7447265386581421,
      "logits/rejected": -0.6512695550918579,
      "logps/chosen": -0.675976574420929,
      "logps/rejected": -0.967578113079071,
      "loss": 0.8655,
      "nll_loss": 0.7974609136581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06757812201976776,
      "rewards/margins": 0.02913970872759819,
      "rewards/rejected": -0.09675292670726776,
      "step": 16160
    },
    {
      "epoch": 0.6059470498960109,
      "grad_norm": 2.2506994649386245,
      "learning_rate": 6.291221515007776e-07,
      "log_odds_chosen": 0.4971679747104645,
      "log_odds_ratio": -0.6371093988418579,
      "logits/chosen": -0.7367187738418579,
      "logits/rejected": -0.642871081829071,
      "logps/chosen": -0.712109386920929,
      "logps/rejected": -1.0441405773162842,
      "loss": 0.881,
      "nll_loss": 0.8349609375,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07125244289636612,
      "rewards/margins": 0.03311309963464737,
      "rewards/rejected": -0.1043701171875,
      "step": 16170
    },
    {
      "epoch": 0.6063217852391748,
      "grad_norm": 1.982098984606279,
      "learning_rate": 6.28927707932078e-07,
      "log_odds_chosen": 0.4280639588832855,
      "log_odds_ratio": -0.626269519329071,
      "logits/chosen": -0.696582019329071,
      "logits/rejected": -0.6468750238418579,
      "logps/chosen": -0.701953113079071,
      "logps/rejected": -0.9644531011581421,
      "loss": 0.8653,
      "nll_loss": 0.891796886920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07016601413488388,
      "rewards/margins": 0.026262665167450905,
      "rewards/rejected": -0.09654541313648224,
      "step": 16180
    },
    {
      "epoch": 0.6066965205823387,
      "grad_norm": 2.1741250375862675,
      "learning_rate": 6.287334445427542e-07,
      "log_odds_chosen": 0.44331055879592896,
      "log_odds_ratio": -0.6275390386581421,
      "logits/chosen": -0.8101562261581421,
      "logits/rejected": -0.6688476800918579,
      "logps/chosen": -0.710156261920929,
      "logps/rejected": -0.9906250238418579,
      "loss": 0.855,
      "nll_loss": 0.807812511920929,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07110595703125,
      "rewards/margins": 0.02806854248046875,
      "rewards/rejected": -0.09898681938648224,
      "step": 16190
    },
    {
      "epoch": 0.6070712559255026,
      "grad_norm": 1.9833250618033595,
      "learning_rate": 6.285393610547089e-07,
      "log_odds_chosen": 0.517224133014679,
      "log_odds_ratio": -0.59765625,
      "logits/chosen": -0.783984363079071,
      "logits/rejected": -0.6805664300918579,
      "logps/chosen": -0.724804699420929,
      "logps/rejected": -1.0673828125,
      "loss": 0.8693,
      "nll_loss": 0.82568359375,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07243652641773224,
      "rewards/margins": 0.034322358667850494,
      "rewards/rejected": -0.1068115234375,
      "step": 16200
    },
    {
      "epoch": 0.6074459912686665,
      "grad_norm": 2.098169345545023,
      "learning_rate": 6.28345457190445e-07,
      "log_odds_chosen": 0.58599853515625,
      "log_odds_ratio": -0.579785168170929,
      "logits/chosen": -0.7523437738418579,
      "logits/rejected": -0.657421886920929,
      "logps/chosen": -0.6898437738418579,
      "logps/rejected": -1.048242211341858,
      "loss": 0.8784,
      "nll_loss": 0.7811523675918579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06900634616613388,
      "rewards/margins": 0.03589172288775444,
      "rewards/rejected": -0.10490722954273224,
      "step": 16210
    },
    {
      "epoch": 0.6078207266118304,
      "grad_norm": 2.1259347526899055,
      "learning_rate": 6.281517326730642e-07,
      "log_odds_chosen": 0.39958494901657104,
      "log_odds_ratio": -0.624218761920929,
      "logits/chosen": -0.685546875,
      "logits/rejected": -0.617968738079071,
      "logps/chosen": -0.6498047113418579,
      "logps/rejected": -0.877734363079071,
      "loss": 0.8976,
      "nll_loss": 0.8082031011581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06499023735523224,
      "rewards/margins": 0.022782135754823685,
      "rewards/rejected": -0.08780517429113388,
      "step": 16220
    },
    {
      "epoch": 0.6081954619549943,
      "grad_norm": 2.1365743336344813,
      "learning_rate": 6.279581872262657e-07,
      "log_odds_chosen": 0.6263672113418579,
      "log_odds_ratio": -0.5785156488418579,
      "logits/chosen": -0.7466796636581421,
      "logits/rejected": -0.654589831829071,
      "logps/chosen": -0.7323242425918579,
      "logps/rejected": -1.162695288658142,
      "loss": 0.8698,
      "nll_loss": 0.813281238079071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07321777194738388,
      "rewards/margins": 0.04307708889245987,
      "rewards/rejected": -0.11628417670726776,
      "step": 16230
    },
    {
      "epoch": 0.6085701972981582,
      "grad_norm": 2.2452995317914555,
      "learning_rate": 6.277648205743445e-07,
      "log_odds_chosen": 0.536511242389679,
      "log_odds_ratio": -0.57763671875,
      "logits/chosen": -0.759570300579071,
      "logits/rejected": -0.684765636920929,
      "logps/chosen": -0.661914050579071,
      "logps/rejected": -0.9964843988418579,
      "loss": 0.8616,
      "nll_loss": 0.7962890863418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06619872897863388,
      "rewards/margins": 0.03345794603228569,
      "rewards/rejected": -0.09963379055261612,
      "step": 16240
    },
    {
      "epoch": 0.6089449326413221,
      "grad_norm": 2.192061774717188,
      "learning_rate": 6.275716324421889e-07,
      "log_odds_chosen": 0.557666003704071,
      "log_odds_ratio": -0.5584961175918579,
      "logits/chosen": -0.7933593988418579,
      "logits/rejected": -0.6781250238418579,
      "logps/chosen": -0.674609363079071,
      "logps/rejected": -1.011132836341858,
      "loss": 0.8674,
      "nll_loss": 0.828906238079071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06749267876148224,
      "rewards/margins": 0.03367462009191513,
      "rewards/rejected": -0.10107421875,
      "step": 16250
    },
    {
      "epoch": 0.609319667984486,
      "grad_norm": 2.027302934918507,
      "learning_rate": 6.273786225552799e-07,
      "log_odds_chosen": 0.46198731660842896,
      "log_odds_ratio": -0.62109375,
      "logits/chosen": -0.746289074420929,
      "logits/rejected": -0.6214843988418579,
      "logps/chosen": -0.69140625,
      "logps/rejected": -1.0126953125,
      "loss": 0.8662,
      "nll_loss": 0.7767578363418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06914062798023224,
      "rewards/margins": 0.03214721754193306,
      "rewards/rejected": -0.10134277492761612,
      "step": 16260
    },
    {
      "epoch": 0.6096944033276498,
      "grad_norm": 2.1790583927935265,
      "learning_rate": 6.271857906396891e-07,
      "log_odds_chosen": 0.4774169921875,
      "log_odds_ratio": -0.60205078125,
      "logits/chosen": -0.7630859613418579,
      "logits/rejected": -0.6572265625,
      "logps/chosen": -0.6341797113418579,
      "logps/rejected": -0.9423828125,
      "loss": 0.8752,
      "nll_loss": 0.7749999761581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06343994289636612,
      "rewards/margins": 0.03073425218462944,
      "rewards/rejected": -0.09417724609375,
      "step": 16270
    },
    {
      "epoch": 0.6100691386708137,
      "grad_norm": 2.232816112477534,
      "learning_rate": 6.269931364220768e-07,
      "log_odds_chosen": 0.6263427734375,
      "log_odds_ratio": -0.5311523675918579,
      "logits/chosen": -0.7515624761581421,
      "logits/rejected": -0.6683593988418579,
      "logps/chosen": -0.6512695550918579,
      "logps/rejected": -1.05078125,
      "loss": 0.8565,
      "nll_loss": 0.7914062738418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.065185546875,
      "rewards/margins": 0.03993682935833931,
      "rewards/rejected": -0.10506591945886612,
      "step": 16280
    },
    {
      "epoch": 0.6104438740139776,
      "grad_norm": 1.9518121924713172,
      "learning_rate": 6.268006596296912e-07,
      "log_odds_chosen": 0.4449462890625,
      "log_odds_ratio": -0.589550793170929,
      "logits/chosen": -0.794921875,
      "logits/rejected": -0.7017577886581421,
      "logps/chosen": -0.669140636920929,
      "logps/rejected": -0.9111328125,
      "loss": 0.8707,
      "nll_loss": 0.7689453363418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06690673530101776,
      "rewards/margins": 0.02413177490234375,
      "rewards/rejected": -0.09095458686351776,
      "step": 16290
    },
    {
      "epoch": 0.6108186093571415,
      "grad_norm": 2.4392299821906516,
      "learning_rate": 6.266083599903659e-07,
      "log_odds_chosen": 0.39008790254592896,
      "log_odds_ratio": -0.6185547113418579,
      "logits/chosen": -0.7515624761581421,
      "logits/rejected": -0.6571289300918579,
      "logps/chosen": -0.68994140625,
      "logps/rejected": -0.939257800579071,
      "loss": 0.8677,
      "nll_loss": 0.842578113079071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06904296576976776,
      "rewards/margins": 0.024938201531767845,
      "rewards/rejected": -0.09394530951976776,
      "step": 16300
    },
    {
      "epoch": 0.6111933447003054,
      "grad_norm": 2.2192632527448373,
      "learning_rate": 6.264162372325185e-07,
      "log_odds_chosen": 0.4077392518520355,
      "log_odds_ratio": -0.626953125,
      "logits/chosen": -0.7621093988418579,
      "logits/rejected": -0.6275390386581421,
      "logps/chosen": -0.736328125,
      "logps/rejected": -0.9935547113418579,
      "loss": 0.8724,
      "nll_loss": 0.866406261920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07365722954273224,
      "rewards/margins": 0.02571563795208931,
      "rewards/rejected": -0.09941406548023224,
      "step": 16310
    },
    {
      "epoch": 0.6115680800434693,
      "grad_norm": 1.9236277987968446,
      "learning_rate": 6.262242910851495e-07,
      "log_odds_chosen": 0.46259766817092896,
      "log_odds_ratio": -0.632617175579071,
      "logits/chosen": -0.783984363079071,
      "logits/rejected": -0.6826171875,
      "logps/chosen": -0.6722656488418579,
      "logps/rejected": -0.9681640863418579,
      "loss": 0.8543,
      "nll_loss": 0.8017578125,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06722412258386612,
      "rewards/margins": 0.02965698204934597,
      "rewards/rejected": -0.09677734225988388,
      "step": 16320
    },
    {
      "epoch": 0.6119428153866332,
      "grad_norm": 2.8724867475035647,
      "learning_rate": 6.2603252127784e-07,
      "log_odds_chosen": 0.3879638612270355,
      "log_odds_ratio": -0.61328125,
      "logits/chosen": -0.7470703125,
      "logits/rejected": -0.658496081829071,
      "logps/chosen": -0.72265625,
      "logps/rejected": -0.964648425579071,
      "loss": 0.8563,
      "nll_loss": 0.837695300579071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07225342094898224,
      "rewards/margins": 0.02415771409869194,
      "rewards/rejected": -0.09644775092601776,
      "step": 16330
    },
    {
      "epoch": 0.6123175507297971,
      "grad_norm": 2.1729128000654545,
      "learning_rate": 6.258409275407508e-07,
      "log_odds_chosen": 0.4244751036167145,
      "log_odds_ratio": -0.630078136920929,
      "logits/chosen": -0.7777343988418579,
      "logits/rejected": -0.6617187261581421,
      "logps/chosen": -0.726367175579071,
      "logps/rejected": -0.990429699420929,
      "loss": 0.8585,
      "nll_loss": 0.830273449420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07268066704273224,
      "rewards/margins": 0.02640685997903347,
      "rewards/rejected": -0.09914550930261612,
      "step": 16340
    },
    {
      "epoch": 0.612692286072961,
      "grad_norm": 1.9635181704315416,
      "learning_rate": 6.256495096046201e-07,
      "log_odds_chosen": 0.4166015684604645,
      "log_odds_ratio": -0.619921863079071,
      "logits/chosen": -0.7099609375,
      "logits/rejected": -0.6546875238418579,
      "logps/chosen": -0.710253894329071,
      "logps/rejected": -0.966015636920929,
      "loss": 0.8663,
      "nll_loss": 0.867382824420929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07100830227136612,
      "rewards/margins": 0.02555999718606472,
      "rewards/rejected": -0.0965576171875,
      "step": 16350
    },
    {
      "epoch": 0.6130670214161249,
      "grad_norm": 2.216584891227614,
      "learning_rate": 6.254582672007622e-07,
      "log_odds_chosen": 0.586071789264679,
      "log_odds_ratio": -0.553027331829071,
      "logits/chosen": -0.819531261920929,
      "logits/rejected": -0.691113293170929,
      "logps/chosen": -0.662304699420929,
      "logps/rejected": -1.0226562023162842,
      "loss": 0.8666,
      "nll_loss": 0.8158203363418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06624756008386612,
      "rewards/margins": 0.03585968166589737,
      "rewards/rejected": -0.10214843600988388,
      "step": 16360
    },
    {
      "epoch": 0.6134417567592888,
      "grad_norm": 2.2058962329017975,
      "learning_rate": 6.25267200061066e-07,
      "log_odds_chosen": 0.710803210735321,
      "log_odds_ratio": -0.5009765625,
      "logits/chosen": -0.751171886920929,
      "logits/rejected": -0.6380859613418579,
      "logps/chosen": -0.6259765625,
      "logps/rejected": -1.064453125,
      "loss": 0.8818,
      "nll_loss": 0.810351550579071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06256103515625,
      "rewards/margins": 0.04389190673828125,
      "rewards/rejected": -0.1064453125,
      "step": 16370
    },
    {
      "epoch": 0.6138164921024526,
      "grad_norm": 1.9804358982583607,
      "learning_rate": 6.250763079179939e-07,
      "log_odds_chosen": 0.5156005620956421,
      "log_odds_ratio": -0.59814453125,
      "logits/chosen": -0.756640613079071,
      "logits/rejected": -0.683789074420929,
      "logps/chosen": -0.734570324420929,
      "logps/rejected": -1.075585961341858,
      "loss": 0.8682,
      "nll_loss": 0.844531238079071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.073486328125,
      "rewards/margins": 0.03406066820025444,
      "rewards/rejected": -0.10751952975988388,
      "step": 16380
    },
    {
      "epoch": 0.6141912274456165,
      "grad_norm": 2.402778190633782,
      "learning_rate": 6.248855905045789e-07,
      "log_odds_chosen": 0.48271483182907104,
      "log_odds_ratio": -0.596972644329071,
      "logits/chosen": -0.6900390386581421,
      "logits/rejected": -0.590624988079071,
      "logps/chosen": -0.653613269329071,
      "logps/rejected": -0.9693359136581421,
      "loss": 0.8755,
      "nll_loss": 0.8724609613418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06535644829273224,
      "rewards/margins": 0.03150177001953125,
      "rewards/rejected": -0.09682617336511612,
      "step": 16390
    },
    {
      "epoch": 0.6145659627887804,
      "grad_norm": 2.1782180370688056,
      "learning_rate": 6.246950475544243e-07,
      "log_odds_chosen": 0.49683839082717896,
      "log_odds_ratio": -0.5814453363418579,
      "logits/chosen": -0.769335925579071,
      "logits/rejected": -0.627734363079071,
      "logps/chosen": -0.644238293170929,
      "logps/rejected": -0.960742175579071,
      "loss": 0.865,
      "nll_loss": 0.800976574420929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06439208984375,
      "rewards/margins": 0.03168487548828125,
      "rewards/rejected": -0.09619140625,
      "step": 16400
    },
    {
      "epoch": 0.6149406981319443,
      "grad_norm": 1.7901803900585005,
      "learning_rate": 6.245046788017015e-07,
      "log_odds_chosen": 0.46112060546875,
      "log_odds_ratio": -0.594921886920929,
      "logits/chosen": -0.7476562261581421,
      "logits/rejected": -0.667187511920929,
      "logps/chosen": -0.727734386920929,
      "logps/rejected": -1.003515601158142,
      "loss": 0.8734,
      "nll_loss": 0.832812488079071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07274170219898224,
      "rewards/margins": 0.027599334716796875,
      "rewards/rejected": -0.100341796875,
      "step": 16410
    },
    {
      "epoch": 0.6153154334751082,
      "grad_norm": 1.9879979177024838,
      "learning_rate": 6.243144839811487e-07,
      "log_odds_chosen": 0.512988269329071,
      "log_odds_ratio": -0.5904296636581421,
      "logits/chosen": -0.689453125,
      "logits/rejected": -0.61572265625,
      "logps/chosen": -0.655957043170929,
      "logps/rejected": -0.9761718511581421,
      "loss": 0.8821,
      "nll_loss": 0.8052734136581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06557617336511612,
      "rewards/margins": 0.03203735500574112,
      "rewards/rejected": -0.0975341796875,
      "step": 16420
    },
    {
      "epoch": 0.6156901688182721,
      "grad_norm": 1.9035510863639826,
      "learning_rate": 6.241244628280692e-07,
      "log_odds_chosen": 0.4826293885707855,
      "log_odds_ratio": -0.60888671875,
      "logits/chosen": -0.7254883050918579,
      "logits/rejected": -0.61572265625,
      "logps/chosen": -0.709179699420929,
      "logps/rejected": -1.0310547351837158,
      "loss": 0.8765,
      "nll_loss": 0.820507824420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0709228515625,
      "rewards/margins": 0.03217010572552681,
      "rewards/rejected": -0.10307617485523224,
      "step": 16430
    },
    {
      "epoch": 0.616064904161436,
      "grad_norm": 2.005534648478783,
      "learning_rate": 6.239346150783301e-07,
      "log_odds_chosen": 0.38719481229782104,
      "log_odds_ratio": -0.6407226324081421,
      "logits/chosen": -0.727734386920929,
      "logits/rejected": -0.6332031488418579,
      "logps/chosen": -0.7232421636581421,
      "logps/rejected": -0.986328125,
      "loss": 0.8632,
      "nll_loss": 0.7998046875,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07237549126148224,
      "rewards/margins": 0.02635497972369194,
      "rewards/rejected": -0.09874267876148224,
      "step": 16440
    },
    {
      "epoch": 0.6164396395045999,
      "grad_norm": 1.9572947527924878,
      "learning_rate": 6.237449404683603e-07,
      "log_odds_chosen": 0.37537842988967896,
      "log_odds_ratio": -0.65283203125,
      "logits/chosen": -0.794921875,
      "logits/rejected": -0.6761718988418579,
      "logps/chosen": -0.7021484375,
      "logps/rejected": -0.962109386920929,
      "loss": 0.8715,
      "nll_loss": 0.8232421875,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07023926079273224,
      "rewards/margins": 0.02595672570168972,
      "rewards/rejected": -0.09619140625,
      "step": 16450
    },
    {
      "epoch": 0.6168143748477638,
      "grad_norm": 1.9971337293877949,
      "learning_rate": 6.235554387351494e-07,
      "log_odds_chosen": 0.4483886659145355,
      "log_odds_ratio": -0.6146484613418579,
      "logits/chosen": -0.773242175579071,
      "logits/rejected": -0.6695312261581421,
      "logps/chosen": -0.6519531011581421,
      "logps/rejected": -0.941210925579071,
      "loss": 0.8719,
      "nll_loss": 0.740039050579071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06514892727136612,
      "rewards/margins": 0.02894439734518528,
      "rewards/rejected": -0.0941162109375,
      "step": 16460
    },
    {
      "epoch": 0.6171891101909277,
      "grad_norm": 2.10663077067857,
      "learning_rate": 6.233661096162459e-07,
      "log_odds_chosen": 0.38872069120407104,
      "log_odds_ratio": -0.6483398675918579,
      "logits/chosen": -0.737109363079071,
      "logits/rejected": -0.6747070550918579,
      "logps/chosen": -0.7632812261581421,
      "logps/rejected": -1.031640648841858,
      "loss": 0.8636,
      "nll_loss": 0.8023437261581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0762939453125,
      "rewards/margins": 0.02687530592083931,
      "rewards/rejected": -0.10312499850988388,
      "step": 16470
    },
    {
      "epoch": 0.6175638455340915,
      "grad_norm": 1.9777498329159526,
      "learning_rate": 6.231769528497558e-07,
      "log_odds_chosen": 0.594647228717804,
      "log_odds_ratio": -0.575878918170929,
      "logits/chosen": -0.7607421875,
      "logits/rejected": -0.6644531488418579,
      "logps/chosen": -0.6851562261581421,
      "logps/rejected": -1.053320288658142,
      "loss": 0.8725,
      "nll_loss": 0.8080078363418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06853027641773224,
      "rewards/margins": 0.03687172010540962,
      "rewards/rejected": -0.10532226413488388,
      "step": 16480
    },
    {
      "epoch": 0.6179385808772554,
      "grad_norm": 2.4768079143638397,
      "learning_rate": 6.229879681743411e-07,
      "log_odds_chosen": 0.45732420682907104,
      "log_odds_ratio": -0.6039062738418579,
      "logits/chosen": -0.6910156011581421,
      "logits/rejected": -0.591992199420929,
      "logps/chosen": -0.6629883050918579,
      "logps/rejected": -0.942578136920929,
      "loss": 0.8544,
      "nll_loss": 0.8070312738418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06633301079273224,
      "rewards/margins": 0.02799529954791069,
      "rewards/rejected": -0.09431152045726776,
      "step": 16490
    },
    {
      "epoch": 0.6183133162204193,
      "grad_norm": 2.0707590298586322,
      "learning_rate": 6.227991553292183e-07,
      "log_odds_chosen": 0.58026123046875,
      "log_odds_ratio": -0.5655273199081421,
      "logits/chosen": -0.7657226324081421,
      "logits/rejected": -0.6561523675918579,
      "logps/chosen": -0.6504882574081421,
      "logps/rejected": -1.007421851158142,
      "loss": 0.8444,
      "nll_loss": 0.75244140625,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.0650634765625,
      "rewards/margins": 0.03566589206457138,
      "rewards/rejected": -0.10073242336511612,
      "step": 16500
    },
    {
      "epoch": 0.6186880515635832,
      "grad_norm": 2.2304792529912922,
      "learning_rate": 6.226105140541567e-07,
      "log_odds_chosen": 0.43889158964157104,
      "log_odds_ratio": -0.6070312261581421,
      "logits/chosen": -0.7582031488418579,
      "logits/rejected": -0.6395508050918579,
      "logps/chosen": -0.668164074420929,
      "logps/rejected": -0.931835949420929,
      "loss": 0.8824,
      "nll_loss": 0.8236328363418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06688232719898224,
      "rewards/margins": 0.026275634765625,
      "rewards/rejected": -0.09316406399011612,
      "step": 16510
    },
    {
      "epoch": 0.6190627869067471,
      "grad_norm": 2.202367133056176,
      "learning_rate": 6.224220440894771e-07,
      "log_odds_chosen": 0.4453125,
      "log_odds_ratio": -0.643359363079071,
      "logits/chosen": -0.701171875,
      "logits/rejected": -0.641406238079071,
      "logps/chosen": -0.687695324420929,
      "logps/rejected": -0.9853515625,
      "loss": 0.8782,
      "nll_loss": 0.8060547113418579,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.06883545219898224,
      "rewards/margins": 0.02971496619284153,
      "rewards/rejected": -0.09844970703125,
      "step": 16520
    },
    {
      "epoch": 0.619437522249911,
      "grad_norm": 2.127878658569917,
      "learning_rate": 6.222337451760501e-07,
      "log_odds_chosen": 0.41435545682907104,
      "log_odds_ratio": -0.611035168170929,
      "logits/chosen": -0.695019543170929,
      "logits/rejected": -0.5933593511581421,
      "logps/chosen": -0.6910156011581421,
      "logps/rejected": -0.973828136920929,
      "loss": 0.8727,
      "nll_loss": 0.8013671636581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06901855766773224,
      "rewards/margins": 0.02833099290728569,
      "rewards/rejected": -0.09727783501148224,
      "step": 16530
    },
    {
      "epoch": 0.6198122575930749,
      "grad_norm": 2.033811124255422,
      "learning_rate": 6.220456170552948e-07,
      "log_odds_chosen": 0.5077880620956421,
      "log_odds_ratio": -0.5855468511581421,
      "logits/chosen": -0.7958984375,
      "logits/rejected": -0.6954101324081421,
      "logps/chosen": -0.6581054925918579,
      "logps/rejected": -0.986523449420929,
      "loss": 0.857,
      "nll_loss": 0.759765625,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06578369438648224,
      "rewards/margins": 0.0328638069331646,
      "rewards/rejected": -0.09862060844898224,
      "step": 16540
    },
    {
      "epoch": 0.6201869929362388,
      "grad_norm": 2.192395024896787,
      "learning_rate": 6.218576594691773e-07,
      "log_odds_chosen": 0.38361817598342896,
      "log_odds_ratio": -0.6436523199081421,
      "logits/chosen": -0.7369140386581421,
      "logits/rejected": -0.6654297113418579,
      "logps/chosen": -0.69287109375,
      "logps/rejected": -0.912890613079071,
      "loss": 0.877,
      "nll_loss": 0.797167956829071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0692138671875,
      "rewards/margins": 0.02205963060259819,
      "rewards/rejected": -0.09133300930261612,
      "step": 16550
    },
    {
      "epoch": 0.6205617282794027,
      "grad_norm": 2.132017700432901,
      "learning_rate": 6.216698721602092e-07,
      "log_odds_chosen": 0.560546875,
      "log_odds_ratio": -0.5621093511581421,
      "logits/chosen": -0.7466796636581421,
      "logits/rejected": -0.642285168170929,
      "logps/chosen": -0.675976574420929,
      "logps/rejected": -0.9984375238418579,
      "loss": 0.8708,
      "nll_loss": 0.8304687738418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06761474907398224,
      "rewards/margins": 0.032262422144412994,
      "rewards/rejected": -0.099853515625,
      "step": 16560
    },
    {
      "epoch": 0.6209364636225666,
      "grad_norm": 2.1463758509904705,
      "learning_rate": 6.214822548714457e-07,
      "log_odds_chosen": 0.525500476360321,
      "log_odds_ratio": -0.5943359136581421,
      "logits/chosen": -0.734667956829071,
      "logits/rejected": -0.601855456829071,
      "logps/chosen": -0.671093761920929,
      "logps/rejected": -1.0281250476837158,
      "loss": 0.8614,
      "nll_loss": 0.7923828363418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06708984076976776,
      "rewards/margins": 0.03569640964269638,
      "rewards/rejected": -0.10275878757238388,
      "step": 16570
    },
    {
      "epoch": 0.6213111989657305,
      "grad_norm": 2.2123812605473394,
      "learning_rate": 6.212948073464848e-07,
      "log_odds_chosen": 0.4690185487270355,
      "log_odds_ratio": -0.604687511920929,
      "logits/chosen": -0.735156238079071,
      "logits/rejected": -0.64697265625,
      "logps/chosen": -0.6781250238418579,
      "logps/rejected": -0.996874988079071,
      "loss": 0.8589,
      "nll_loss": 0.8382812738418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06785888969898224,
      "rewards/margins": 0.03183593600988388,
      "rewards/rejected": -0.09968261420726776,
      "step": 16580
    },
    {
      "epoch": 0.6216859343088943,
      "grad_norm": 2.538516757652647,
      "learning_rate": 6.211075293294654e-07,
      "log_odds_chosen": 0.614697277545929,
      "log_odds_ratio": -0.5892578363418579,
      "logits/chosen": -0.7685546875,
      "logits/rejected": -0.66162109375,
      "logps/chosen": -0.666015625,
      "logps/rejected": -1.032617211341858,
      "loss": 0.8754,
      "nll_loss": 0.8140624761581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06658935546875,
      "rewards/margins": 0.03662414476275444,
      "rewards/rejected": -0.10324706882238388,
      "step": 16590
    },
    {
      "epoch": 0.6220606696520582,
      "grad_norm": 2.407059300934661,
      "learning_rate": 6.209204205650662e-07,
      "log_odds_chosen": 0.4037109315395355,
      "log_odds_ratio": -0.6412109136581421,
      "logits/chosen": -0.7240234613418579,
      "logits/rejected": -0.6499999761581421,
      "logps/chosen": -0.709765613079071,
      "logps/rejected": -0.9683593511581421,
      "loss": 0.8508,
      "nll_loss": 0.849609375,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07102050632238388,
      "rewards/margins": 0.02594604529440403,
      "rewards/rejected": -0.09699706733226776,
      "step": 16600
    },
    {
      "epoch": 0.6224354049952221,
      "grad_norm": 2.7480286975843624,
      "learning_rate": 6.207334807985037e-07,
      "log_odds_chosen": 0.43597412109375,
      "log_odds_ratio": -0.5947265625,
      "logits/chosen": -0.78125,
      "logits/rejected": -0.6318359375,
      "logps/chosen": -0.653613269329071,
      "logps/rejected": -0.9281250238418579,
      "loss": 0.8739,
      "nll_loss": 0.7841796875,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06536865234375,
      "rewards/margins": 0.02746276929974556,
      "rewards/rejected": -0.09279785305261612,
      "step": 16610
    },
    {
      "epoch": 0.622810140338386,
      "grad_norm": 2.082727848988053,
      "learning_rate": 6.205467097755309e-07,
      "log_odds_chosen": 0.5150390863418579,
      "log_odds_ratio": -0.590283215045929,
      "logits/chosen": -0.6773437261581421,
      "logits/rejected": -0.571093738079071,
      "logps/chosen": -0.643750011920929,
      "logps/rejected": -0.9605468511581421,
      "loss": 0.8447,
      "nll_loss": 0.8130859136581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0643310546875,
      "rewards/margins": 0.0316314697265625,
      "rewards/rejected": -0.09600830078125,
      "step": 16620
    },
    {
      "epoch": 0.6231848756815499,
      "grad_norm": 2.218913120864041,
      "learning_rate": 6.203601072424364e-07,
      "log_odds_chosen": 0.43803709745407104,
      "log_odds_ratio": -0.610058605670929,
      "logits/chosen": -0.7369140386581421,
      "logits/rejected": -0.6126953363418579,
      "logps/chosen": -0.6683593988418579,
      "logps/rejected": -0.940234363079071,
      "loss": 0.8509,
      "nll_loss": 0.7412109375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06682129204273224,
      "rewards/margins": 0.02717132493853569,
      "rewards/rejected": -0.09396972507238388,
      "step": 16630
    },
    {
      "epoch": 0.6235596110247138,
      "grad_norm": 2.1591742340116187,
      "learning_rate": 6.201736729460422e-07,
      "log_odds_chosen": 0.6062256097793579,
      "log_odds_ratio": -0.5628906488418579,
      "logits/chosen": -0.749218761920929,
      "logits/rejected": -0.640332043170929,
      "logps/chosen": -0.638476550579071,
      "logps/rejected": -0.999218761920929,
      "loss": 0.8551,
      "nll_loss": 0.7962890863418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06383056938648224,
      "rewards/margins": 0.03598938137292862,
      "rewards/rejected": -0.09989013522863388,
      "step": 16640
    },
    {
      "epoch": 0.6239343463678777,
      "grad_norm": 2.2595759960203172,
      "learning_rate": 6.199874066337029e-07,
      "log_odds_chosen": 0.5676025152206421,
      "log_odds_ratio": -0.5716797113418579,
      "logits/chosen": -0.7367187738418579,
      "logits/rejected": -0.6434570550918579,
      "logps/chosen": -0.6377929449081421,
      "logps/rejected": -0.9820312261581421,
      "loss": 0.8755,
      "nll_loss": 0.809765636920929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06379394233226776,
      "rewards/margins": 0.03436584398150444,
      "rewards/rejected": -0.09824218600988388,
      "step": 16650
    },
    {
      "epoch": 0.6243090817110416,
      "grad_norm": 2.583141108831104,
      "learning_rate": 6.198013080533033e-07,
      "log_odds_chosen": 0.555023193359375,
      "log_odds_ratio": -0.5943359136581421,
      "logits/chosen": -0.712109386920929,
      "logits/rejected": -0.6241210699081421,
      "logps/chosen": -0.6478515863418579,
      "logps/rejected": -0.9671875238418579,
      "loss": 0.8907,
      "nll_loss": 0.800585925579071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06479492038488388,
      "rewards/margins": 0.0318756103515625,
      "rewards/rejected": -0.09663085639476776,
      "step": 16660
    },
    {
      "epoch": 0.6246838170542055,
      "grad_norm": 1.9578746040914385,
      "learning_rate": 6.196153769532584e-07,
      "log_odds_chosen": 0.6221923828125,
      "log_odds_ratio": -0.5521484613418579,
      "logits/chosen": -0.7476562261581421,
      "logits/rejected": -0.614062488079071,
      "logps/chosen": -0.6410156488418579,
      "logps/rejected": -1.009765625,
      "loss": 0.8568,
      "nll_loss": 0.7796875238418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06407471001148224,
      "rewards/margins": 0.03692169114947319,
      "rewards/rejected": -0.1009521484375,
      "step": 16670
    },
    {
      "epoch": 0.6250585523973694,
      "grad_norm": 2.2850406681142204,
      "learning_rate": 6.194296130825109e-07,
      "log_odds_chosen": 0.5267089605331421,
      "log_odds_ratio": -0.56787109375,
      "logits/chosen": -0.729785144329071,
      "logits/rejected": -0.6341797113418579,
      "logps/chosen": -0.6502929925918579,
      "logps/rejected": -0.9867187738418579,
      "loss": 0.8382,
      "nll_loss": 0.768261730670929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06501464545726776,
      "rewards/margins": 0.03363952785730362,
      "rewards/rejected": -0.09869384765625,
      "step": 16680
    },
    {
      "epoch": 0.6254332877405332,
      "grad_norm": 2.0054054442504574,
      "learning_rate": 6.192440161905297e-07,
      "log_odds_chosen": 0.27783203125,
      "log_odds_ratio": -0.664355456829071,
      "logits/chosen": -0.7300781011581421,
      "logits/rejected": -0.651660144329071,
      "logps/chosen": -0.6693359613418579,
      "logps/rejected": -0.8482421636581421,
      "loss": 0.8619,
      "nll_loss": 0.817187488079071,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.06694336235523224,
      "rewards/margins": 0.017903899773955345,
      "rewards/rejected": -0.08482666313648224,
      "step": 16690
    },
    {
      "epoch": 0.6258080230836971,
      "grad_norm": 2.1632422396541027,
      "learning_rate": 6.190585860273094e-07,
      "log_odds_chosen": 0.49189454317092896,
      "log_odds_ratio": -0.6136718988418579,
      "logits/chosen": -0.762499988079071,
      "logits/rejected": -0.6532226800918579,
      "logps/chosen": -0.675976574420929,
      "logps/rejected": -0.979687511920929,
      "loss": 0.8651,
      "nll_loss": 0.824023425579071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06761474907398224,
      "rewards/margins": 0.030342865735292435,
      "rewards/rejected": -0.09808349609375,
      "step": 16700
    },
    {
      "epoch": 0.626182758426861,
      "grad_norm": 2.30887874174024,
      "learning_rate": 6.18873322343368e-07,
      "log_odds_chosen": 0.51190185546875,
      "log_odds_ratio": -0.5845702886581421,
      "logits/chosen": -0.811718761920929,
      "logits/rejected": -0.688183605670929,
      "logps/chosen": -0.6766601800918579,
      "logps/rejected": -1.0002930164337158,
      "loss": 0.8627,
      "nll_loss": 0.797656238079071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.067626953125,
      "rewards/margins": 0.03241882473230362,
      "rewards/rejected": -0.10004882514476776,
      "step": 16710
    },
    {
      "epoch": 0.6265574937700249,
      "grad_norm": 1.9186275052413266,
      "learning_rate": 6.186882248897459e-07,
      "log_odds_chosen": 0.42918699979782104,
      "log_odds_ratio": -0.62646484375,
      "logits/chosen": -0.7525390386581421,
      "logits/rejected": -0.627636730670929,
      "logps/chosen": -0.64208984375,
      "logps/rejected": -0.914257824420929,
      "loss": 0.8637,
      "nll_loss": 0.7945312261581421,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.064208984375,
      "rewards/margins": 0.02726440504193306,
      "rewards/rejected": -0.09144286811351776,
      "step": 16720
    },
    {
      "epoch": 0.6269322291131888,
      "grad_norm": 2.070485407843171,
      "learning_rate": 6.185032934180045e-07,
      "log_odds_chosen": 0.5403198003768921,
      "log_odds_ratio": -0.573046863079071,
      "logits/chosen": -0.811718761920929,
      "logits/rejected": -0.6898437738418579,
      "logps/chosen": -0.649121105670929,
      "logps/rejected": -0.995312511920929,
      "loss": 0.8854,
      "nll_loss": 0.8203125,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06489257514476776,
      "rewards/margins": 0.03468017652630806,
      "rewards/rejected": -0.099609375,
      "step": 16730
    },
    {
      "epoch": 0.6273069644563527,
      "grad_norm": 2.0102007991369173,
      "learning_rate": 6.183185276802243e-07,
      "log_odds_chosen": 0.48155516386032104,
      "log_odds_ratio": -0.596972644329071,
      "logits/chosen": -0.775390625,
      "logits/rejected": -0.7164062261581421,
      "logps/chosen": -0.6591796875,
      "logps/rejected": -0.9507812261581421,
      "loss": 0.854,
      "nll_loss": 0.834179699420929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06588134914636612,
      "rewards/margins": 0.02922515943646431,
      "rewards/rejected": -0.09514160454273224,
      "step": 16740
    },
    {
      "epoch": 0.6276816997995166,
      "grad_norm": 2.144334900980369,
      "learning_rate": 6.181339274290046e-07,
      "log_odds_chosen": 0.4290832579135895,
      "log_odds_ratio": -0.61181640625,
      "logits/chosen": -0.7972656488418579,
      "logits/rejected": -0.7137695550918579,
      "logps/chosen": -0.6822265386581421,
      "logps/rejected": -0.9556640386581421,
      "loss": 0.8734,
      "nll_loss": 0.838671863079071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06829833984375,
      "rewards/margins": 0.02728881873190403,
      "rewards/rejected": -0.09555663913488388,
      "step": 16750
    },
    {
      "epoch": 0.6280564351426805,
      "grad_norm": 1.9008658672265504,
      "learning_rate": 6.179494924174608e-07,
      "log_odds_chosen": 0.41552734375,
      "log_odds_ratio": -0.6170898675918579,
      "logits/chosen": -0.7347656488418579,
      "logits/rejected": -0.6299804449081421,
      "logps/chosen": -0.657421886920929,
      "logps/rejected": -0.925000011920929,
      "loss": 0.8634,
      "nll_loss": 0.8426758050918579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06578369438648224,
      "rewards/margins": 0.0266876220703125,
      "rewards/rejected": -0.09243164211511612,
      "step": 16760
    },
    {
      "epoch": 0.6284311704858444,
      "grad_norm": 1.879704717649125,
      "learning_rate": 6.177652223992242e-07,
      "log_odds_chosen": 0.5585571527481079,
      "log_odds_ratio": -0.57666015625,
      "logits/chosen": -0.749804675579071,
      "logits/rejected": -0.627246081829071,
      "logps/chosen": -0.680468738079071,
      "logps/rejected": -1.0419921875,
      "loss": 0.852,
      "nll_loss": 0.760937511920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06812743842601776,
      "rewards/margins": 0.03610992431640625,
      "rewards/rejected": -0.10417480766773224,
      "step": 16770
    },
    {
      "epoch": 0.6288059058290083,
      "grad_norm": 2.3590562354659665,
      "learning_rate": 6.175811171284396e-07,
      "log_odds_chosen": 0.47279053926467896,
      "log_odds_ratio": -0.6124023199081421,
      "logits/chosen": -0.731640636920929,
      "logits/rejected": -0.670117199420929,
      "logps/chosen": -0.691210925579071,
      "logps/rejected": -1.017187476158142,
      "loss": 0.8885,
      "nll_loss": 0.8763672113418579,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06912841647863388,
      "rewards/margins": 0.032524872571229935,
      "rewards/rejected": -0.10167236626148224,
      "step": 16780
    },
    {
      "epoch": 0.6291806411721722,
      "grad_norm": 2.489644058215413,
      "learning_rate": 6.173971763597644e-07,
      "log_odds_chosen": 0.567883312702179,
      "log_odds_ratio": -0.564648449420929,
      "logits/chosen": -0.797656238079071,
      "logits/rejected": -0.6507812738418579,
      "logps/chosen": -0.6625000238418579,
      "logps/rejected": -1.005273461341858,
      "loss": 0.8586,
      "nll_loss": 0.809374988079071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06624756008386612,
      "rewards/margins": 0.03440093994140625,
      "rewards/rejected": -0.10061035305261612,
      "step": 16790
    },
    {
      "epoch": 0.629555376515336,
      "grad_norm": 2.212070160133073,
      "learning_rate": 6.172133998483677e-07,
      "log_odds_chosen": 0.5162353515625,
      "log_odds_ratio": -0.585644543170929,
      "logits/chosen": -0.7494140863418579,
      "logits/rejected": -0.6197265386581421,
      "logps/chosen": -0.728515625,
      "logps/rejected": -1.0732421875,
      "loss": 0.863,
      "nll_loss": 0.8158203363418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07288818061351776,
      "rewards/margins": 0.03441924974322319,
      "rewards/rejected": -0.10722656548023224,
      "step": 16800
    },
    {
      "epoch": 0.6299301118584999,
      "grad_norm": 2.047412514753427,
      "learning_rate": 6.170297873499277e-07,
      "log_odds_chosen": 0.4535156190395355,
      "log_odds_ratio": -0.612011730670929,
      "logits/chosen": -0.765625,
      "logits/rejected": -0.639453113079071,
      "logps/chosen": -0.72998046875,
      "logps/rejected": -1.048242211341858,
      "loss": 0.8522,
      "nll_loss": 0.8169921636581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07298584282398224,
      "rewards/margins": 0.03178100660443306,
      "rewards/rejected": -0.10478515923023224,
      "step": 16810
    },
    {
      "epoch": 0.6303048472016638,
      "grad_norm": 1.9872920813428077,
      "learning_rate": 6.168463386206317e-07,
      "log_odds_chosen": 0.392822265625,
      "log_odds_ratio": -0.636914074420929,
      "logits/chosen": -0.6888672113418579,
      "logits/rejected": -0.6025390625,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -0.9703124761581421,
      "loss": 0.8414,
      "nll_loss": 0.8255859613418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07166747748851776,
      "rewards/margins": 0.02536315843462944,
      "rewards/rejected": -0.09707031399011612,
      "step": 16820
    },
    {
      "epoch": 0.6306795825448277,
      "grad_norm": 2.273601756894349,
      "learning_rate": 6.166630534171737e-07,
      "log_odds_chosen": 0.3898681700229645,
      "log_odds_ratio": -0.650585949420929,
      "logits/chosen": -0.7412109375,
      "logits/rejected": -0.645703136920929,
      "logps/chosen": -0.672070324420929,
      "logps/rejected": -0.925976574420929,
      "loss": 0.8633,
      "nll_loss": 0.834179699420929,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06722412258386612,
      "rewards/margins": 0.025399018079042435,
      "rewards/rejected": -0.09256591647863388,
      "step": 16830
    },
    {
      "epoch": 0.6310543178879916,
      "grad_norm": 1.9886855608058125,
      "learning_rate": 6.164799314967538e-07,
      "log_odds_chosen": 0.35307615995407104,
      "log_odds_ratio": -0.66162109375,
      "logits/chosen": -0.75390625,
      "logits/rejected": -0.649121105670929,
      "logps/chosen": -0.704785168170929,
      "logps/rejected": -0.9375976324081421,
      "loss": 0.8878,
      "nll_loss": 0.830273449420929,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07044677436351776,
      "rewards/margins": 0.02338561974465847,
      "rewards/rejected": -0.09381103515625,
      "step": 16840
    },
    {
      "epoch": 0.6314290532311555,
      "grad_norm": 2.020894266243709,
      "learning_rate": 6.162969726170763e-07,
      "log_odds_chosen": 0.5347656011581421,
      "log_odds_ratio": -0.597460925579071,
      "logits/chosen": -0.775195300579071,
      "logits/rejected": -0.6297851800918579,
      "logps/chosen": -0.693554699420929,
      "logps/rejected": -1.0593750476837158,
      "loss": 0.8599,
      "nll_loss": 0.797656238079071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06938476860523224,
      "rewards/margins": 0.03652801364660263,
      "rewards/rejected": -0.10585937649011612,
      "step": 16850
    },
    {
      "epoch": 0.6318037885743194,
      "grad_norm": 1.918825477150599,
      "learning_rate": 6.161141765363486e-07,
      "log_odds_chosen": 0.528796374797821,
      "log_odds_ratio": -0.5927734375,
      "logits/chosen": -0.7652343511581421,
      "logits/rejected": -0.662109375,
      "logps/chosen": -0.6991211175918579,
      "logps/rejected": -1.044921875,
      "loss": 0.8624,
      "nll_loss": 0.7734375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06993408501148224,
      "rewards/margins": 0.034621428698301315,
      "rewards/rejected": -0.10457763820886612,
      "step": 16860
    },
    {
      "epoch": 0.6321785239174833,
      "grad_norm": 2.36253334211033,
      "learning_rate": 6.159315430132796e-07,
      "log_odds_chosen": 0.44451904296875,
      "log_odds_ratio": -0.6006835699081421,
      "logits/chosen": -0.73046875,
      "logits/rejected": -0.6273437738418579,
      "logps/chosen": -0.6826171875,
      "logps/rejected": -0.982421875,
      "loss": 0.8732,
      "nll_loss": 0.806640625,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06831054389476776,
      "rewards/margins": 0.03002471849322319,
      "rewards/rejected": -0.09819336235523224,
      "step": 16870
    },
    {
      "epoch": 0.6325532592606472,
      "grad_norm": 2.476415561762487,
      "learning_rate": 6.157490718070792e-07,
      "log_odds_chosen": 0.58734130859375,
      "log_odds_ratio": -0.584667980670929,
      "logits/chosen": -0.767382800579071,
      "logits/rejected": -0.6103515625,
      "logps/chosen": -0.6429687738418579,
      "logps/rejected": -0.9921875,
      "loss": 0.8644,
      "nll_loss": 0.776562511920929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06427001953125,
      "rewards/margins": 0.03496398776769638,
      "rewards/rejected": -0.09916992485523224,
      "step": 16880
    },
    {
      "epoch": 0.6329279946038111,
      "grad_norm": 2.099009896978701,
      "learning_rate": 6.155667626774557e-07,
      "log_odds_chosen": 0.632568359375,
      "log_odds_ratio": -0.5376952886581421,
      "logits/chosen": -0.784960925579071,
      "logits/rejected": -0.6566406488418579,
      "logps/chosen": -0.6421874761581421,
      "logps/rejected": -1.0654296875,
      "loss": 0.8624,
      "nll_loss": 0.7626953125,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06419678032398224,
      "rewards/margins": 0.04233093187212944,
      "rewards/rejected": -0.1065673828125,
      "step": 16890
    },
    {
      "epoch": 0.6333027299469749,
      "grad_norm": 2.4083449027121198,
      "learning_rate": 6.153846153846154e-07,
      "log_odds_chosen": 0.412933349609375,
      "log_odds_ratio": -0.61669921875,
      "logits/chosen": -0.775195300579071,
      "logits/rejected": -0.692675769329071,
      "logps/chosen": -0.6678711175918579,
      "logps/rejected": -0.93212890625,
      "loss": 0.8523,
      "nll_loss": 0.771679699420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06691894680261612,
      "rewards/margins": 0.026267241686582565,
      "rewards/rejected": -0.09321288764476776,
      "step": 16900
    },
    {
      "epoch": 0.6336774652901388,
      "grad_norm": 2.1810513776793967,
      "learning_rate": 6.152026296892608e-07,
      "log_odds_chosen": 0.3439697325229645,
      "log_odds_ratio": -0.676562488079071,
      "logits/chosen": -0.719921886920929,
      "logits/rejected": -0.6396484375,
      "logps/chosen": -0.7300781011581421,
      "logps/rejected": -0.966015636920929,
      "loss": 0.857,
      "nll_loss": 0.8599609136581421,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.07307128608226776,
      "rewards/margins": 0.02352294884622097,
      "rewards/rejected": -0.09660644829273224,
      "step": 16910
    },
    {
      "epoch": 0.6340522006333027,
      "grad_norm": 2.1147140624864984,
      "learning_rate": 6.150208053525901e-07,
      "log_odds_chosen": 0.45814818143844604,
      "log_odds_ratio": -0.608105480670929,
      "logits/chosen": -0.6849609613418579,
      "logits/rejected": -0.6058593988418579,
      "logps/chosen": -0.6865234375,
      "logps/rejected": -0.975390613079071,
      "loss": 0.868,
      "nll_loss": 0.7982422113418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06870117038488388,
      "rewards/margins": 0.02890167199075222,
      "rewards/rejected": -0.09754638373851776,
      "step": 16920
    },
    {
      "epoch": 0.6344269359764666,
      "grad_norm": 2.27144065365765,
      "learning_rate": 6.148391421362942e-07,
      "log_odds_chosen": 0.601147472858429,
      "log_odds_ratio": -0.55224609375,
      "logits/chosen": -0.7046874761581421,
      "logits/rejected": -0.6246093511581421,
      "logps/chosen": -0.6376953125,
      "logps/rejected": -1.0226562023162842,
      "loss": 0.8632,
      "nll_loss": 0.80078125,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06378173828125,
      "rewards/margins": 0.03847808763384819,
      "rewards/rejected": -0.10220947116613388,
      "step": 16930
    },
    {
      "epoch": 0.6348016713196305,
      "grad_norm": 2.167273007491457,
      "learning_rate": 6.146576398025575e-07,
      "log_odds_chosen": 0.43023681640625,
      "log_odds_ratio": -0.595019519329071,
      "logits/chosen": -0.7037109136581421,
      "logits/rejected": -0.6405273675918579,
      "logps/chosen": -0.655078113079071,
      "logps/rejected": -0.9017578363418579,
      "loss": 0.8553,
      "nll_loss": 0.830761730670929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06556396186351776,
      "rewards/margins": 0.024658966809511185,
      "rewards/rejected": -0.09022216498851776,
      "step": 16940
    },
    {
      "epoch": 0.6351764066627944,
      "grad_norm": 2.085081232798312,
      "learning_rate": 6.144762981140549e-07,
      "log_odds_chosen": 0.38056641817092896,
      "log_odds_ratio": -0.6415039300918579,
      "logits/chosen": -0.7291015386581421,
      "logits/rejected": -0.634765625,
      "logps/chosen": -0.6502929925918579,
      "logps/rejected": -0.9019531011581421,
      "loss": 0.8611,
      "nll_loss": 0.8197265863418579,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06503906100988388,
      "rewards/margins": 0.02523345872759819,
      "rewards/rejected": -0.09028320014476776,
      "step": 16950
    },
    {
      "epoch": 0.6355511420059583,
      "grad_norm": 2.078047640282113,
      "learning_rate": 6.142951168339512e-07,
      "log_odds_chosen": 0.7417236566543579,
      "log_odds_ratio": -0.523632824420929,
      "logits/chosen": -0.7250000238418579,
      "logits/rejected": -0.649121105670929,
      "logps/chosen": -0.70751953125,
      "logps/rejected": -1.171484351158142,
      "loss": 0.8395,
      "nll_loss": 0.812695324420929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07071533054113388,
      "rewards/margins": 0.04644012451171875,
      "rewards/rejected": -0.11713866889476776,
      "step": 16960
    },
    {
      "epoch": 0.6359258773491222,
      "grad_norm": 2.2770930826473634,
      "learning_rate": 6.141140957259e-07,
      "log_odds_chosen": 0.6820923089981079,
      "log_odds_ratio": -0.5516601800918579,
      "logits/chosen": -0.7266601324081421,
      "logits/rejected": -0.648242175579071,
      "logps/chosen": -0.694628894329071,
      "logps/rejected": -1.141015648841858,
      "loss": 0.8627,
      "nll_loss": 0.8720703125,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06948242336511612,
      "rewards/margins": 0.044617462903261185,
      "rewards/rejected": -0.11409912258386612,
      "step": 16970
    },
    {
      "epoch": 0.6363006126922861,
      "grad_norm": 2.6167058768628246,
      "learning_rate": 6.139332345540418e-07,
      "log_odds_chosen": 0.5335693359375,
      "log_odds_ratio": -0.5721679925918579,
      "logits/chosen": -0.7494140863418579,
      "logits/rejected": -0.6499999761581421,
      "logps/chosen": -0.6830078363418579,
      "logps/rejected": -1.009179711341858,
      "loss": 0.856,
      "nll_loss": 0.8150390386581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.068359375,
      "rewards/margins": 0.032706450670957565,
      "rewards/rejected": -0.10097656399011612,
      "step": 16980
    },
    {
      "epoch": 0.63667534803545,
      "grad_norm": 2.3314131052597586,
      "learning_rate": 6.137525330830035e-07,
      "log_odds_chosen": 0.503125011920929,
      "log_odds_ratio": -0.605273425579071,
      "logits/chosen": -0.7035156488418579,
      "logits/rejected": -0.59619140625,
      "logps/chosen": -0.6949218511581421,
      "logps/rejected": -1.023046851158142,
      "loss": 0.8616,
      "nll_loss": 0.78125,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06956787407398224,
      "rewards/margins": 0.03278655931353569,
      "rewards/rejected": -0.10239257663488388,
      "step": 16990
    },
    {
      "epoch": 0.6370500833786139,
      "grad_norm": 2.1432628363215764,
      "learning_rate": 6.135719910778963e-07,
      "log_odds_chosen": 0.45863038301467896,
      "log_odds_ratio": -0.609179675579071,
      "logits/chosen": -0.7265625,
      "logits/rejected": -0.620410144329071,
      "logps/chosen": -0.738574206829071,
      "logps/rejected": -1.0271484851837158,
      "loss": 0.8669,
      "nll_loss": 0.7906249761581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07388915866613388,
      "rewards/margins": 0.028888702392578125,
      "rewards/rejected": -0.10270996391773224,
      "step": 17000
    },
    {
      "epoch": 0.6374248187217777,
      "grad_norm": 2.1010710575044147,
      "learning_rate": 6.133916083043149e-07,
      "log_odds_chosen": 0.510449230670929,
      "log_odds_ratio": -0.5894531011581421,
      "logits/chosen": -0.77392578125,
      "logits/rejected": -0.6615234613418579,
      "logps/chosen": -0.677441418170929,
      "logps/rejected": -1.009765625,
      "loss": 0.8571,
      "nll_loss": 0.772656261920929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06771240383386612,
      "rewards/margins": 0.03326416015625,
      "rewards/rejected": -0.10097656399011612,
      "step": 17010
    },
    {
      "epoch": 0.6377995540649416,
      "grad_norm": 2.145643580934792,
      "learning_rate": 6.132113845283359e-07,
      "log_odds_chosen": 0.511279284954071,
      "log_odds_ratio": -0.599609375,
      "logits/chosen": -0.748046875,
      "logits/rejected": -0.6298828125,
      "logps/chosen": -0.698535144329071,
      "logps/rejected": -1.0437500476837158,
      "loss": 0.8457,
      "nll_loss": 0.8248046636581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06981201469898224,
      "rewards/margins": 0.034543227404356,
      "rewards/rejected": -0.10432128608226776,
      "step": 17020
    },
    {
      "epoch": 0.6381742894081055,
      "grad_norm": 2.4044507398744144,
      "learning_rate": 6.13031319516517e-07,
      "log_odds_chosen": 0.45404052734375,
      "log_odds_ratio": -0.6102539300918579,
      "logits/chosen": -0.7455078363418579,
      "logits/rejected": -0.624218761920929,
      "logps/chosen": -0.6283203363418579,
      "logps/rejected": -0.8929687738418579,
      "loss": 0.8422,
      "nll_loss": 0.786328136920929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06287841498851776,
      "rewards/margins": 0.02642974816262722,
      "rewards/rejected": -0.0892333984375,
      "step": 17030
    },
    {
      "epoch": 0.6385490247512694,
      "grad_norm": 2.4515093204789404,
      "learning_rate": 6.128514130358955e-07,
      "log_odds_chosen": 0.43425291776657104,
      "log_odds_ratio": -0.618847668170929,
      "logits/chosen": -0.795703113079071,
      "logits/rejected": -0.659375011920929,
      "logps/chosen": -0.7220703363418579,
      "logps/rejected": -1.0089843273162842,
      "loss": 0.8475,
      "nll_loss": 0.801953136920929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07216797024011612,
      "rewards/margins": 0.02871399000287056,
      "rewards/rejected": -0.10090331733226776,
      "step": 17040
    },
    {
      "epoch": 0.6389237600944333,
      "grad_norm": 2.282454037360414,
      "learning_rate": 6.126716648539868e-07,
      "log_odds_chosen": 0.5330566167831421,
      "log_odds_ratio": -0.58056640625,
      "logits/chosen": -0.7300781011581421,
      "logits/rejected": -0.616406261920929,
      "logps/chosen": -0.6986328363418579,
      "logps/rejected": -1.04296875,
      "loss": 0.8573,
      "nll_loss": 0.834179699420929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06987304985523224,
      "rewards/margins": 0.03439636155962944,
      "rewards/rejected": -0.10434570163488388,
      "step": 17050
    },
    {
      "epoch": 0.6392984954375972,
      "grad_norm": 1.927198190478997,
      "learning_rate": 6.124920747387834e-07,
      "log_odds_chosen": 0.6166747808456421,
      "log_odds_ratio": -0.567089855670929,
      "logits/chosen": -0.7513672113418579,
      "logits/rejected": -0.639453113079071,
      "logps/chosen": -0.6878906488418579,
      "logps/rejected": -1.0769531726837158,
      "loss": 0.8494,
      "nll_loss": 0.7681640386581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06873778998851776,
      "rewards/margins": 0.03900756686925888,
      "rewards/rejected": -0.10776367038488388,
      "step": 17060
    },
    {
      "epoch": 0.6396732307807611,
      "grad_norm": 2.496262389423842,
      "learning_rate": 6.123126424587535e-07,
      "log_odds_chosen": 0.507373034954071,
      "log_odds_ratio": -0.59912109375,
      "logits/chosen": -0.712109386920929,
      "logits/rejected": -0.6063476800918579,
      "logps/chosen": -0.6651366949081421,
      "logps/rejected": -1.000390648841858,
      "loss": 0.855,
      "nll_loss": 0.7730468511581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06651611626148224,
      "rewards/margins": 0.033527374267578125,
      "rewards/rejected": -0.10001220554113388,
      "step": 17070
    },
    {
      "epoch": 0.640047966123925,
      "grad_norm": 2.235239477744223,
      "learning_rate": 6.121333677828399e-07,
      "log_odds_chosen": 0.4558959901332855,
      "log_odds_ratio": -0.603710949420929,
      "logits/chosen": -0.7337890863418579,
      "logits/rejected": -0.6533203125,
      "logps/chosen": -0.7083984613418579,
      "logps/rejected": -1.0031249523162842,
      "loss": 0.8455,
      "nll_loss": 0.779492199420929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07082519680261612,
      "rewards/margins": 0.02947692945599556,
      "rewards/rejected": -0.10031738132238388,
      "step": 17080
    },
    {
      "epoch": 0.6404227014670889,
      "grad_norm": 2.3497131847883357,
      "learning_rate": 6.119542504804587e-07,
      "log_odds_chosen": 0.44868165254592896,
      "log_odds_ratio": -0.6240234375,
      "logits/chosen": -0.7724609375,
      "logits/rejected": -0.6675781011581421,
      "logps/chosen": -0.696093738079071,
      "logps/rejected": -0.9638671875,
      "loss": 0.8637,
      "nll_loss": 0.796093761920929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06965331733226776,
      "rewards/margins": 0.02678832970559597,
      "rewards/rejected": -0.09639892727136612,
      "step": 17090
    },
    {
      "epoch": 0.6407974368102528,
      "grad_norm": 2.0823067872921674,
      "learning_rate": 6.11775290321498e-07,
      "log_odds_chosen": 0.41069334745407104,
      "log_odds_ratio": -0.6451171636581421,
      "logits/chosen": -0.744335949420929,
      "logits/rejected": -0.61962890625,
      "logps/chosen": -0.7147461175918579,
      "logps/rejected": -0.987500011920929,
      "loss": 0.8549,
      "nll_loss": 0.78125,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07154540717601776,
      "rewards/margins": 0.02716979943215847,
      "rewards/rejected": -0.0986328125,
      "step": 17100
    },
    {
      "epoch": 0.6411721721534166,
      "grad_norm": 2.367102937779876,
      "learning_rate": 6.115964870763166e-07,
      "log_odds_chosen": 0.686206042766571,
      "log_odds_ratio": -0.5428222417831421,
      "logits/chosen": -0.745312511920929,
      "logits/rejected": -0.6231445074081421,
      "logps/chosen": -0.6703125238418579,
      "logps/rejected": -1.1193358898162842,
      "loss": 0.8712,
      "nll_loss": 0.7953125238418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06704101711511612,
      "rewards/margins": 0.04489745944738388,
      "rewards/rejected": -0.1119384765625,
      "step": 17110
    },
    {
      "epoch": 0.6415469074965805,
      "grad_norm": 2.7205649633205207,
      "learning_rate": 6.114178405157431e-07,
      "log_odds_chosen": 0.672802746295929,
      "log_odds_ratio": -0.5741211175918579,
      "logits/chosen": -0.763671875,
      "logits/rejected": -0.626171886920929,
      "logps/chosen": -0.701367199420929,
      "logps/rejected": -1.1349608898162842,
      "loss": 0.8486,
      "nll_loss": 0.781445324420929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07014159858226776,
      "rewards/margins": 0.04333343356847763,
      "rewards/rejected": -0.11347655951976776,
      "step": 17120
    },
    {
      "epoch": 0.6419216428397444,
      "grad_norm": 2.3089104948149366,
      "learning_rate": 6.112393504110738e-07,
      "log_odds_chosen": 0.5883544683456421,
      "log_odds_ratio": -0.6001952886581421,
      "logits/chosen": -0.730273425579071,
      "logits/rejected": -0.658007800579071,
      "logps/chosen": -0.6826171875,
      "logps/rejected": -1.0499999523162842,
      "loss": 0.8517,
      "nll_loss": 0.7789062261581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06828613579273224,
      "rewards/margins": 0.03667144849896431,
      "rewards/rejected": -0.10498046875,
      "step": 17130
    },
    {
      "epoch": 0.6422963781829083,
      "grad_norm": 2.0937426542193074,
      "learning_rate": 6.110610165340729e-07,
      "log_odds_chosen": 0.47749024629592896,
      "log_odds_ratio": -0.576171875,
      "logits/chosen": -0.755859375,
      "logits/rejected": -0.6434570550918579,
      "logps/chosen": -0.684765636920929,
      "logps/rejected": -0.9751952886581421,
      "loss": 0.8608,
      "nll_loss": 0.8125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06845702975988388,
      "rewards/margins": 0.029052734375,
      "rewards/rejected": -0.09750976413488388,
      "step": 17140
    },
    {
      "epoch": 0.6426711135260722,
      "grad_norm": 2.0384966509238325,
      "learning_rate": 6.1088283865697e-07,
      "log_odds_chosen": 0.5920654535293579,
      "log_odds_ratio": -0.55322265625,
      "logits/chosen": -0.7427734136581421,
      "logits/rejected": -0.623828113079071,
      "logps/chosen": -0.6761718988418579,
      "logps/rejected": -1.061914086341858,
      "loss": 0.8519,
      "nll_loss": 0.820507824420929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06754150241613388,
      "rewards/margins": 0.038596343249082565,
      "rewards/rejected": -0.106201171875,
      "step": 17150
    },
    {
      "epoch": 0.6430458488692361,
      "grad_norm": 2.0981788703812145,
      "learning_rate": 6.107048165524593e-07,
      "log_odds_chosen": 0.5622802972793579,
      "log_odds_ratio": -0.585742175579071,
      "logits/chosen": -0.7593749761581421,
      "logits/rejected": -0.66259765625,
      "logps/chosen": -0.651171863079071,
      "logps/rejected": -1.0100586414337158,
      "loss": 0.8345,
      "nll_loss": 0.8076171875,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06507568061351776,
      "rewards/margins": 0.03580169752240181,
      "rewards/rejected": -0.10096435248851776,
      "step": 17160
    },
    {
      "epoch": 0.6434205842124,
      "grad_norm": 2.329472249588748,
      "learning_rate": 6.105269499936986e-07,
      "log_odds_chosen": 0.4488769471645355,
      "log_odds_ratio": -0.6094726324081421,
      "logits/chosen": -0.780566394329071,
      "logits/rejected": -0.6322265863418579,
      "logps/chosen": -0.724316418170929,
      "logps/rejected": -1.021093726158142,
      "loss": 0.8618,
      "nll_loss": 0.7894531488418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07241211086511612,
      "rewards/margins": 0.029700469225645065,
      "rewards/rejected": -0.10212402045726776,
      "step": 17170
    },
    {
      "epoch": 0.6437953195555639,
      "grad_norm": 2.1567794853243605,
      "learning_rate": 6.103492387543075e-07,
      "log_odds_chosen": 0.55029296875,
      "log_odds_ratio": -0.5654296875,
      "logits/chosen": -0.7435547113418579,
      "logits/rejected": -0.6625000238418579,
      "logps/chosen": -0.6463867425918579,
      "logps/rejected": -0.961718738079071,
      "loss": 0.8614,
      "nll_loss": 0.800585925579071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06463623046875,
      "rewards/margins": 0.03149871900677681,
      "rewards/rejected": -0.09614257514476776,
      "step": 17180
    },
    {
      "epoch": 0.6441700548987278,
      "grad_norm": 2.08254350313143,
      "learning_rate": 6.101716826083674e-07,
      "log_odds_chosen": 0.604504406452179,
      "log_odds_ratio": -0.54541015625,
      "logits/chosen": -0.748828113079071,
      "logits/rejected": -0.6246093511581421,
      "logps/chosen": -0.6669921875,
      "logps/rejected": -1.0341796875,
      "loss": 0.8611,
      "nll_loss": 0.7796875238418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06667480617761612,
      "rewards/margins": 0.03671569749712944,
      "rewards/rejected": -0.10332031548023224,
      "step": 17190
    },
    {
      "epoch": 0.6445447902418917,
      "grad_norm": 2.0923343750403496,
      "learning_rate": 6.099942813304186e-07,
      "log_odds_chosen": 0.6726318597793579,
      "log_odds_ratio": -0.529003918170929,
      "logits/chosen": -0.728808581829071,
      "logits/rejected": -0.6407226324081421,
      "logps/chosen": -0.63134765625,
      "logps/rejected": -1.0437500476837158,
      "loss": 0.8608,
      "nll_loss": 0.7691406011581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06312255561351776,
      "rewards/margins": 0.04127807542681694,
      "rewards/rejected": -0.10429687798023224,
      "step": 17200
    },
    {
      "epoch": 0.6449195255850556,
      "grad_norm": 2.152989417421694,
      "learning_rate": 6.098170346954607e-07,
      "log_odds_chosen": 0.4710449278354645,
      "log_odds_ratio": -0.600878894329071,
      "logits/chosen": -0.748046875,
      "logits/rejected": -0.640820324420929,
      "logps/chosen": -0.7001953125,
      "logps/rejected": -1.017187476158142,
      "loss": 0.8631,
      "nll_loss": 0.814257800579071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07009277492761612,
      "rewards/margins": 0.03160400316119194,
      "rewards/rejected": -0.10163573920726776,
      "step": 17210
    },
    {
      "epoch": 0.6452942609282194,
      "grad_norm": 2.037660035532382,
      "learning_rate": 6.0963994247895e-07,
      "log_odds_chosen": 0.48388671875,
      "log_odds_ratio": -0.5947265625,
      "logits/chosen": -0.7705078125,
      "logits/rejected": -0.631542980670929,
      "logps/chosen": -0.68212890625,
      "logps/rejected": -0.9769531488418579,
      "loss": 0.8555,
      "nll_loss": 0.7660156488418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06817626953125,
      "rewards/margins": 0.02951965294778347,
      "rewards/rejected": -0.09775390475988388,
      "step": 17220
    },
    {
      "epoch": 0.6456689962713833,
      "grad_norm": 2.1630838982574168,
      "learning_rate": 6.094630044567996e-07,
      "log_odds_chosen": 0.49254149198532104,
      "log_odds_ratio": -0.5931640863418579,
      "logits/chosen": -0.762011706829071,
      "logits/rejected": -0.670214831829071,
      "logps/chosen": -0.677734375,
      "logps/rejected": -0.9814453125,
      "loss": 0.8533,
      "nll_loss": 0.805859386920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06776122748851776,
      "rewards/margins": 0.03043365478515625,
      "rewards/rejected": -0.09814453125,
      "step": 17230
    },
    {
      "epoch": 0.6460437316145472,
      "grad_norm": 1.999692674445545,
      "learning_rate": 6.092862204053773e-07,
      "log_odds_chosen": 0.531420886516571,
      "log_odds_ratio": -0.594433605670929,
      "logits/chosen": -0.771289050579071,
      "logits/rejected": -0.6241210699081421,
      "logps/chosen": -0.7109375,
      "logps/rejected": -1.0632812976837158,
      "loss": 0.8584,
      "nll_loss": 0.8130859136581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07113037258386612,
      "rewards/margins": 0.03523559495806694,
      "rewards/rejected": -0.10627441108226776,
      "step": 17240
    },
    {
      "epoch": 0.6464184669577111,
      "grad_norm": 2.271991387963414,
      "learning_rate": 6.091095901015048e-07,
      "log_odds_chosen": 0.5920776128768921,
      "log_odds_ratio": -0.550000011920929,
      "logits/chosen": -0.7789062261581421,
      "logits/rejected": -0.66943359375,
      "logps/chosen": -0.649121105670929,
      "logps/rejected": -0.9908202886581421,
      "loss": 0.86,
      "nll_loss": 0.7724609375,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06491699069738388,
      "rewards/margins": 0.0341644287109375,
      "rewards/rejected": -0.09904785454273224,
      "step": 17250
    },
    {
      "epoch": 0.646793202300875,
      "grad_norm": 2.049090269210873,
      "learning_rate": 6.089331133224562e-07,
      "log_odds_chosen": 0.513012707233429,
      "log_odds_ratio": -0.591015636920929,
      "logits/chosen": -0.7275390625,
      "logits/rejected": -0.6328125,
      "logps/chosen": -0.68505859375,
      "logps/rejected": -1.0134766101837158,
      "loss": 0.8642,
      "nll_loss": 0.784960925579071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06850586086511612,
      "rewards/margins": 0.03284301608800888,
      "rewards/rejected": -0.10136719048023224,
      "step": 17260
    },
    {
      "epoch": 0.6471679376440389,
      "grad_norm": 2.155580425624839,
      "learning_rate": 6.087567898459576e-07,
      "log_odds_chosen": 0.3864502012729645,
      "log_odds_ratio": -0.6138671636581421,
      "logits/chosen": -0.744921863079071,
      "logits/rejected": -0.6390625238418579,
      "logps/chosen": -0.677734375,
      "logps/rejected": -0.880859375,
      "loss": 0.8519,
      "nll_loss": 0.8392578363418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06770019233226776,
      "rewards/margins": 0.02031555213034153,
      "rewards/rejected": -0.08811035007238388,
      "step": 17270
    },
    {
      "epoch": 0.6475426729872028,
      "grad_norm": 2.314701550961634,
      "learning_rate": 6.085806194501844e-07,
      "log_odds_chosen": 0.4640869200229645,
      "log_odds_ratio": -0.602832019329071,
      "logits/chosen": -0.733203113079071,
      "logits/rejected": -0.6366211175918579,
      "logps/chosen": -0.7232421636581421,
      "logps/rejected": -1.0089843273162842,
      "loss": 0.8414,
      "nll_loss": 0.8150390386581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0723876953125,
      "rewards/margins": 0.028564453125,
      "rewards/rejected": -0.10092773288488388,
      "step": 17280
    },
    {
      "epoch": 0.6479174083303667,
      "grad_norm": 2.3158022771131392,
      "learning_rate": 6.084046019137626e-07,
      "log_odds_chosen": 0.41241455078125,
      "log_odds_ratio": -0.6397460699081421,
      "logits/chosen": -0.6927734613418579,
      "logits/rejected": -0.650585949420929,
      "logps/chosen": -0.6986328363418579,
      "logps/rejected": -0.9750000238418579,
      "loss": 0.838,
      "nll_loss": 0.801562488079071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06983642280101776,
      "rewards/margins": 0.02764282189309597,
      "rewards/rejected": -0.0975341796875,
      "step": 17290
    },
    {
      "epoch": 0.6482921436735306,
      "grad_norm": 2.111111235915591,
      "learning_rate": 6.082287370157644e-07,
      "log_odds_chosen": 0.5863037109375,
      "log_odds_ratio": -0.570019543170929,
      "logits/chosen": -0.739550769329071,
      "logits/rejected": -0.5889648199081421,
      "logps/chosen": -0.6644531488418579,
      "logps/rejected": -1.019140601158142,
      "loss": 0.8521,
      "nll_loss": 0.8431640863418579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06639404594898224,
      "rewards/margins": 0.03546752780675888,
      "rewards/rejected": -0.10185547173023224,
      "step": 17300
    },
    {
      "epoch": 0.6486668790166945,
      "grad_norm": 2.4875457579629714,
      "learning_rate": 6.0805302453571e-07,
      "log_odds_chosen": 0.4066162109375,
      "log_odds_ratio": -0.6043945550918579,
      "logits/chosen": -0.742968738079071,
      "logits/rejected": -0.6324218511581421,
      "logps/chosen": -0.709667980670929,
      "logps/rejected": -0.961132824420929,
      "loss": 0.8511,
      "nll_loss": 0.811718761920929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07095947116613388,
      "rewards/margins": 0.025171661749482155,
      "rewards/rejected": -0.09614257514476776,
      "step": 17310
    },
    {
      "epoch": 0.6490416143598583,
      "grad_norm": 2.245576388548396,
      "learning_rate": 6.078774642535648e-07,
      "log_odds_chosen": 0.529980480670929,
      "log_odds_ratio": -0.561230480670929,
      "logits/chosen": -0.798632800579071,
      "logits/rejected": -0.6592773199081421,
      "logps/chosen": -0.6783202886581421,
      "logps/rejected": -0.9990234375,
      "loss": 0.8496,
      "nll_loss": 0.796093761920929,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06785888969898224,
      "rewards/margins": 0.03206024318933487,
      "rewards/rejected": -0.099853515625,
      "step": 17320
    },
    {
      "epoch": 0.6494163497030222,
      "grad_norm": 2.304771145018764,
      "learning_rate": 6.077020559497388e-07,
      "log_odds_chosen": 0.4766601622104645,
      "log_odds_ratio": -0.6109374761581421,
      "logits/chosen": -0.7544921636581421,
      "logits/rejected": -0.6611328125,
      "logps/chosen": -0.686328113079071,
      "logps/rejected": -0.9921875,
      "loss": 0.8552,
      "nll_loss": 0.804492175579071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06861571967601776,
      "rewards/margins": 0.03050689771771431,
      "rewards/rejected": -0.09908447414636612,
      "step": 17330
    },
    {
      "epoch": 0.6497910850461861,
      "grad_norm": 2.1629681716229023,
      "learning_rate": 6.07526799405085e-07,
      "log_odds_chosen": 0.4643188416957855,
      "log_odds_ratio": -0.613574206829071,
      "logits/chosen": -0.7919921875,
      "logits/rejected": -0.6830078363418579,
      "logps/chosen": -0.726757824420929,
      "logps/rejected": -1.023046851158142,
      "loss": 0.8622,
      "nll_loss": 0.8501952886581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07268066704273224,
      "rewards/margins": 0.029644012451171875,
      "rewards/rejected": -0.10239257663488388,
      "step": 17340
    },
    {
      "epoch": 0.65016582038935,
      "grad_norm": 1.8499408783559235,
      "learning_rate": 6.073516944008986e-07,
      "log_odds_chosen": 0.425537109375,
      "log_odds_ratio": -0.628125011920929,
      "logits/chosen": -0.750195324420929,
      "logits/rejected": -0.674609363079071,
      "logps/chosen": -0.682812511920929,
      "logps/rejected": -0.955859363079071,
      "loss": 0.8609,
      "nll_loss": 0.8482421636581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06837157905101776,
      "rewards/margins": 0.027265166863799095,
      "rewards/rejected": -0.09555663913488388,
      "step": 17350
    },
    {
      "epoch": 0.6505405557325139,
      "grad_norm": 2.3828080074139346,
      "learning_rate": 6.071767407189163e-07,
      "log_odds_chosen": 0.47211915254592896,
      "log_odds_ratio": -0.5931640863418579,
      "logits/chosen": -0.8003906011581421,
      "logits/rejected": -0.644335925579071,
      "logps/chosen": -0.681640625,
      "logps/rejected": -0.9765625,
      "loss": 0.8488,
      "nll_loss": 0.7779296636581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06813964992761612,
      "rewards/margins": 0.02952880784869194,
      "rewards/rejected": -0.09771728515625,
      "step": 17360
    },
    {
      "epoch": 0.6509152910756778,
      "grad_norm": 2.141612800141214,
      "learning_rate": 6.070019381413139e-07,
      "log_odds_chosen": 0.494873046875,
      "log_odds_ratio": -0.582812488079071,
      "logits/chosen": -0.7464843988418579,
      "logits/rejected": -0.6244140863418579,
      "logps/chosen": -0.659960925579071,
      "logps/rejected": -0.9769531488418579,
      "loss": 0.8534,
      "nll_loss": 0.8343750238418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06595458835363388,
      "rewards/margins": 0.03175201267004013,
      "rewards/rejected": -0.09766845405101776,
      "step": 17370
    },
    {
      "epoch": 0.6512900264188417,
      "grad_norm": 2.0977104489561227,
      "learning_rate": 6.068272864507064e-07,
      "log_odds_chosen": 0.5837157964706421,
      "log_odds_ratio": -0.5770508050918579,
      "logits/chosen": -0.718554675579071,
      "logits/rejected": -0.645312488079071,
      "logps/chosen": -0.677929699420929,
      "logps/rejected": -1.067968726158142,
      "loss": 0.8352,
      "nll_loss": 0.818164050579071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06777343899011612,
      "rewards/margins": 0.03907165676355362,
      "rewards/rejected": -0.10686035454273224,
      "step": 17380
    },
    {
      "epoch": 0.6516647617620056,
      "grad_norm": 2.4689330519921526,
      "learning_rate": 6.066527854301463e-07,
      "log_odds_chosen": 0.43339842557907104,
      "log_odds_ratio": -0.5982421636581421,
      "logits/chosen": -0.73095703125,
      "logits/rejected": -0.61083984375,
      "logps/chosen": -0.6424804925918579,
      "logps/rejected": -0.8958984613418579,
      "loss": 0.8387,
      "nll_loss": 0.772656261920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06425781548023224,
      "rewards/margins": 0.02529602125287056,
      "rewards/rejected": -0.08966064453125,
      "step": 17390
    },
    {
      "epoch": 0.6520394971051695,
      "grad_norm": 1.9535029549214924,
      "learning_rate": 6.064784348631227e-07,
      "log_odds_chosen": 0.4193115234375,
      "log_odds_ratio": -0.638476550579071,
      "logits/chosen": -0.759960949420929,
      "logits/rejected": -0.654980480670929,
      "logps/chosen": -0.671875,
      "logps/rejected": -0.976855456829071,
      "loss": 0.854,
      "nll_loss": 0.8265625238418579,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06716308742761612,
      "rewards/margins": 0.030516814440488815,
      "rewards/rejected": -0.09785155951976776,
      "step": 17400
    },
    {
      "epoch": 0.6524142324483334,
      "grad_norm": 2.1382680040506417,
      "learning_rate": 6.063042345335596e-07,
      "log_odds_chosen": 0.4848876893520355,
      "log_odds_ratio": -0.59716796875,
      "logits/chosen": -0.7191406488418579,
      "logits/rejected": -0.6363281011581421,
      "logps/chosen": -0.682812511920929,
      "logps/rejected": -0.959765613079071,
      "loss": 0.8516,
      "nll_loss": 0.765429675579071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06829833984375,
      "rewards/margins": 0.02772674523293972,
      "rewards/rejected": -0.09603271633386612,
      "step": 17410
    },
    {
      "epoch": 0.6527889677914973,
      "grad_norm": 2.1346872720661123,
      "learning_rate": 6.061301842258155e-07,
      "log_odds_chosen": 0.5405029058456421,
      "log_odds_ratio": -0.5824218988418579,
      "logits/chosen": -0.7357422113418579,
      "logits/rejected": -0.623339831829071,
      "logps/chosen": -0.6595703363418579,
      "logps/rejected": -1.0154297351837158,
      "loss": 0.8258,
      "nll_loss": 0.7406250238418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06596679985523224,
      "rewards/margins": 0.035533905029296875,
      "rewards/rejected": -0.10151366889476776,
      "step": 17420
    },
    {
      "epoch": 0.6531637031346611,
      "grad_norm": 2.231716168213613,
      "learning_rate": 6.059562837246821e-07,
      "log_odds_chosen": 0.561328113079071,
      "log_odds_ratio": -0.5682617425918579,
      "logits/chosen": -0.664355456829071,
      "logits/rejected": -0.60888671875,
      "logps/chosen": -0.594042956829071,
      "logps/rejected": -0.9517577886581421,
      "loss": 0.8647,
      "nll_loss": 0.7730468511581421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.05943603441119194,
      "rewards/margins": 0.03567199781537056,
      "rewards/rejected": -0.09508056938648224,
      "step": 17430
    },
    {
      "epoch": 0.653538438477825,
      "grad_norm": 1.8749361460867966,
      "learning_rate": 6.057825328153826e-07,
      "log_odds_chosen": 0.5667968988418579,
      "log_odds_ratio": -0.551464855670929,
      "logits/chosen": -0.717578113079071,
      "logits/rejected": -0.5746093988418579,
      "logps/chosen": -0.6572265625,
      "logps/rejected": -0.9964843988418579,
      "loss": 0.85,
      "nll_loss": 0.7769531011581421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06573486328125,
      "rewards/margins": 0.0338897705078125,
      "rewards/rejected": -0.09956054389476776,
      "step": 17440
    },
    {
      "epoch": 0.6539131738209889,
      "grad_norm": 2.299837315249304,
      "learning_rate": 6.056089312835716e-07,
      "log_odds_chosen": 0.5732177495956421,
      "log_odds_ratio": -0.572265625,
      "logits/chosen": -0.700390636920929,
      "logits/rejected": -0.5990234613418579,
      "logps/chosen": -0.683789074420929,
      "logps/rejected": -1.017968773841858,
      "loss": 0.8623,
      "nll_loss": 0.8521484136581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06840820610523224,
      "rewards/margins": 0.03346557542681694,
      "rewards/rejected": -0.10178222507238388,
      "step": 17450
    },
    {
      "epoch": 0.6542879091641528,
      "grad_norm": 2.4322615084587857,
      "learning_rate": 6.054354789153331e-07,
      "log_odds_chosen": 0.626879870891571,
      "log_odds_ratio": -0.558789074420929,
      "logits/chosen": -0.7362304925918579,
      "logits/rejected": -0.6119140386581421,
      "logps/chosen": -0.7240234613418579,
      "logps/rejected": -1.1376953125,
      "loss": 0.8466,
      "nll_loss": 0.8167968988418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07243652641773224,
      "rewards/margins": 0.04131164401769638,
      "rewards/rejected": -0.11374511569738388,
      "step": 17460
    },
    {
      "epoch": 0.6546626445073167,
      "grad_norm": 2.282952492249755,
      "learning_rate": 6.052621754971796e-07,
      "log_odds_chosen": 0.49425047636032104,
      "log_odds_ratio": -0.60986328125,
      "logits/chosen": -0.7421875,
      "logits/rejected": -0.6363281011581421,
      "logps/chosen": -0.7041015625,
      "logps/rejected": -1.03125,
      "loss": 0.833,
      "nll_loss": 0.7447265386581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07048340141773224,
      "rewards/margins": 0.03263092041015625,
      "rewards/rejected": -0.10307617485523224,
      "step": 17470
    },
    {
      "epoch": 0.6550373798504806,
      "grad_norm": 2.340469308747552,
      "learning_rate": 6.050890208160515e-07,
      "log_odds_chosen": 0.581402599811554,
      "log_odds_ratio": -0.575390636920929,
      "logits/chosen": -0.784960925579071,
      "logits/rejected": -0.66357421875,
      "logps/chosen": -0.6568359136581421,
      "logps/rejected": -1.037109375,
      "loss": 0.8418,
      "nll_loss": 0.7757812738418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06562499701976776,
      "rewards/margins": 0.03799591213464737,
      "rewards/rejected": -0.10356445610523224,
      "step": 17480
    },
    {
      "epoch": 0.6554121151936445,
      "grad_norm": 2.1000236779252877,
      "learning_rate": 6.049160146593155e-07,
      "log_odds_chosen": 0.551513671875,
      "log_odds_ratio": -0.588085949420929,
      "logits/chosen": -0.7408202886581421,
      "logits/rejected": -0.6207031011581421,
      "logps/chosen": -0.7059570550918579,
      "logps/rejected": -1.0695312023162842,
      "loss": 0.8761,
      "nll_loss": 0.818359375,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07058105617761612,
      "rewards/margins": 0.036408234387636185,
      "rewards/rejected": -0.10695800930261612,
      "step": 17490
    },
    {
      "epoch": 0.6557868505368084,
      "grad_norm": 2.138877805448851,
      "learning_rate": 6.047431568147635e-07,
      "log_odds_chosen": 0.456787109375,
      "log_odds_ratio": -0.61767578125,
      "logits/chosen": -0.73193359375,
      "logits/rejected": -0.6031249761581421,
      "logps/chosen": -0.6522461175918579,
      "logps/rejected": -0.9574218988418579,
      "loss": 0.8634,
      "nll_loss": 0.8160156011581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06523437798023224,
      "rewards/margins": 0.03042297437787056,
      "rewards/rejected": -0.09560547024011612,
      "step": 17500
    },
    {
      "epoch": 0.6561615858799723,
      "grad_norm": 2.6313769222829606,
      "learning_rate": 6.045704470706117e-07,
      "log_odds_chosen": 0.4776367247104645,
      "log_odds_ratio": -0.596875011920929,
      "logits/chosen": -0.716015636920929,
      "logits/rejected": -0.648242175579071,
      "logps/chosen": -0.6714843511581421,
      "logps/rejected": -0.9876953363418579,
      "loss": 0.8503,
      "nll_loss": 0.738085925579071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06722412258386612,
      "rewards/margins": 0.031580351293087006,
      "rewards/rejected": -0.09879150241613388,
      "step": 17510
    },
    {
      "epoch": 0.6565363212231362,
      "grad_norm": 2.1395099490354714,
      "learning_rate": 6.043978852154994e-07,
      "log_odds_chosen": 0.6026855707168579,
      "log_odds_ratio": -0.5511718988418579,
      "logits/chosen": -0.7445312738418579,
      "logits/rejected": -0.64892578125,
      "logps/chosen": -0.623828113079071,
      "logps/rejected": -1.0021483898162842,
      "loss": 0.8523,
      "nll_loss": 0.787109375,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06235351413488388,
      "rewards/margins": 0.03780822828412056,
      "rewards/rejected": -0.10029296576976776,
      "step": 17520
    },
    {
      "epoch": 0.6569110565663,
      "grad_norm": 2.814529676343496,
      "learning_rate": 6.04225471038488e-07,
      "log_odds_chosen": 0.4065918028354645,
      "log_odds_ratio": -0.6133788824081421,
      "logits/chosen": -0.821093738079071,
      "logits/rejected": -0.71044921875,
      "logps/chosen": -0.7060546875,
      "logps/rejected": -0.967968761920929,
      "loss": 0.8631,
      "nll_loss": 0.7994140386581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07061767578125,
      "rewards/margins": 0.02611999586224556,
      "rewards/rejected": -0.0968017578125,
      "step": 17530
    },
    {
      "epoch": 0.6572857919094639,
      "grad_norm": 2.186574188462889,
      "learning_rate": 6.040532043290601e-07,
      "log_odds_chosen": 0.4897216856479645,
      "log_odds_ratio": -0.605761706829071,
      "logits/chosen": -0.8125,
      "logits/rejected": -0.676953136920929,
      "logps/chosen": -0.661328136920929,
      "logps/rejected": -0.999804675579071,
      "loss": 0.8592,
      "nll_loss": 0.824414074420929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06618652492761612,
      "rewards/margins": 0.03373413160443306,
      "rewards/rejected": -0.09990234673023224,
      "step": 17540
    },
    {
      "epoch": 0.6576605272526278,
      "grad_norm": 2.2060358067036185,
      "learning_rate": 6.038810848771178e-07,
      "log_odds_chosen": 0.5455566644668579,
      "log_odds_ratio": -0.588574230670929,
      "logits/chosen": -0.7359374761581421,
      "logits/rejected": -0.63037109375,
      "logps/chosen": -0.659375011920929,
      "logps/rejected": -0.9947265386581421,
      "loss": 0.8499,
      "nll_loss": 0.743359386920929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06594238430261612,
      "rewards/margins": 0.03347625583410263,
      "rewards/rejected": -0.09949950873851776,
      "step": 17550
    },
    {
      "epoch": 0.6580352625957917,
      "grad_norm": 1.890226946878349,
      "learning_rate": 6.037091124729821e-07,
      "log_odds_chosen": 0.6336425542831421,
      "log_odds_ratio": -0.5628906488418579,
      "logits/chosen": -0.747265636920929,
      "logits/rejected": -0.6622070074081421,
      "logps/chosen": -0.681640625,
      "logps/rejected": -1.067773461341858,
      "loss": 0.858,
      "nll_loss": 0.798828125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06821288913488388,
      "rewards/margins": 0.03854980319738388,
      "rewards/rejected": -0.10683593899011612,
      "step": 17560
    },
    {
      "epoch": 0.6584099979389556,
      "grad_norm": 2.441639901828192,
      "learning_rate": 6.03537286907392e-07,
      "log_odds_chosen": 0.4679931700229645,
      "log_odds_ratio": -0.5962890386581421,
      "logits/chosen": -0.7247070074081421,
      "logits/rejected": -0.6205078363418579,
      "logps/chosen": -0.636425793170929,
      "logps/rejected": -0.9271484613418579,
      "loss": 0.8418,
      "nll_loss": 0.7466796636581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06354980170726776,
      "rewards/margins": 0.02915802039206028,
      "rewards/rejected": -0.09268798679113388,
      "step": 17570
    },
    {
      "epoch": 0.6587847332821195,
      "grad_norm": 2.378270228742851,
      "learning_rate": 6.033656079715029e-07,
      "log_odds_chosen": 0.601806640625,
      "log_odds_ratio": -0.58349609375,
      "logits/chosen": -0.7904297113418579,
      "logits/rejected": -0.677734375,
      "logps/chosen": -0.693359375,
      "logps/rejected": -1.100976586341858,
      "loss": 0.86,
      "nll_loss": 0.8255859613418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06929931789636612,
      "rewards/margins": 0.0406951904296875,
      "rewards/rejected": -0.1099853515625,
      "step": 17580
    },
    {
      "epoch": 0.6591594686252834,
      "grad_norm": 2.116257359337494,
      "learning_rate": 6.031940754568862e-07,
      "log_odds_chosen": 0.5650390386581421,
      "log_odds_ratio": -0.57177734375,
      "logits/chosen": -0.729296863079071,
      "logits/rejected": -0.6431640386581421,
      "logps/chosen": -0.71044921875,
      "logps/rejected": -1.0744140148162842,
      "loss": 0.8612,
      "nll_loss": 0.830273449420929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07102050632238388,
      "rewards/margins": 0.03635406494140625,
      "rewards/rejected": -0.10748291015625,
      "step": 17590
    },
    {
      "epoch": 0.6595342039684473,
      "grad_norm": 2.0593858236623253,
      "learning_rate": 6.030226891555272e-07,
      "log_odds_chosen": 0.6539306640625,
      "log_odds_ratio": -0.558154284954071,
      "logits/chosen": -0.6695312261581421,
      "logits/rejected": -0.607714831829071,
      "logps/chosen": -0.7203124761581421,
      "logps/rejected": -1.1572265625,
      "loss": 0.8687,
      "nll_loss": 0.8441406488418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07200928032398224,
      "rewards/margins": 0.04372100904583931,
      "rewards/rejected": -0.11575927585363388,
      "step": 17600
    },
    {
      "epoch": 0.6599089393116112,
      "grad_norm": 2.3744834980467604,
      "learning_rate": 6.028514488598253e-07,
      "log_odds_chosen": 0.4997802674770355,
      "log_odds_ratio": -0.632128894329071,
      "logits/chosen": -0.8251953125,
      "logits/rejected": -0.6744140386581421,
      "logps/chosen": -0.714062511920929,
      "logps/rejected": -1.0537109375,
      "loss": 0.8603,
      "nll_loss": 0.822070300579071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07145996391773224,
      "rewards/margins": 0.03409118577837944,
      "rewards/rejected": -0.10551758110523224,
      "step": 17610
    },
    {
      "epoch": 0.6602836746547751,
      "grad_norm": 2.6791057948455492,
      "learning_rate": 6.026803543625922e-07,
      "log_odds_chosen": 0.5094360113143921,
      "log_odds_ratio": -0.612011730670929,
      "logits/chosen": -0.7681640386581421,
      "logits/rejected": -0.686718761920929,
      "logps/chosen": -0.724316418170929,
      "logps/rejected": -1.0583984851837158,
      "loss": 0.8658,
      "nll_loss": 0.800585925579071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07252196967601776,
      "rewards/margins": 0.033355712890625,
      "rewards/rejected": -0.10585937649011612,
      "step": 17620
    },
    {
      "epoch": 0.6606584099979389,
      "grad_norm": 2.129901723880823,
      "learning_rate": 6.025094054570507e-07,
      "log_odds_chosen": 0.535473644733429,
      "log_odds_ratio": -0.5650390386581421,
      "logits/chosen": -0.726757824420929,
      "logits/rejected": -0.6280273199081421,
      "logps/chosen": -0.666308581829071,
      "logps/rejected": -0.984570324420929,
      "loss": 0.8416,
      "nll_loss": 0.746874988079071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06663818657398224,
      "rewards/margins": 0.03182373195886612,
      "rewards/rejected": -0.098388671875,
      "step": 17630
    },
    {
      "epoch": 0.6610331453411028,
      "grad_norm": 2.0137383108967715,
      "learning_rate": 6.023386019368341e-07,
      "log_odds_chosen": 0.44245606660842896,
      "log_odds_ratio": -0.628710925579071,
      "logits/chosen": -0.774218738079071,
      "logits/rejected": -0.662792980670929,
      "logps/chosen": -0.7412109375,
      "logps/rejected": -1.0244140625,
      "loss": 0.8391,
      "nll_loss": 0.8451172113418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.0740966796875,
      "rewards/margins": 0.02833862230181694,
      "rewards/rejected": -0.1024169921875,
      "step": 17640
    },
    {
      "epoch": 0.6614078806842667,
      "grad_norm": 2.3658676995187387,
      "learning_rate": 6.021679435959851e-07,
      "log_odds_chosen": 0.5511230230331421,
      "log_odds_ratio": -0.592089831829071,
      "logits/chosen": -0.7435547113418579,
      "logits/rejected": -0.6541992425918579,
      "logps/chosen": -0.6767578125,
      "logps/rejected": -1.0007812976837158,
      "loss": 0.8429,
      "nll_loss": 0.855664074420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06776122748851776,
      "rewards/margins": 0.03244934231042862,
      "rewards/rejected": -0.10012207180261612,
      "step": 17650
    },
    {
      "epoch": 0.6617826160274306,
      "grad_norm": 1.9215053703430731,
      "learning_rate": 6.019974302289545e-07,
      "log_odds_chosen": 0.42412108182907104,
      "log_odds_ratio": -0.6231445074081421,
      "logits/chosen": -0.7837890386581421,
      "logits/rejected": -0.687207043170929,
      "logps/chosen": -0.6810547113418579,
      "logps/rejected": -0.953125,
      "loss": 0.8491,
      "nll_loss": 0.8026367425918579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06807861477136612,
      "rewards/margins": 0.027167510241270065,
      "rewards/rejected": -0.09528808295726776,
      "step": 17660
    },
    {
      "epoch": 0.6621573513705945,
      "grad_norm": 2.427295118713614,
      "learning_rate": 6.018270616306004e-07,
      "log_odds_chosen": 0.54443359375,
      "log_odds_ratio": -0.5907226800918579,
      "logits/chosen": -0.7777343988418579,
      "logits/rejected": -0.6504882574081421,
      "logps/chosen": -0.654589831829071,
      "logps/rejected": -1.0164062976837158,
      "loss": 0.8434,
      "nll_loss": 0.7953125238418579,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.06545410305261612,
      "rewards/margins": 0.03617553785443306,
      "rewards/rejected": -0.10161133110523224,
      "step": 17670
    },
    {
      "epoch": 0.6625320867137584,
      "grad_norm": 2.187908310675271,
      "learning_rate": 6.016568375961868e-07,
      "log_odds_chosen": 0.43958741426467896,
      "log_odds_ratio": -0.638378918170929,
      "logits/chosen": -0.7022460699081421,
      "logits/rejected": -0.657421886920929,
      "logps/chosen": -0.645214855670929,
      "logps/rejected": -0.9234374761581421,
      "loss": 0.8523,
      "nll_loss": 0.7445312738418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06456299126148224,
      "rewards/margins": 0.02782287634909153,
      "rewards/rejected": -0.09234619140625,
      "step": 17680
    },
    {
      "epoch": 0.6629068220569223,
      "grad_norm": 2.101263848299474,
      "learning_rate": 6.014867579213833e-07,
      "log_odds_chosen": 0.37139892578125,
      "log_odds_ratio": -0.647265613079071,
      "logits/chosen": -0.7203124761581421,
      "logits/rejected": -0.63671875,
      "logps/chosen": -0.6783202886581421,
      "logps/rejected": -0.9413086175918579,
      "loss": 0.8653,
      "nll_loss": 0.7955077886581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06783447414636612,
      "rewards/margins": 0.02634277381002903,
      "rewards/rejected": -0.09410400688648224,
      "step": 17690
    },
    {
      "epoch": 0.6632815574000862,
      "grad_norm": 2.2265249886134595,
      "learning_rate": 6.013168224022631e-07,
      "log_odds_chosen": 0.5283447504043579,
      "log_odds_ratio": -0.577441394329071,
      "logits/chosen": -0.7818359136581421,
      "logits/rejected": -0.6654297113418579,
      "logps/chosen": -0.660351574420929,
      "logps/rejected": -1.001074194908142,
      "loss": 0.8475,
      "nll_loss": 0.7837890386581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06596679985523224,
      "rewards/margins": 0.03407173231244087,
      "rewards/rejected": -0.10007324069738388,
      "step": 17700
    },
    {
      "epoch": 0.6636562927432501,
      "grad_norm": 2.0845838639600416,
      "learning_rate": 6.011470308353028e-07,
      "log_odds_chosen": 0.5171264410018921,
      "log_odds_ratio": -0.5986328125,
      "logits/chosen": -0.7337890863418579,
      "logits/rejected": -0.60791015625,
      "logps/chosen": -0.71484375,
      "logps/rejected": -1.0597655773162842,
      "loss": 0.8597,
      "nll_loss": 0.835742175579071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07144775241613388,
      "rewards/margins": 0.034520722925662994,
      "rewards/rejected": -0.1060791015625,
      "step": 17710
    },
    {
      "epoch": 0.664031028086414,
      "grad_norm": 2.305784918872104,
      "learning_rate": 6.00977383017381e-07,
      "log_odds_chosen": 0.4330078065395355,
      "log_odds_ratio": -0.6109374761581421,
      "logits/chosen": -0.766796886920929,
      "logits/rejected": -0.635058581829071,
      "logps/chosen": -0.679882824420929,
      "logps/rejected": -0.961718738079071,
      "loss": 0.8518,
      "nll_loss": 0.7880859375,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06794433295726776,
      "rewards/margins": 0.028159331530332565,
      "rewards/rejected": -0.09613037109375,
      "step": 17720
    },
    {
      "epoch": 0.6644057634295779,
      "grad_norm": 2.267522996223134,
      "learning_rate": 6.008078787457772e-07,
      "log_odds_chosen": 0.558911144733429,
      "log_odds_ratio": -0.5845702886581421,
      "logits/chosen": -0.7369140386581421,
      "logits/rejected": -0.6366211175918579,
      "logps/chosen": -0.688183605670929,
      "logps/rejected": -1.0476562976837158,
      "loss": 0.8443,
      "nll_loss": 0.819140613079071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06878662109375,
      "rewards/margins": 0.036104582250118256,
      "rewards/rejected": -0.10480956733226776,
      "step": 17730
    },
    {
      "epoch": 0.6647804987727417,
      "grad_norm": 2.2234524522060264,
      "learning_rate": 6.006385178181711e-07,
      "log_odds_chosen": 0.50335693359375,
      "log_odds_ratio": -0.603710949420929,
      "logits/chosen": -0.7447265386581421,
      "logits/rejected": -0.66259765625,
      "logps/chosen": -0.6693359613418579,
      "logps/rejected": -1.0167968273162842,
      "loss": 0.856,
      "nll_loss": 0.836718738079071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06697998195886612,
      "rewards/margins": 0.034734345972537994,
      "rewards/rejected": -0.10178222507238388,
      "step": 17740
    },
    {
      "epoch": 0.6651552341159056,
      "grad_norm": 2.1900593058600712,
      "learning_rate": 6.004693000326412e-07,
      "log_odds_chosen": 0.5011841058731079,
      "log_odds_ratio": -0.6148437261581421,
      "logits/chosen": -0.7464843988418579,
      "logits/rejected": -0.653515636920929,
      "logps/chosen": -0.71875,
      "logps/rejected": -1.0623047351837158,
      "loss": 0.8617,
      "nll_loss": 0.794140636920929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.0718994140625,
      "rewards/margins": 0.0343170166015625,
      "rewards/rejected": -0.106201171875,
      "step": 17750
    },
    {
      "epoch": 0.6655299694590695,
      "grad_norm": 2.3294084723260315,
      "learning_rate": 6.003002251876642e-07,
      "log_odds_chosen": 0.5483764410018921,
      "log_odds_ratio": -0.577832043170929,
      "logits/chosen": -0.732226550579071,
      "logits/rejected": -0.6524413824081421,
      "logps/chosen": -0.644238293170929,
      "logps/rejected": -0.961132824420929,
      "loss": 0.8535,
      "nll_loss": 0.773242175579071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06442870944738388,
      "rewards/margins": 0.03168334811925888,
      "rewards/rejected": -0.09604492038488388,
      "step": 17760
    },
    {
      "epoch": 0.6659047048022334,
      "grad_norm": 2.5689656835359975,
      "learning_rate": 6.001312930821136e-07,
      "log_odds_chosen": 0.509533703327179,
      "log_odds_ratio": -0.584667980670929,
      "logits/chosen": -0.7431640625,
      "logits/rejected": -0.671093761920929,
      "logps/chosen": -0.6441406011581421,
      "logps/rejected": -0.9564453363418579,
      "loss": 0.8733,
      "nll_loss": 0.828125,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.064453125,
      "rewards/margins": 0.03118438646197319,
      "rewards/rejected": -0.09562988579273224,
      "step": 17770
    },
    {
      "epoch": 0.6662794401453973,
      "grad_norm": 2.1553966074437336,
      "learning_rate": 5.999625035152588e-07,
      "log_odds_chosen": 0.64794921875,
      "log_odds_ratio": -0.5716797113418579,
      "logits/chosen": -0.7549804449081421,
      "logits/rejected": -0.6304687261581421,
      "logps/chosen": -0.661328136920929,
      "logps/rejected": -1.096093773841858,
      "loss": 0.8584,
      "nll_loss": 0.78515625,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06614990532398224,
      "rewards/margins": 0.04338989406824112,
      "rewards/rejected": -0.10966797173023224,
      "step": 17780
    },
    {
      "epoch": 0.6666541754885612,
      "grad_norm": 2.1281427475588117,
      "learning_rate": 5.997938562867645e-07,
      "log_odds_chosen": 0.573974609375,
      "log_odds_ratio": -0.5584961175918579,
      "logits/chosen": -0.781054675579071,
      "logits/rejected": -0.6612304449081421,
      "logps/chosen": -0.6336914300918579,
      "logps/rejected": -0.98974609375,
      "loss": 0.8409,
      "nll_loss": 0.783007800579071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06333007663488388,
      "rewards/margins": 0.035605620592832565,
      "rewards/rejected": -0.09895019233226776,
      "step": 17790
    },
    {
      "epoch": 0.6670289108317251,
      "grad_norm": 2.351077797106513,
      "learning_rate": 5.996253511966891e-07,
      "log_odds_chosen": 0.4430175721645355,
      "log_odds_ratio": -0.6131836175918579,
      "logits/chosen": -0.71875,
      "logits/rejected": -0.6514648199081421,
      "logps/chosen": -0.6830078363418579,
      "logps/rejected": -0.9583984613418579,
      "loss": 0.8298,
      "nll_loss": 0.8203125,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06833495944738388,
      "rewards/margins": 0.027509307488799095,
      "rewards/rejected": -0.09583739936351776,
      "step": 17800
    },
    {
      "epoch": 0.667403646174889,
      "grad_norm": 2.032860130474191,
      "learning_rate": 5.994569880454842e-07,
      "log_odds_chosen": 0.49858397245407104,
      "log_odds_ratio": -0.605761706829071,
      "logits/chosen": -0.767773449420929,
      "logits/rejected": -0.6478515863418579,
      "logps/chosen": -0.6446288824081421,
      "logps/rejected": -0.9642578363418579,
      "loss": 0.822,
      "nll_loss": 0.730664074420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06444092094898224,
      "rewards/margins": 0.03203582763671875,
      "rewards/rejected": -0.09645996242761612,
      "step": 17810
    },
    {
      "epoch": 0.6677783815180529,
      "grad_norm": 2.0019226603485363,
      "learning_rate": 5.99288766633993e-07,
      "log_odds_chosen": 0.606579601764679,
      "log_odds_ratio": -0.5555664300918579,
      "logits/chosen": -0.7548828125,
      "logits/rejected": -0.669140636920929,
      "logps/chosen": -0.67529296875,
      "logps/rejected": -1.051367163658142,
      "loss": 0.8374,
      "nll_loss": 0.7822265625,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0675048828125,
      "rewards/margins": 0.03764190524816513,
      "rewards/rejected": -0.10509033501148224,
      "step": 17820
    },
    {
      "epoch": 0.6681531168612168,
      "grad_norm": 2.2018269393543175,
      "learning_rate": 5.991206867634499e-07,
      "log_odds_chosen": 0.534863293170929,
      "log_odds_ratio": -0.583203136920929,
      "logits/chosen": -0.7542968988418579,
      "logits/rejected": -0.6234375238418579,
      "logps/chosen": -0.667773425579071,
      "logps/rejected": -0.9931640625,
      "loss": 0.854,
      "nll_loss": 0.7943359613418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0667724609375,
      "rewards/margins": 0.03258056566119194,
      "rewards/rejected": -0.099365234375,
      "step": 17830
    },
    {
      "epoch": 0.6685278522043806,
      "grad_norm": 1.9929844275943303,
      "learning_rate": 5.989527482354798e-07,
      "log_odds_chosen": 0.5048828125,
      "log_odds_ratio": -0.619140625,
      "logits/chosen": -0.753125011920929,
      "logits/rejected": -0.669628918170929,
      "logps/chosen": -0.674609363079071,
      "logps/rejected": -1.0294921398162842,
      "loss": 0.846,
      "nll_loss": 0.7900390625,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06743164360523224,
      "rewards/margins": 0.03552856296300888,
      "rewards/rejected": -0.10297851264476776,
      "step": 17840
    },
    {
      "epoch": 0.6689025875475445,
      "grad_norm": 2.2518171303768457,
      "learning_rate": 5.987849508520958e-07,
      "log_odds_chosen": 0.6131347417831421,
      "log_odds_ratio": -0.553906261920929,
      "logits/chosen": -0.718554675579071,
      "logits/rejected": -0.6273437738418579,
      "logps/chosen": -0.6490234136581421,
      "logps/rejected": -1.003320336341858,
      "loss": 0.8498,
      "nll_loss": 0.8179687261581421,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06492920219898224,
      "rewards/margins": 0.03543548658490181,
      "rewards/rejected": -0.10036621242761612,
      "step": 17850
    },
    {
      "epoch": 0.6692773228907084,
      "grad_norm": 2.3384022260317545,
      "learning_rate": 5.986172944156994e-07,
      "log_odds_chosen": 0.528247058391571,
      "log_odds_ratio": -0.599316418170929,
      "logits/chosen": -0.764453113079071,
      "logits/rejected": -0.6993163824081421,
      "logps/chosen": -0.60546875,
      "logps/rejected": -0.9664062261581421,
      "loss": 0.8437,
      "nll_loss": 0.755078136920929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06055908277630806,
      "rewards/margins": 0.036089323461055756,
      "rewards/rejected": -0.09665527194738388,
      "step": 17860
    },
    {
      "epoch": 0.6696520582338723,
      "grad_norm": 2.182724638738855,
      "learning_rate": 5.984497787290794e-07,
      "log_odds_chosen": 0.621899425983429,
      "log_odds_ratio": -0.57470703125,
      "logits/chosen": -0.7503906488418579,
      "logits/rejected": -0.6513671875,
      "logps/chosen": -0.6278320550918579,
      "logps/rejected": -1.037500023841858,
      "loss": 0.8374,
      "nll_loss": 0.799023449420929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.062744140625,
      "rewards/margins": 0.040937043726444244,
      "rewards/rejected": -0.10371093451976776,
      "step": 17870
    },
    {
      "epoch": 0.6700267935770362,
      "grad_norm": 2.3692888742582623,
      "learning_rate": 5.982824035954103e-07,
      "log_odds_chosen": 0.627978503704071,
      "log_odds_ratio": -0.551464855670929,
      "logits/chosen": -0.722363293170929,
      "logits/rejected": -0.6170898675918579,
      "logps/chosen": -0.6884765625,
      "logps/rejected": -1.083984375,
      "loss": 0.8558,
      "nll_loss": 0.7626953125,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06889648735523224,
      "rewards/margins": 0.03947143629193306,
      "rewards/rejected": -0.10832519829273224,
      "step": 17880
    },
    {
      "epoch": 0.6704015289202001,
      "grad_norm": 2.3265474680125027,
      "learning_rate": 5.98115168818252e-07,
      "log_odds_chosen": 0.40980225801467896,
      "log_odds_ratio": -0.616992175579071,
      "logits/chosen": -0.728320300579071,
      "logits/rejected": -0.63720703125,
      "logps/chosen": -0.7099609375,
      "logps/rejected": -0.984179675579071,
      "loss": 0.8383,
      "nll_loss": 0.7925781011581421,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07099609076976776,
      "rewards/margins": 0.027422333136200905,
      "rewards/rejected": -0.09843750298023224,
      "step": 17890
    },
    {
      "epoch": 0.670776264263364,
      "grad_norm": 2.2512708812954396,
      "learning_rate": 5.979480742015487e-07,
      "log_odds_chosen": 0.48621827363967896,
      "log_odds_ratio": -0.5926758050918579,
      "logits/chosen": -0.771679699420929,
      "logits/rejected": -0.6719726324081421,
      "logps/chosen": -0.690234363079071,
      "logps/rejected": -1.0126953125,
      "loss": 0.8548,
      "nll_loss": 0.765820324420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06905517727136612,
      "rewards/margins": 0.03228912502527237,
      "rewards/rejected": -0.101318359375,
      "step": 17900
    },
    {
      "epoch": 0.6711509996065279,
      "grad_norm": 2.3757427233835164,
      "learning_rate": 5.977811195496272e-07,
      "log_odds_chosen": 0.49799805879592896,
      "log_odds_ratio": -0.583984375,
      "logits/chosen": -0.787304699420929,
      "logits/rejected": -0.6810547113418579,
      "logps/chosen": -0.6566406488418579,
      "logps/rejected": -0.981249988079071,
      "loss": 0.8761,
      "nll_loss": 0.8017578125,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06560058891773224,
      "rewards/margins": 0.03240814059972763,
      "rewards/rejected": -0.09812011569738388,
      "step": 17910
    },
    {
      "epoch": 0.6715257349496918,
      "grad_norm": 2.0448740464266044,
      "learning_rate": 5.976143046671967e-07,
      "log_odds_chosen": 0.47889405488967896,
      "log_odds_ratio": -0.5918945074081421,
      "logits/chosen": -0.775390625,
      "logits/rejected": -0.6714843511581421,
      "logps/chosen": -0.655468761920929,
      "logps/rejected": -0.9683593511581421,
      "loss": 0.8642,
      "nll_loss": 0.8056640625,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06549072265625,
      "rewards/margins": 0.03133239597082138,
      "rewards/rejected": -0.09685058891773224,
      "step": 17920
    },
    {
      "epoch": 0.6719004702928557,
      "grad_norm": 2.509463000616497,
      "learning_rate": 5.974476293593484e-07,
      "log_odds_chosen": 0.6220703125,
      "log_odds_ratio": -0.5386718511581421,
      "logits/chosen": -0.729296863079071,
      "logits/rejected": -0.6041015386581421,
      "logps/chosen": -0.676953136920929,
      "logps/rejected": -1.0822265148162842,
      "loss": 0.8562,
      "nll_loss": 0.767578125,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06768798828125,
      "rewards/margins": 0.04051513597369194,
      "rewards/rejected": -0.10830078274011612,
      "step": 17930
    },
    {
      "epoch": 0.6722752056360196,
      "grad_norm": 2.106931705670021,
      "learning_rate": 5.97281093431553e-07,
      "log_odds_chosen": 0.5127929449081421,
      "log_odds_ratio": -0.564990222454071,
      "logits/chosen": -0.750781238079071,
      "logits/rejected": -0.6265624761581421,
      "logps/chosen": -0.6431640386581421,
      "logps/rejected": -0.9676758050918579,
      "loss": 0.8277,
      "nll_loss": 0.745800793170929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06430663913488388,
      "rewards/margins": 0.03244171291589737,
      "rewards/rejected": -0.09674072265625,
      "step": 17940
    },
    {
      "epoch": 0.6726499409791834,
      "grad_norm": 2.2735389918894655,
      "learning_rate": 5.971146966896607e-07,
      "log_odds_chosen": 0.530322253704071,
      "log_odds_ratio": -0.563281238079071,
      "logits/chosen": -0.78466796875,
      "logits/rejected": -0.682324230670929,
      "logps/chosen": -0.6509765386581421,
      "logps/rejected": -0.9820312261581421,
      "loss": 0.8626,
      "nll_loss": 0.7505859136581421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06511230766773224,
      "rewards/margins": 0.03309326246380806,
      "rewards/rejected": -0.09814453125,
      "step": 17950
    },
    {
      "epoch": 0.6730246763223473,
      "grad_norm": 2.0663673033445886,
      "learning_rate": 5.969484389399003e-07,
      "log_odds_chosen": 0.649182140827179,
      "log_odds_ratio": -0.5438476800918579,
      "logits/chosen": -0.7421875,
      "logits/rejected": -0.629589855670929,
      "logps/chosen": -0.6664062738418579,
      "logps/rejected": -1.07421875,
      "loss": 0.8516,
      "nll_loss": 0.8173828125,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.066650390625,
      "rewards/margins": 0.04089965671300888,
      "rewards/rejected": -0.10758056491613388,
      "step": 17960
    },
    {
      "epoch": 0.6733994116655112,
      "grad_norm": 2.0875904966388195,
      "learning_rate": 5.967823199888781e-07,
      "log_odds_chosen": 0.508618175983429,
      "log_odds_ratio": -0.5874999761581421,
      "logits/chosen": -0.785937488079071,
      "logits/rejected": -0.681835949420929,
      "logps/chosen": -0.68359375,
      "logps/rejected": -0.9873046875,
      "loss": 0.8468,
      "nll_loss": 0.797070324420929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06843261420726776,
      "rewards/margins": 0.030336761847138405,
      "rewards/rejected": -0.09873046725988388,
      "step": 17970
    },
    {
      "epoch": 0.6737741470086751,
      "grad_norm": 2.409155580246329,
      "learning_rate": 5.966163396435766e-07,
      "log_odds_chosen": 0.692138671875,
      "log_odds_ratio": -0.5423828363418579,
      "logits/chosen": -0.751171886920929,
      "logits/rejected": -0.6407226324081421,
      "logps/chosen": -0.662109375,
      "logps/rejected": -1.089453101158142,
      "loss": 0.8481,
      "nll_loss": 0.805468738079071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06619872897863388,
      "rewards/margins": 0.04277191311120987,
      "rewards/rejected": -0.10905762016773224,
      "step": 17980
    },
    {
      "epoch": 0.674148882351839,
      "grad_norm": 1.915081866155863,
      "learning_rate": 5.964504977113543e-07,
      "log_odds_chosen": 0.622729480266571,
      "log_odds_ratio": -0.561230480670929,
      "logits/chosen": -0.718945324420929,
      "logits/rejected": -0.603320300579071,
      "logps/chosen": -0.6576172113418579,
      "logps/rejected": -1.068750023841858,
      "loss": 0.8393,
      "nll_loss": 0.8154296875,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06575927883386612,
      "rewards/margins": 0.04103851318359375,
      "rewards/rejected": -0.10683593899011612,
      "step": 17990
    },
    {
      "epoch": 0.6745236176950029,
      "grad_norm": 2.5732378769489204,
      "learning_rate": 5.962847939999439e-07,
      "log_odds_chosen": 0.4985107481479645,
      "log_odds_ratio": -0.5830078125,
      "logits/chosen": -0.783007800579071,
      "logits/rejected": -0.630175769329071,
      "logps/chosen": -0.626953125,
      "logps/rejected": -0.923046886920929,
      "loss": 0.8441,
      "nll_loss": 0.71435546875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06269530951976776,
      "rewards/margins": 0.029567718505859375,
      "rewards/rejected": -0.09233398735523224,
      "step": 18000
    },
    {
      "epoch": 0.6748983530381668,
      "grad_norm": 2.22454488405295,
      "learning_rate": 5.961192283174521e-07,
      "log_odds_chosen": 0.3572631776332855,
      "log_odds_ratio": -0.6524902582168579,
      "logits/chosen": -0.7474609613418579,
      "logits/rejected": -0.657031238079071,
      "logps/chosen": -0.66552734375,
      "logps/rejected": -0.90625,
      "loss": 0.8414,
      "nll_loss": 0.737109363079071,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.06657715141773224,
      "rewards/margins": 0.024016570299863815,
      "rewards/rejected": -0.09056396782398224,
      "step": 18010
    },
    {
      "epoch": 0.6752730883813307,
      "grad_norm": 2.7029467046114055,
      "learning_rate": 5.959538004723581e-07,
      "log_odds_chosen": 0.4507812559604645,
      "log_odds_ratio": -0.627246081829071,
      "logits/chosen": -0.7138671875,
      "logits/rejected": -0.617382824420929,
      "logps/chosen": -0.6981445550918579,
      "logps/rejected": -0.9912109375,
      "loss": 0.8656,
      "nll_loss": 0.7964843511581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06979980319738388,
      "rewards/margins": 0.02926940843462944,
      "rewards/rejected": -0.09914550930261612,
      "step": 18020
    },
    {
      "epoch": 0.6756478237244946,
      "grad_norm": 2.347064915648583,
      "learning_rate": 5.957885102735133e-07,
      "log_odds_chosen": 0.594958484172821,
      "log_odds_ratio": -0.5748046636581421,
      "logits/chosen": -0.762890636920929,
      "logits/rejected": -0.654492199420929,
      "logps/chosen": -0.6693359613418579,
      "logps/rejected": -1.020898461341858,
      "loss": 0.8289,
      "nll_loss": 0.7720702886581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06693115085363388,
      "rewards/margins": 0.035164643079042435,
      "rewards/rejected": -0.1021728515625,
      "step": 18030
    },
    {
      "epoch": 0.6760225590676585,
      "grad_norm": 2.2435868238735526,
      "learning_rate": 5.956233575301397e-07,
      "log_odds_chosen": 0.550854504108429,
      "log_odds_ratio": -0.586718738079071,
      "logits/chosen": -0.75390625,
      "logits/rejected": -0.658007800579071,
      "logps/chosen": -0.631542980670929,
      "logps/rejected": -0.982226550579071,
      "loss": 0.8497,
      "nll_loss": 0.7632812261581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06317138671875,
      "rewards/margins": 0.03507842868566513,
      "rewards/rejected": -0.09821777045726776,
      "step": 18040
    },
    {
      "epoch": 0.6763972944108223,
      "grad_norm": 2.6429652425523806,
      "learning_rate": 5.954583420518295e-07,
      "log_odds_chosen": 0.4182372987270355,
      "log_odds_ratio": -0.62890625,
      "logits/chosen": -0.715039074420929,
      "logits/rejected": -0.632128894329071,
      "logps/chosen": -0.658984363079071,
      "logps/rejected": -0.943164050579071,
      "loss": 0.8679,
      "nll_loss": 0.7431640625,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06594238430261612,
      "rewards/margins": 0.02833404578268528,
      "rewards/rejected": -0.0943603515625,
      "step": 18050
    },
    {
      "epoch": 0.6767720297539862,
      "grad_norm": 2.0948527275278486,
      "learning_rate": 5.952934636485436e-07,
      "log_odds_chosen": 0.553784191608429,
      "log_odds_ratio": -0.593457043170929,
      "logits/chosen": -0.761914074420929,
      "logits/rejected": -0.666210949420929,
      "logps/chosen": -0.6708008050918579,
      "logps/rejected": -1.0105469226837158,
      "loss": 0.8552,
      "nll_loss": 0.797070324420929,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06706543266773224,
      "rewards/margins": 0.03404998779296875,
      "rewards/rejected": -0.10109863430261612,
      "step": 18060
    },
    {
      "epoch": 0.6771467650971501,
      "grad_norm": 2.1561715416982463,
      "learning_rate": 5.951287221306115e-07,
      "log_odds_chosen": 0.5695434808731079,
      "log_odds_ratio": -0.5528320074081421,
      "logits/chosen": -0.7474609613418579,
      "logits/rejected": -0.637988269329071,
      "logps/chosen": -0.691210925579071,
      "logps/rejected": -1.062890648841858,
      "loss": 0.8501,
      "nll_loss": 0.786328136920929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06910400092601776,
      "rewards/margins": 0.03718719631433487,
      "rewards/rejected": -0.10629882663488388,
      "step": 18070
    },
    {
      "epoch": 0.677521500440314,
      "grad_norm": 2.124392305357487,
      "learning_rate": 5.949641173087295e-07,
      "log_odds_chosen": 0.5726684331893921,
      "log_odds_ratio": -0.584277331829071,
      "logits/chosen": -0.7486327886581421,
      "logits/rejected": -0.6527343988418579,
      "logps/chosen": -0.681640625,
      "logps/rejected": -1.0642578601837158,
      "loss": 0.8449,
      "nll_loss": 0.8681640625,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.068115234375,
      "rewards/margins": 0.03818206861615181,
      "rewards/rejected": -0.10634765774011612,
      "step": 18080
    },
    {
      "epoch": 0.6778962357834779,
      "grad_norm": 2.195767988076906,
      "learning_rate": 5.947996489939607e-07,
      "log_odds_chosen": 0.6246093511581421,
      "log_odds_ratio": -0.5506836175918579,
      "logits/chosen": -0.7705078125,
      "logits/rejected": -0.66845703125,
      "logps/chosen": -0.6568359136581421,
      "logps/rejected": -1.0498046875,
      "loss": 0.8454,
      "nll_loss": 0.7861328125,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06569824367761612,
      "rewards/margins": 0.03923339769244194,
      "rewards/rejected": -0.10488281399011612,
      "step": 18090
    },
    {
      "epoch": 0.6782709711266418,
      "grad_norm": 2.1387239843992463,
      "learning_rate": 5.94635316997733e-07,
      "log_odds_chosen": 0.4261718690395355,
      "log_odds_ratio": -0.6224609613418579,
      "logits/chosen": -0.7828124761581421,
      "logits/rejected": -0.674511730670929,
      "logps/chosen": -0.675000011920929,
      "logps/rejected": -0.9458984136581421,
      "loss": 0.8382,
      "nll_loss": 0.743359386920929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06745605170726776,
      "rewards/margins": 0.02711181715130806,
      "rewards/rejected": -0.09455566108226776,
      "step": 18100
    },
    {
      "epoch": 0.6786457064698057,
      "grad_norm": 2.010910284073033,
      "learning_rate": 5.944711211318392e-07,
      "log_odds_chosen": 0.47327882051467896,
      "log_odds_ratio": -0.5746093988418579,
      "logits/chosen": -0.7369140386581421,
      "logits/rejected": -0.619335949420929,
      "logps/chosen": -0.656054675579071,
      "logps/rejected": -0.9398437738418579,
      "loss": 0.8279,
      "nll_loss": 0.7630859613418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06557617336511612,
      "rewards/margins": 0.02835388109087944,
      "rewards/rejected": -0.09401855617761612,
      "step": 18110
    },
    {
      "epoch": 0.6790204418129696,
      "grad_norm": 1.9163870197732935,
      "learning_rate": 5.943070612084358e-07,
      "log_odds_chosen": 0.505615234375,
      "log_odds_ratio": -0.5858398675918579,
      "logits/chosen": -0.7837890386581421,
      "logits/rejected": -0.6871093511581421,
      "logps/chosen": -0.6421874761581421,
      "logps/rejected": -0.927734375,
      "loss": 0.837,
      "nll_loss": 0.7710937261581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.064208984375,
      "rewards/margins": 0.02850952185690403,
      "rewards/rejected": -0.09274902194738388,
      "step": 18120
    },
    {
      "epoch": 0.6793951771561335,
      "grad_norm": 2.607606557057078,
      "learning_rate": 5.941431370400415e-07,
      "log_odds_chosen": 0.390380859375,
      "log_odds_ratio": -0.634570300579071,
      "logits/chosen": -0.7030273675918579,
      "logits/rejected": -0.62451171875,
      "logps/chosen": -0.6927734613418579,
      "logps/rejected": -0.9439452886581421,
      "loss": 0.8698,
      "nll_loss": 0.774609386920929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06925048679113388,
      "rewards/margins": 0.02511444129049778,
      "rewards/rejected": -0.09451904147863388,
      "step": 18130
    },
    {
      "epoch": 0.6797699124992974,
      "grad_norm": 2.175053917466622,
      "learning_rate": 5.939793484395371e-07,
      "log_odds_chosen": 0.4057861268520355,
      "log_odds_ratio": -0.6161133050918579,
      "logits/chosen": -0.7904297113418579,
      "logits/rejected": -0.6640625,
      "logps/chosen": -0.6605468988418579,
      "logps/rejected": -0.9292968511581421,
      "loss": 0.8436,
      "nll_loss": 0.787890613079071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06600341945886612,
      "rewards/margins": 0.02682190015912056,
      "rewards/rejected": -0.09291992336511612,
      "step": 18140
    },
    {
      "epoch": 0.6801446478424613,
      "grad_norm": 2.184619578835148,
      "learning_rate": 5.938156952201644e-07,
      "log_odds_chosen": 0.4158569276332855,
      "log_odds_ratio": -0.614550769329071,
      "logits/chosen": -0.774218738079071,
      "logits/rejected": -0.6322265863418579,
      "logps/chosen": -0.69970703125,
      "logps/rejected": -0.9619140625,
      "loss": 0.8552,
      "nll_loss": 0.778124988079071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07003173977136612,
      "rewards/margins": 0.02613678015768528,
      "rewards/rejected": -0.09609375149011612,
      "step": 18150
    },
    {
      "epoch": 0.6805193831856251,
      "grad_norm": 2.0017800228881777,
      "learning_rate": 5.936521771955247e-07,
      "log_odds_chosen": 0.47032469511032104,
      "log_odds_ratio": -0.590527355670929,
      "logits/chosen": -0.7791992425918579,
      "logits/rejected": -0.6581054925918579,
      "logps/chosen": -0.686816394329071,
      "logps/rejected": -0.971484363079071,
      "loss": 0.8577,
      "nll_loss": 0.790820300579071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06865234673023224,
      "rewards/margins": 0.02850799635052681,
      "rewards/rejected": -0.09710693359375,
      "step": 18160
    },
    {
      "epoch": 0.680894118528789,
      "grad_norm": 2.51178500722113,
      "learning_rate": 5.934887941795792e-07,
      "log_odds_chosen": 0.513623058795929,
      "log_odds_ratio": -0.6029297113418579,
      "logits/chosen": -0.751757800579071,
      "logits/rejected": -0.624707043170929,
      "logps/chosen": -0.7181640863418579,
      "logps/rejected": -1.0341796875,
      "loss": 0.8633,
      "nll_loss": 0.8541015386581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07175292819738388,
      "rewards/margins": 0.03152618557214737,
      "rewards/rejected": -0.10329589992761612,
      "step": 18170
    },
    {
      "epoch": 0.6812688538719529,
      "grad_norm": 2.397810830803487,
      "learning_rate": 5.933255459866463e-07,
      "log_odds_chosen": 0.5341552495956421,
      "log_odds_ratio": -0.609179675579071,
      "logits/chosen": -0.7525390386581421,
      "logits/rejected": -0.618847668170929,
      "logps/chosen": -0.7123047113418579,
      "logps/rejected": -1.066796898841858,
      "loss": 0.845,
      "nll_loss": 0.81640625,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07127685844898224,
      "rewards/margins": 0.03551330417394638,
      "rewards/rejected": -0.10672607272863388,
      "step": 18180
    },
    {
      "epoch": 0.6816435892151168,
      "grad_norm": 2.4058217668467794,
      "learning_rate": 5.931624324314028e-07,
      "log_odds_chosen": 0.4303955137729645,
      "log_odds_ratio": -0.601855456829071,
      "logits/chosen": -0.7210937738418579,
      "logits/rejected": -0.6343749761581421,
      "logps/chosen": -0.680859386920929,
      "logps/rejected": -0.957226574420929,
      "loss": 0.8457,
      "nll_loss": 0.79296875,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06809081882238388,
      "rewards/margins": 0.027627181261777878,
      "rewards/rejected": -0.09573974460363388,
      "step": 18190
    },
    {
      "epoch": 0.6820183245582807,
      "grad_norm": 2.114850268971609,
      "learning_rate": 5.929994533288809e-07,
      "log_odds_chosen": 0.45393067598342896,
      "log_odds_ratio": -0.6146484613418579,
      "logits/chosen": -0.693652331829071,
      "logits/rejected": -0.584667980670929,
      "logps/chosen": -0.658203125,
      "logps/rejected": -0.9624999761581421,
      "loss": 0.8294,
      "nll_loss": 0.762499988079071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06577148288488388,
      "rewards/margins": 0.03043975867331028,
      "rewards/rejected": -0.09616699069738388,
      "step": 18200
    },
    {
      "epoch": 0.6823930599014446,
      "grad_norm": 2.328724667032255,
      "learning_rate": 5.928366084944692e-07,
      "log_odds_chosen": 0.4648681581020355,
      "log_odds_ratio": -0.6080077886581421,
      "logits/chosen": -0.7093750238418579,
      "logits/rejected": -0.6451171636581421,
      "logps/chosen": -0.6825195550918579,
      "logps/rejected": -0.9771484136581421,
      "loss": 0.8549,
      "nll_loss": 0.8695312738418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06831054389476776,
      "rewards/margins": 0.02937164343893528,
      "rewards/rejected": -0.09771728515625,
      "step": 18210
    },
    {
      "epoch": 0.6827677952446085,
      "grad_norm": 2.378999015341307,
      "learning_rate": 5.926738977439106e-07,
      "log_odds_chosen": 0.4966674745082855,
      "log_odds_ratio": -0.578808605670929,
      "logits/chosen": -0.746289074420929,
      "logits/rejected": -0.6407226324081421,
      "logps/chosen": -0.632031261920929,
      "logps/rejected": -0.9458984136581421,
      "loss": 0.8448,
      "nll_loss": 0.7613281011581421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06322021782398224,
      "rewards/margins": 0.03139953687787056,
      "rewards/rejected": -0.09462890774011612,
      "step": 18220
    },
    {
      "epoch": 0.6831425305877724,
      "grad_norm": 2.1377611069375715,
      "learning_rate": 5.925113208933017e-07,
      "log_odds_chosen": 0.3836303651332855,
      "log_odds_ratio": -0.628710925579071,
      "logits/chosen": -0.73828125,
      "logits/rejected": -0.641406238079071,
      "logps/chosen": -0.709765613079071,
      "logps/rejected": -0.950976550579071,
      "loss": 0.8522,
      "nll_loss": 0.8109375238418579,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.071044921875,
      "rewards/margins": 0.02408752404153347,
      "rewards/rejected": -0.0950927734375,
      "step": 18230
    },
    {
      "epoch": 0.6835172659309363,
      "grad_norm": 2.06120025640744,
      "learning_rate": 5.923488777590923e-07,
      "log_odds_chosen": 0.40662842988967896,
      "log_odds_ratio": -0.6044921875,
      "logits/chosen": -0.729687511920929,
      "logits/rejected": -0.627636730670929,
      "logps/chosen": -0.733203113079071,
      "logps/rejected": -0.996874988079071,
      "loss": 0.8349,
      "nll_loss": 0.814648449420929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07333984225988388,
      "rewards/margins": 0.02630920335650444,
      "rewards/rejected": -0.09965820610523224,
      "step": 18240
    },
    {
      "epoch": 0.6838920012741002,
      "grad_norm": 2.0337952123549004,
      "learning_rate": 5.921865681580842e-07,
      "log_odds_chosen": 0.5507446527481079,
      "log_odds_ratio": -0.605175793170929,
      "logits/chosen": -0.77734375,
      "logits/rejected": -0.6541992425918579,
      "logps/chosen": -0.6527343988418579,
      "logps/rejected": -0.9632812738418579,
      "loss": 0.8381,
      "nll_loss": 0.758984386920929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06528320163488388,
      "rewards/margins": 0.03109436109662056,
      "rewards/rejected": -0.0963134765625,
      "step": 18250
    },
    {
      "epoch": 0.684266736617264,
      "grad_norm": 2.1021515650247906,
      "learning_rate": 5.920243919074303e-07,
      "log_odds_chosen": 0.4693969786167145,
      "log_odds_ratio": -0.578417956829071,
      "logits/chosen": -0.7289062738418579,
      "logits/rejected": -0.63818359375,
      "logps/chosen": -0.641308605670929,
      "logps/rejected": -0.9156249761581421,
      "loss": 0.8217,
      "nll_loss": 0.720898449420929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06411132961511612,
      "rewards/margins": 0.02746734581887722,
      "rewards/rejected": -0.09157714992761612,
      "step": 18260
    },
    {
      "epoch": 0.6846414719604279,
      "grad_norm": 2.286189433528599,
      "learning_rate": 5.918623488246337e-07,
      "log_odds_chosen": 0.518627941608429,
      "log_odds_ratio": -0.5741211175918579,
      "logits/chosen": -0.8037109375,
      "logits/rejected": -0.705859363079071,
      "logps/chosen": -0.633593738079071,
      "logps/rejected": -0.955078125,
      "loss": 0.8467,
      "nll_loss": 0.742968738079071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0633544921875,
      "rewards/margins": 0.03217468410730362,
      "rewards/rejected": -0.09553222358226776,
      "step": 18270
    },
    {
      "epoch": 0.6850162073035918,
      "grad_norm": 2.009248643548574,
      "learning_rate": 5.917004387275473e-07,
      "log_odds_chosen": 0.37687379121780396,
      "log_odds_ratio": -0.618359386920929,
      "logits/chosen": -0.741015613079071,
      "logits/rejected": -0.6357421875,
      "logps/chosen": -0.6851562261581421,
      "logps/rejected": -0.9476562738418579,
      "loss": 0.8601,
      "nll_loss": 0.835742175579071,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06851806491613388,
      "rewards/margins": 0.026358794420957565,
      "rewards/rejected": -0.09477539360523224,
      "step": 18280
    },
    {
      "epoch": 0.6853909426467557,
      "grad_norm": 2.3072228247881617,
      "learning_rate": 5.915386614343725e-07,
      "log_odds_chosen": 0.5135253667831421,
      "log_odds_ratio": -0.5799804925918579,
      "logits/chosen": -0.7320312261581421,
      "logits/rejected": -0.610156238079071,
      "logps/chosen": -0.6773437261581421,
      "logps/rejected": -1.002343773841858,
      "loss": 0.8432,
      "nll_loss": 0.8070312738418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06777343899011612,
      "rewards/margins": 0.03252868726849556,
      "rewards/rejected": -0.10029296576976776,
      "step": 18290
    },
    {
      "epoch": 0.6857656779899196,
      "grad_norm": 2.6103540176856344,
      "learning_rate": 5.913770167636582e-07,
      "log_odds_chosen": 0.534191906452179,
      "log_odds_ratio": -0.6015625,
      "logits/chosen": -0.75146484375,
      "logits/rejected": -0.6541992425918579,
      "logps/chosen": -0.6732422113418579,
      "logps/rejected": -1.0207030773162842,
      "loss": 0.8453,
      "nll_loss": 0.810546875,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06735839694738388,
      "rewards/margins": 0.03465881198644638,
      "rewards/rejected": -0.10202636569738388,
      "step": 18300
    },
    {
      "epoch": 0.6861404133330835,
      "grad_norm": 2.5665365007719525,
      "learning_rate": 5.912155045343007e-07,
      "log_odds_chosen": 0.5334533452987671,
      "log_odds_ratio": -0.56201171875,
      "logits/chosen": -0.7479492425918579,
      "logits/rejected": -0.63818359375,
      "logps/chosen": -0.6449218988418579,
      "logps/rejected": -0.9818359613418579,
      "loss": 0.8239,
      "nll_loss": 0.7523437738418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06448974460363388,
      "rewards/margins": 0.03365974500775337,
      "rewards/rejected": -0.09819336235523224,
      "step": 18310
    },
    {
      "epoch": 0.6865151486762474,
      "grad_norm": 2.6654561992578865,
      "learning_rate": 5.910541245655417e-07,
      "log_odds_chosen": 0.43492430448532104,
      "log_odds_ratio": -0.5850585699081421,
      "logits/chosen": -0.732421875,
      "logits/rejected": -0.6368163824081421,
      "logps/chosen": -0.677050769329071,
      "logps/rejected": -0.9222656488418579,
      "loss": 0.8378,
      "nll_loss": 0.7935546636581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06773681938648224,
      "rewards/margins": 0.024505615234375,
      "rewards/rejected": -0.09221191704273224,
      "step": 18320
    },
    {
      "epoch": 0.6868898840194113,
      "grad_norm": 2.186526257030806,
      "learning_rate": 5.908928766769686e-07,
      "log_odds_chosen": 0.6202758550643921,
      "log_odds_ratio": -0.5679687261581421,
      "logits/chosen": -0.7378906011581421,
      "logits/rejected": -0.624218761920929,
      "logps/chosen": -0.6537109613418579,
      "logps/rejected": -1.035742163658142,
      "loss": 0.8436,
      "nll_loss": 0.7738281488418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06540527194738388,
      "rewards/margins": 0.03829498216509819,
      "rewards/rejected": -0.10361327975988388,
      "step": 18330
    },
    {
      "epoch": 0.6872646193625752,
      "grad_norm": 2.436775952278477,
      "learning_rate": 5.907317606885129e-07,
      "log_odds_chosen": 0.36663818359375,
      "log_odds_ratio": -0.637402355670929,
      "logits/chosen": -0.762499988079071,
      "logits/rejected": -0.6421874761581421,
      "logps/chosen": -0.6924804449081421,
      "logps/rejected": -0.9253906011581421,
      "loss": 0.8634,
      "nll_loss": 0.854296863079071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06920166313648224,
      "rewards/margins": 0.02333984337747097,
      "rewards/rejected": -0.09267578274011612,
      "step": 18340
    },
    {
      "epoch": 0.6876393547057391,
      "grad_norm": 2.309897666487282,
      "learning_rate": 5.905707764204498e-07,
      "log_odds_chosen": 0.5551513433456421,
      "log_odds_ratio": -0.579882800579071,
      "logits/chosen": -0.716015636920929,
      "logits/rejected": -0.60595703125,
      "logps/chosen": -0.633007824420929,
      "logps/rejected": -1.001953125,
      "loss": 0.8524,
      "nll_loss": 0.7894531488418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06328125298023224,
      "rewards/margins": 0.036840058863162994,
      "rewards/rejected": -0.10013427585363388,
      "step": 18350
    },
    {
      "epoch": 0.688014090048903,
      "grad_norm": 2.6354892284463323,
      "learning_rate": 5.904099236933968e-07,
      "log_odds_chosen": 0.554272472858429,
      "log_odds_ratio": -0.5960937738418579,
      "logits/chosen": -0.7392578125,
      "logits/rejected": -0.6446288824081421,
      "logps/chosen": -0.708789050579071,
      "logps/rejected": -1.0390625,
      "loss": 0.8426,
      "nll_loss": 0.8031250238418579,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07088623195886612,
      "rewards/margins": 0.03299102932214737,
      "rewards/rejected": -0.10382080078125,
      "step": 18360
    },
    {
      "epoch": 0.6883888253920668,
      "grad_norm": 2.5657662741506853,
      "learning_rate": 5.902492023283137e-07,
      "log_odds_chosen": 0.48485106229782104,
      "log_odds_ratio": -0.578417956829071,
      "logits/chosen": -0.763671875,
      "logits/rejected": -0.644726574420929,
      "logps/chosen": -0.6854492425918579,
      "logps/rejected": -0.9974609613418579,
      "loss": 0.8273,
      "nll_loss": 0.773242175579071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06857910007238388,
      "rewards/margins": 0.03111724928021431,
      "rewards/rejected": -0.09965820610523224,
      "step": 18370
    },
    {
      "epoch": 0.6887635607352307,
      "grad_norm": 2.007919420391105,
      "learning_rate": 5.90088612146501e-07,
      "log_odds_chosen": 0.4258666932582855,
      "log_odds_ratio": -0.622265636920929,
      "logits/chosen": -0.7158203125,
      "logits/rejected": -0.599902331829071,
      "logps/chosen": -0.6485351324081421,
      "logps/rejected": -0.922070324420929,
      "loss": 0.8344,
      "nll_loss": 0.770703136920929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0648193359375,
      "rewards/margins": 0.027332305908203125,
      "rewards/rejected": -0.09207763522863388,
      "step": 18380
    },
    {
      "epoch": 0.6891382960783946,
      "grad_norm": 2.089089820951873,
      "learning_rate": 5.899281529695992e-07,
      "log_odds_chosen": 0.581835925579071,
      "log_odds_ratio": -0.568359375,
      "logits/chosen": -0.737109363079071,
      "logits/rejected": -0.601757824420929,
      "logps/chosen": -0.6874023675918579,
      "logps/rejected": -1.056640625,
      "loss": 0.8438,
      "nll_loss": 0.7978515625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0687255859375,
      "rewards/margins": 0.03697967529296875,
      "rewards/rejected": -0.10563965141773224,
      "step": 18390
    },
    {
      "epoch": 0.6895130314215585,
      "grad_norm": 2.0010268588608278,
      "learning_rate": 5.897678246195885e-07,
      "log_odds_chosen": 0.5496581792831421,
      "log_odds_ratio": -0.6221679449081421,
      "logits/chosen": -0.745898425579071,
      "logits/rejected": -0.63720703125,
      "logps/chosen": -0.659960925579071,
      "logps/rejected": -1.0314452648162842,
      "loss": 0.8351,
      "nll_loss": 0.818164050579071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06596679985523224,
      "rewards/margins": 0.03710174560546875,
      "rewards/rejected": -0.10301513969898224,
      "step": 18400
    },
    {
      "epoch": 0.6898877667647224,
      "grad_norm": 2.038828099955187,
      "learning_rate": 5.896076269187873e-07,
      "log_odds_chosen": 0.51690673828125,
      "log_odds_ratio": -0.5992187261581421,
      "logits/chosen": -0.6966797113418579,
      "logits/rejected": -0.614550769329071,
      "logps/chosen": -0.672167956829071,
      "logps/rejected": -1.009374976158142,
      "loss": 0.8412,
      "nll_loss": 0.808789074420929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06719970703125,
      "rewards/margins": 0.03372955322265625,
      "rewards/rejected": -0.10096435248851776,
      "step": 18410
    },
    {
      "epoch": 0.6902625021078863,
      "grad_norm": 2.002763006548472,
      "learning_rate": 5.894475596898518e-07,
      "log_odds_chosen": 0.45716553926467896,
      "log_odds_ratio": -0.621289074420929,
      "logits/chosen": -0.759960949420929,
      "logits/rejected": -0.65625,
      "logps/chosen": -0.6717773675918579,
      "logps/rejected": -0.9932616949081421,
      "loss": 0.8572,
      "nll_loss": 0.782031238079071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06721191108226776,
      "rewards/margins": 0.032247163355350494,
      "rewards/rejected": -0.09934081882238388,
      "step": 18420
    },
    {
      "epoch": 0.6906372374510502,
      "grad_norm": 2.4073755889182924,
      "learning_rate": 5.892876227557749e-07,
      "log_odds_chosen": 0.6166137456893921,
      "log_odds_ratio": -0.5640624761581421,
      "logits/chosen": -0.8011718988418579,
      "logits/rejected": -0.6851562261581421,
      "logps/chosen": -0.668652355670929,
      "logps/rejected": -1.0578124523162842,
      "loss": 0.8398,
      "nll_loss": 0.7930663824081421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06688232719898224,
      "rewards/margins": 0.03890075534582138,
      "rewards/rejected": -0.10573730617761612,
      "step": 18430
    },
    {
      "epoch": 0.6910119727942141,
      "grad_norm": 2.269004711074293,
      "learning_rate": 5.891278159398855e-07,
      "log_odds_chosen": 0.5820068120956421,
      "log_odds_ratio": -0.560839831829071,
      "logits/chosen": -0.7904297113418579,
      "logits/rejected": -0.6773437261581421,
      "logps/chosen": -0.6734374761581421,
      "logps/rejected": -1.052148461341858,
      "loss": 0.842,
      "nll_loss": 0.805859386920929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0673828125,
      "rewards/margins": 0.03786926344037056,
      "rewards/rejected": -0.105224609375,
      "step": 18440
    },
    {
      "epoch": 0.691386708137378,
      "grad_norm": 2.091894806069676,
      "learning_rate": 5.889681390658482e-07,
      "log_odds_chosen": 0.5831054449081421,
      "log_odds_ratio": -0.565673828125,
      "logits/chosen": -0.777539074420929,
      "logits/rejected": -0.645312488079071,
      "logps/chosen": -0.707226574420929,
      "logps/rejected": -1.078710913658142,
      "loss": 0.8409,
      "nll_loss": 0.7738281488418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07069091498851776,
      "rewards/margins": 0.03709106519818306,
      "rewards/rejected": -0.10773925483226776,
      "step": 18450
    },
    {
      "epoch": 0.6917614434805419,
      "grad_norm": 2.287032456285514,
      "learning_rate": 5.888085919576612e-07,
      "log_odds_chosen": 0.511486828327179,
      "log_odds_ratio": -0.583691418170929,
      "logits/chosen": -0.7142578363418579,
      "logits/rejected": -0.628222644329071,
      "logps/chosen": -0.642578125,
      "logps/rejected": -0.971484363079071,
      "loss": 0.8494,
      "nll_loss": 0.7691406011581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06427001953125,
      "rewards/margins": 0.0328521728515625,
      "rewards/rejected": -0.09714355319738388,
      "step": 18460
    },
    {
      "epoch": 0.6921361788237057,
      "grad_norm": 2.5699878446022213,
      "learning_rate": 5.886491744396568e-07,
      "log_odds_chosen": 0.6522582769393921,
      "log_odds_ratio": -0.5801757574081421,
      "logits/chosen": -0.741406261920929,
      "logits/rejected": -0.5912109613418579,
      "logps/chosen": -0.6830078363418579,
      "logps/rejected": -1.1150391101837158,
      "loss": 0.8551,
      "nll_loss": 0.852343738079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06829833984375,
      "rewards/margins": 0.043239593505859375,
      "rewards/rejected": -0.111572265625,
      "step": 18470
    },
    {
      "epoch": 0.6925109141668696,
      "grad_norm": 2.352978036525165,
      "learning_rate": 5.884898863364996e-07,
      "log_odds_chosen": 0.5731201171875,
      "log_odds_ratio": -0.561230480670929,
      "logits/chosen": -0.797070324420929,
      "logits/rejected": -0.670703113079071,
      "logps/chosen": -0.6583007574081421,
      "logps/rejected": -1.0334961414337158,
      "loss": 0.8456,
      "nll_loss": 0.792187511920929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06575927883386612,
      "rewards/margins": 0.03757476806640625,
      "rewards/rejected": -0.10343017429113388,
      "step": 18480
    },
    {
      "epoch": 0.6928856495100335,
      "grad_norm": 2.376979506509014,
      "learning_rate": 5.883307274731868e-07,
      "log_odds_chosen": 0.593920886516571,
      "log_odds_ratio": -0.553515613079071,
      "logits/chosen": -0.8125,
      "logits/rejected": -0.6429687738418579,
      "logps/chosen": -0.6439453363418579,
      "logps/rejected": -1.040624976158142,
      "loss": 0.8235,
      "nll_loss": 0.740429699420929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06436767429113388,
      "rewards/margins": 0.0396728515625,
      "rewards/rejected": -0.10407714545726776,
      "step": 18490
    },
    {
      "epoch": 0.6932603848531974,
      "grad_norm": 2.060215263998189,
      "learning_rate": 5.881716976750462e-07,
      "log_odds_chosen": 0.52490234375,
      "log_odds_ratio": -0.6005859375,
      "logits/chosen": -0.754101574420929,
      "logits/rejected": -0.6265624761581421,
      "logps/chosen": -0.687695324420929,
      "logps/rejected": -1.0232422351837158,
      "loss": 0.8355,
      "nll_loss": 0.7708984613418579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06875000149011612,
      "rewards/margins": 0.03351440280675888,
      "rewards/rejected": -0.10231933742761612,
      "step": 18500
    },
    {
      "epoch": 0.6936351201963613,
      "grad_norm": 2.519536434016266,
      "learning_rate": 5.88012796767736e-07,
      "log_odds_chosen": 0.49720460176467896,
      "log_odds_ratio": -0.6063476800918579,
      "logits/chosen": -0.6568359136581421,
      "logits/rejected": -0.5941406488418579,
      "logps/chosen": -0.656933605670929,
      "logps/rejected": -0.951367199420929,
      "loss": 0.8531,
      "nll_loss": 0.7890625,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06563720852136612,
      "rewards/margins": 0.029460906982421875,
      "rewards/rejected": -0.09519042819738388,
      "step": 18510
    },
    {
      "epoch": 0.6940098555395252,
      "grad_norm": 2.0741152327158776,
      "learning_rate": 5.878540245772441e-07,
      "log_odds_chosen": 0.608654797077179,
      "log_odds_ratio": -0.551562488079071,
      "logits/chosen": -0.7318359613418579,
      "logits/rejected": -0.627734363079071,
      "logps/chosen": -0.669238269329071,
      "logps/rejected": -1.0574219226837158,
      "loss": 0.8437,
      "nll_loss": 0.777148425579071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06687011569738388,
      "rewards/margins": 0.03893585130572319,
      "rewards/rejected": -0.10576172173023224,
      "step": 18520
    },
    {
      "epoch": 0.6943845908826891,
      "grad_norm": 2.2497334328522696,
      "learning_rate": 5.876953809298869e-07,
      "log_odds_chosen": 0.6408935785293579,
      "log_odds_ratio": -0.553417980670929,
      "logits/chosen": -0.7059570550918579,
      "logits/rejected": -0.633984386920929,
      "logps/chosen": -0.6126953363418579,
      "logps/rejected": -0.980761706829071,
      "loss": 0.8243,
      "nll_loss": 0.7704101800918579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.061279296875,
      "rewards/margins": 0.03685493394732475,
      "rewards/rejected": -0.09812011569738388,
      "step": 18530
    },
    {
      "epoch": 0.694759326225853,
      "grad_norm": 2.4071756425743858,
      "learning_rate": 5.87536865652309e-07,
      "log_odds_chosen": 0.517321765422821,
      "log_odds_ratio": -0.597949206829071,
      "logits/chosen": -0.7705078125,
      "logits/rejected": -0.663769543170929,
      "logps/chosen": -0.694140613079071,
      "logps/rejected": -1.024804711341858,
      "loss": 0.852,
      "nll_loss": 0.8101562261581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06943359225988388,
      "rewards/margins": 0.03310089185833931,
      "rewards/rejected": -0.10258789360523224,
      "step": 18540
    },
    {
      "epoch": 0.6951340615690169,
      "grad_norm": 2.349984189698268,
      "learning_rate": 5.873784785714818e-07,
      "log_odds_chosen": 0.39977723360061646,
      "log_odds_ratio": -0.63671875,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.6214843988418579,
      "logps/chosen": -0.7139648199081421,
      "logps/rejected": -0.968945324420929,
      "loss": 0.8546,
      "nll_loss": 0.84375,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0714111328125,
      "rewards/margins": 0.02558593824505806,
      "rewards/rejected": -0.09702148288488388,
      "step": 18550
    },
    {
      "epoch": 0.6955087969121808,
      "grad_norm": 2.255337437024598,
      "learning_rate": 5.872202195147034e-07,
      "log_odds_chosen": 0.5032714605331421,
      "log_odds_ratio": -0.591015636920929,
      "logits/chosen": -0.717578113079071,
      "logits/rejected": -0.6302734613418579,
      "logps/chosen": -0.6849609613418579,
      "logps/rejected": -1.0148437023162842,
      "loss": 0.8529,
      "nll_loss": 0.810351550579071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06859131157398224,
      "rewards/margins": 0.03305663913488388,
      "rewards/rejected": -0.10161133110523224,
      "step": 18560
    },
    {
      "epoch": 0.6958835322553447,
      "grad_norm": 2.0816140963935825,
      "learning_rate": 5.870620883095973e-07,
      "log_odds_chosen": 0.64959716796875,
      "log_odds_ratio": -0.5380859375,
      "logits/chosen": -0.7503906488418579,
      "logits/rejected": -0.6107422113418579,
      "logps/chosen": -0.624218761920929,
      "logps/rejected": -1.0291016101837158,
      "loss": 0.8414,
      "nll_loss": 0.8062499761581421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06239013746380806,
      "rewards/margins": 0.04052581638097763,
      "rewards/rejected": -0.10288085788488388,
      "step": 18570
    },
    {
      "epoch": 0.6962582675985085,
      "grad_norm": 1.8795353084420252,
      "learning_rate": 5.869040847841115e-07,
      "log_odds_chosen": 0.5065673589706421,
      "log_odds_ratio": -0.5771484375,
      "logits/chosen": -0.743945300579071,
      "logits/rejected": -0.637890636920929,
      "logps/chosen": -0.648632824420929,
      "logps/rejected": -0.9644531011581421,
      "loss": 0.8488,
      "nll_loss": 0.7652343511581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06484375149011612,
      "rewards/margins": 0.03154449537396431,
      "rewards/rejected": -0.09636230766773224,
      "step": 18580
    },
    {
      "epoch": 0.6966330029416724,
      "grad_norm": 2.1629367617822752,
      "learning_rate": 5.867462087665184e-07,
      "log_odds_chosen": 0.5291748046875,
      "log_odds_ratio": -0.589648425579071,
      "logits/chosen": -0.7489258050918579,
      "logits/rejected": -0.6534179449081421,
      "logps/chosen": -0.622851550579071,
      "logps/rejected": -0.9634765386581421,
      "loss": 0.8552,
      "nll_loss": 0.7749999761581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06230468675494194,
      "rewards/margins": 0.0340118408203125,
      "rewards/rejected": -0.09632568061351776,
      "step": 18590
    },
    {
      "epoch": 0.6970077382848363,
      "grad_norm": 2.694224704441416,
      "learning_rate": 5.865884600854132e-07,
      "log_odds_chosen": 0.548596203327179,
      "log_odds_ratio": -0.584277331829071,
      "logits/chosen": -0.719531238079071,
      "logits/rejected": -0.619921863079071,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -1.081445336341858,
      "loss": 0.8475,
      "nll_loss": 0.835156261920929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07161865383386612,
      "rewards/margins": 0.03652191162109375,
      "rewards/rejected": -0.10814209282398224,
      "step": 18600
    },
    {
      "epoch": 0.6973824736280002,
      "grad_norm": 2.244179855693108,
      "learning_rate": 5.864308385697138e-07,
      "log_odds_chosen": 0.5421142578125,
      "log_odds_ratio": -0.579882800579071,
      "logits/chosen": -0.685742199420929,
      "logits/rejected": -0.574414074420929,
      "logps/chosen": -0.6732422113418579,
      "logps/rejected": -1.021484375,
      "loss": 0.8418,
      "nll_loss": 0.83984375,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06743164360523224,
      "rewards/margins": 0.034698486328125,
      "rewards/rejected": -0.10214843600988388,
      "step": 18610
    },
    {
      "epoch": 0.6977572089711641,
      "grad_norm": 1.9735934840443112,
      "learning_rate": 5.862733440486595e-07,
      "log_odds_chosen": 0.6102539300918579,
      "log_odds_ratio": -0.582275390625,
      "logits/chosen": -0.7027343511581421,
      "logits/rejected": -0.615234375,
      "logps/chosen": -0.666308581829071,
      "logps/rejected": -1.062890648841858,
      "loss": 0.8366,
      "nll_loss": 0.7655273675918579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06661377102136612,
      "rewards/margins": 0.039707183837890625,
      "rewards/rejected": -0.1063232421875,
      "step": 18620
    },
    {
      "epoch": 0.698131944314328,
      "grad_norm": 2.2386695034144743,
      "learning_rate": 5.861159763518106e-07,
      "log_odds_chosen": 0.626171886920929,
      "log_odds_ratio": -0.565625011920929,
      "logits/chosen": -0.7210937738418579,
      "logits/rejected": -0.6214843988418579,
      "logps/chosen": -0.65283203125,
      "logps/rejected": -1.0458984375,
      "loss": 0.8361,
      "nll_loss": 0.772265613079071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06529541313648224,
      "rewards/margins": 0.03940124437212944,
      "rewards/rejected": -0.10458984225988388,
      "step": 18630
    },
    {
      "epoch": 0.6985066796574919,
      "grad_norm": 2.328828727105925,
      "learning_rate": 5.859587353090476e-07,
      "log_odds_chosen": 0.47309571504592896,
      "log_odds_ratio": -0.602343738079071,
      "logits/chosen": -0.696093738079071,
      "logits/rejected": -0.617871105670929,
      "logps/chosen": -0.699999988079071,
      "logps/rejected": -0.98828125,
      "loss": 0.8471,
      "nll_loss": 0.738574206829071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06998290866613388,
      "rewards/margins": 0.028827667236328125,
      "rewards/rejected": -0.09877929836511612,
      "step": 18640
    },
    {
      "epoch": 0.6988814150006558,
      "grad_norm": 2.0825370723827286,
      "learning_rate": 5.858016207505699e-07,
      "log_odds_chosen": 0.616546630859375,
      "log_odds_ratio": -0.591015636920929,
      "logits/chosen": -0.7603515386581421,
      "logits/rejected": -0.638671875,
      "logps/chosen": -0.7388671636581421,
      "logps/rejected": -1.1748046875,
      "loss": 0.8616,
      "nll_loss": 0.8394531011581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07386474311351776,
      "rewards/margins": 0.04359130933880806,
      "rewards/rejected": -0.11749267578125,
      "step": 18650
    },
    {
      "epoch": 0.6992561503438197,
      "grad_norm": 3.153672595534753,
      "learning_rate": 5.856446325068959e-07,
      "log_odds_chosen": 0.533862292766571,
      "log_odds_ratio": -0.592089831829071,
      "logits/chosen": -0.71630859375,
      "logits/rejected": -0.5926758050918579,
      "logps/chosen": -0.67578125,
      "logps/rejected": -1.0271484851837158,
      "loss": 0.8312,
      "nll_loss": 0.820507824420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06759033352136612,
      "rewards/margins": 0.03511924669146538,
      "rewards/rejected": -0.10262451320886612,
      "step": 18660
    },
    {
      "epoch": 0.6996308856869836,
      "grad_norm": 2.2787401289988676,
      "learning_rate": 5.854877704088614e-07,
      "log_odds_chosen": 0.4462524354457855,
      "log_odds_ratio": -0.6246093511581421,
      "logits/chosen": -0.701855480670929,
      "logits/rejected": -0.603222668170929,
      "logps/chosen": -0.69189453125,
      "logps/rejected": -1.0071289539337158,
      "loss": 0.8509,
      "nll_loss": 0.8189452886581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06918945163488388,
      "rewards/margins": 0.03153533861041069,
      "rewards/rejected": -0.10076904296875,
      "step": 18670
    },
    {
      "epoch": 0.7000056210301474,
      "grad_norm": 2.3143886165473995,
      "learning_rate": 5.853310342876193e-07,
      "log_odds_chosen": 0.5130981206893921,
      "log_odds_ratio": -0.5816406011581421,
      "logits/chosen": -0.780468761920929,
      "logits/rejected": -0.6751953363418579,
      "logps/chosen": -0.65869140625,
      "logps/rejected": -0.979296863079071,
      "loss": 0.8573,
      "nll_loss": 0.7860351800918579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06589355319738388,
      "rewards/margins": 0.03204955905675888,
      "rewards/rejected": -0.09791259467601776,
      "step": 18680
    },
    {
      "epoch": 0.7003803563733113,
      "grad_norm": 2.151290929760984,
      "learning_rate": 5.851744239746388e-07,
      "log_odds_chosen": 0.520031750202179,
      "log_odds_ratio": -0.5799804925918579,
      "logits/chosen": -0.741992175579071,
      "logits/rejected": -0.614453136920929,
      "logps/chosen": -0.6552734375,
      "logps/rejected": -1.0046875476837158,
      "loss": 0.8441,
      "nll_loss": 0.7802734375,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.0655517578125,
      "rewards/margins": 0.03493041917681694,
      "rewards/rejected": -0.10048828274011612,
      "step": 18690
    },
    {
      "epoch": 0.7007550917164752,
      "grad_norm": 2.421028806838942,
      "learning_rate": 5.850179393017045e-07,
      "log_odds_chosen": 0.4901367127895355,
      "log_odds_ratio": -0.595410168170929,
      "logits/chosen": -0.718554675579071,
      "logits/rejected": -0.6299804449081421,
      "logps/chosen": -0.719531238079071,
      "logps/rejected": -1.0261719226837158,
      "loss": 0.8267,
      "nll_loss": 0.7757812738418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07198486477136612,
      "rewards/margins": 0.03069610521197319,
      "rewards/rejected": -0.10267333686351776,
      "step": 18700
    },
    {
      "epoch": 0.7011298270596391,
      "grad_norm": 2.189889666815357,
      "learning_rate": 5.848615801009158e-07,
      "log_odds_chosen": 0.46534425020217896,
      "log_odds_ratio": -0.5853515863418579,
      "logits/chosen": -0.790820300579071,
      "logits/rejected": -0.69580078125,
      "logps/chosen": -0.6156250238418579,
      "logps/rejected": -0.8863281011581421,
      "loss": 0.8178,
      "nll_loss": 0.7088867425918579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06159668043255806,
      "rewards/margins": 0.027099609375,
      "rewards/rejected": -0.08863525092601776,
      "step": 18710
    },
    {
      "epoch": 0.701504562402803,
      "grad_norm": 2.3386204255789265,
      "learning_rate": 5.847053462046862e-07,
      "log_odds_chosen": 0.6034301519393921,
      "log_odds_ratio": -0.5772460699081421,
      "logits/chosen": -0.756152331829071,
      "logits/rejected": -0.6532226800918579,
      "logps/chosen": -0.6641601324081421,
      "logps/rejected": -1.057031273841858,
      "loss": 0.8344,
      "nll_loss": 0.758007824420929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06639404594898224,
      "rewards/margins": 0.03932800143957138,
      "rewards/rejected": -0.10581054538488388,
      "step": 18720
    },
    {
      "epoch": 0.7018792977459669,
      "grad_norm": 2.120526129562906,
      "learning_rate": 5.845492374457418e-07,
      "log_odds_chosen": 0.47972410917282104,
      "log_odds_ratio": -0.59619140625,
      "logits/chosen": -0.709765613079071,
      "logits/rejected": -0.6333984136581421,
      "logps/chosen": -0.680957019329071,
      "logps/rejected": -0.9404296875,
      "loss": 0.8403,
      "nll_loss": 0.8037109375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06802978366613388,
      "rewards/margins": 0.026001740247011185,
      "rewards/rejected": -0.09409179538488388,
      "step": 18730
    },
    {
      "epoch": 0.7022540330891308,
      "grad_norm": 2.56060965410774,
      "learning_rate": 5.843932536571219e-07,
      "log_odds_chosen": 0.5028076171875,
      "log_odds_ratio": -0.604785144329071,
      "logits/chosen": -0.708300769329071,
      "logits/rejected": -0.63623046875,
      "logps/chosen": -0.665332019329071,
      "logps/rejected": -0.9970703125,
      "loss": 0.8257,
      "nll_loss": 0.755078136920929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06656493991613388,
      "rewards/margins": 0.03314208984375,
      "rewards/rejected": -0.0997314453125,
      "step": 18740
    },
    {
      "epoch": 0.7026287684322947,
      "grad_norm": 2.0108479216178514,
      "learning_rate": 5.842373946721771e-07,
      "log_odds_chosen": 0.6169189214706421,
      "log_odds_ratio": -0.5645507574081421,
      "logits/chosen": -0.7978515625,
      "logits/rejected": -0.693164050579071,
      "logps/chosen": -0.6810547113418579,
      "logps/rejected": -1.082421898841858,
      "loss": 0.8482,
      "nll_loss": 0.7955077886581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06805419921875,
      "rewards/margins": 0.040084075182676315,
      "rewards/rejected": -0.108154296875,
      "step": 18750
    },
    {
      "epoch": 0.7030035037754586,
      "grad_norm": 2.281702063522685,
      "learning_rate": 5.840816603245691e-07,
      "log_odds_chosen": 0.438018798828125,
      "log_odds_ratio": -0.595507800579071,
      "logits/chosen": -0.766796886920929,
      "logits/rejected": -0.6434570550918579,
      "logps/chosen": -0.646777331829071,
      "logps/rejected": -0.9283202886581421,
      "loss": 0.8518,
      "nll_loss": 0.771484375,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06467285007238388,
      "rewards/margins": 0.028141021728515625,
      "rewards/rejected": -0.09279785305261612,
      "step": 18760
    },
    {
      "epoch": 0.7033782391186225,
      "grad_norm": 2.213022795394803,
      "learning_rate": 5.839260504482696e-07,
      "log_odds_chosen": 0.592724621295929,
      "log_odds_ratio": -0.5858398675918579,
      "logits/chosen": -0.775390625,
      "logits/rejected": -0.6329101324081421,
      "logps/chosen": -0.7099609375,
      "logps/rejected": -1.0978515148162842,
      "loss": 0.8388,
      "nll_loss": 0.8052734136581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07099609076976776,
      "rewards/margins": 0.03888092190027237,
      "rewards/rejected": -0.10981445014476776,
      "step": 18770
    },
    {
      "epoch": 0.7037529744617864,
      "grad_norm": 3.0132616262647605,
      "learning_rate": 5.837705648775598e-07,
      "log_odds_chosen": 0.5257568359375,
      "log_odds_ratio": -0.592578113079071,
      "logits/chosen": -0.741992175579071,
      "logits/rejected": -0.6226562261581421,
      "logps/chosen": -0.724804699420929,
      "logps/rejected": -1.0866210460662842,
      "loss": 0.8438,
      "nll_loss": 0.7943359613418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07248535007238388,
      "rewards/margins": 0.03612060472369194,
      "rewards/rejected": -0.10860595852136612,
      "step": 18780
    },
    {
      "epoch": 0.7041277098049502,
      "grad_norm": 2.2510804160131186,
      "learning_rate": 5.836152034470301e-07,
      "log_odds_chosen": 0.45649415254592896,
      "log_odds_ratio": -0.595996081829071,
      "logits/chosen": -0.747851550579071,
      "logits/rejected": -0.650585949420929,
      "logps/chosen": -0.6888672113418579,
      "logps/rejected": -0.9683593511581421,
      "loss": 0.8408,
      "nll_loss": 0.800488293170929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06889648735523224,
      "rewards/margins": 0.027915190905332565,
      "rewards/rejected": -0.09687499701976776,
      "step": 18790
    },
    {
      "epoch": 0.7045024451481141,
      "grad_norm": 2.0381946060200042,
      "learning_rate": 5.834599659915782e-07,
      "log_odds_chosen": 0.38720703125,
      "log_odds_ratio": -0.6219726800918579,
      "logits/chosen": -0.7388671636581421,
      "logits/rejected": -0.661425769329071,
      "logps/chosen": -0.6402343511581421,
      "logps/rejected": -0.853710949420929,
      "loss": 0.8384,
      "nll_loss": 0.7603515386581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06403808295726776,
      "rewards/margins": 0.02131042443215847,
      "rewards/rejected": -0.08540038764476776,
      "step": 18800
    },
    {
      "epoch": 0.704877180491278,
      "grad_norm": 2.127194351983736,
      "learning_rate": 5.833048523464095e-07,
      "log_odds_chosen": 0.40083009004592896,
      "log_odds_ratio": -0.6045898199081421,
      "logits/chosen": -0.6966797113418579,
      "logits/rejected": -0.639355480670929,
      "logps/chosen": -0.6903320550918579,
      "logps/rejected": -0.9292968511581421,
      "loss": 0.8406,
      "nll_loss": 0.814453125,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06906738132238388,
      "rewards/margins": 0.02380676195025444,
      "rewards/rejected": -0.09287109225988388,
      "step": 18810
    },
    {
      "epoch": 0.7052519158344419,
      "grad_norm": 2.481094223992911,
      "learning_rate": 5.831498623470357e-07,
      "log_odds_chosen": 0.694628894329071,
      "log_odds_ratio": -0.528515636920929,
      "logits/chosen": -0.7398437261581421,
      "logits/rejected": -0.620312511920929,
      "logps/chosen": -0.638671875,
      "logps/rejected": -1.0802733898162842,
      "loss": 0.8377,
      "nll_loss": 0.754589855670929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06381835788488388,
      "rewards/margins": 0.04426269605755806,
      "rewards/rejected": -0.10805664211511612,
      "step": 18820
    },
    {
      "epoch": 0.7056266511776058,
      "grad_norm": 2.591555923380613,
      "learning_rate": 5.829949958292743e-07,
      "log_odds_chosen": 0.517987072467804,
      "log_odds_ratio": -0.54443359375,
      "logits/chosen": -0.7408202886581421,
      "logits/rejected": -0.6172851324081421,
      "logps/chosen": -0.6488281488418579,
      "logps/rejected": -0.95849609375,
      "loss": 0.8311,
      "nll_loss": 0.732128918170929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06485595554113388,
      "rewards/margins": 0.03105316124856472,
      "rewards/rejected": -0.09589843451976776,
      "step": 18830
    },
    {
      "epoch": 0.7060013865207697,
      "grad_norm": 2.140595137160974,
      "learning_rate": 5.828402526292479e-07,
      "log_odds_chosen": 0.32905274629592896,
      "log_odds_ratio": -0.648242175579071,
      "logits/chosen": -0.766406238079071,
      "logits/rejected": -0.677539050579071,
      "logps/chosen": -0.7232421636581421,
      "logps/rejected": -0.925097644329071,
      "loss": 0.8587,
      "nll_loss": 0.7935546636581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07231445610523224,
      "rewards/margins": 0.02021637000143528,
      "rewards/rejected": -0.09250488132238388,
      "step": 18840
    },
    {
      "epoch": 0.7063761218639336,
      "grad_norm": 2.751028184202947,
      "learning_rate": 5.826856325833838e-07,
      "log_odds_chosen": 0.47993165254592896,
      "log_odds_ratio": -0.5962890386581421,
      "logits/chosen": -0.7320312261581421,
      "logits/rejected": -0.611621081829071,
      "logps/chosen": -0.682910144329071,
      "logps/rejected": -0.9759765863418579,
      "loss": 0.8477,
      "nll_loss": 0.7763671875,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06833495944738388,
      "rewards/margins": 0.02930908277630806,
      "rewards/rejected": -0.0975341796875,
      "step": 18850
    },
    {
      "epoch": 0.7067508572070975,
      "grad_norm": 2.379669939517642,
      "learning_rate": 5.825311355284121e-07,
      "log_odds_chosen": 0.526806652545929,
      "log_odds_ratio": -0.6109374761581421,
      "logits/chosen": -0.7451171875,
      "logits/rejected": -0.633105456829071,
      "logps/chosen": -0.670703113079071,
      "logps/rejected": -1.0437500476837158,
      "loss": 0.8323,
      "nll_loss": 0.800585925579071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06712646782398224,
      "rewards/margins": 0.03728942945599556,
      "rewards/rejected": -0.10434570163488388,
      "step": 18860
    },
    {
      "epoch": 0.7071255925502614,
      "grad_norm": 2.2156585857823328,
      "learning_rate": 5.823767613013663e-07,
      "log_odds_chosen": 0.45130616426467896,
      "log_odds_ratio": -0.592968761920929,
      "logits/chosen": -0.758007824420929,
      "logits/rejected": -0.656445324420929,
      "logps/chosen": -0.6587890386581421,
      "logps/rejected": -0.9351562261581421,
      "loss": 0.8229,
      "nll_loss": 0.720703125,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06584472954273224,
      "rewards/margins": 0.027625273913145065,
      "rewards/rejected": -0.09353027492761612,
      "step": 18870
    },
    {
      "epoch": 0.7075003278934253,
      "grad_norm": 2.0799932114396538,
      "learning_rate": 5.82222509739582e-07,
      "log_odds_chosen": 0.43818360567092896,
      "log_odds_ratio": -0.622851550579071,
      "logits/chosen": -0.7613281011581421,
      "logits/rejected": -0.6566406488418579,
      "logps/chosen": -0.654003918170929,
      "logps/rejected": -0.921875,
      "loss": 0.8517,
      "nll_loss": 0.779492199420929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06541748344898224,
      "rewards/margins": 0.02671203576028347,
      "rewards/rejected": -0.09224853664636612,
      "step": 18880
    },
    {
      "epoch": 0.7078750632365891,
      "grad_norm": 2.6720583508905484,
      "learning_rate": 5.820683806806961e-07,
      "log_odds_chosen": 0.43597412109375,
      "log_odds_ratio": -0.6405273675918579,
      "logits/chosen": -0.685742199420929,
      "logits/rejected": -0.652050793170929,
      "logps/chosen": -0.6856445074081421,
      "logps/rejected": -0.962695300579071,
      "loss": 0.8362,
      "nll_loss": 0.7305663824081421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06855468451976776,
      "rewards/margins": 0.0276031494140625,
      "rewards/rejected": -0.09622802585363388,
      "step": 18890
    },
    {
      "epoch": 0.708249798579753,
      "grad_norm": 2.8656336070109107,
      "learning_rate": 5.819143739626463e-07,
      "log_odds_chosen": 0.258056640625,
      "log_odds_ratio": -0.684277355670929,
      "logits/chosen": -0.745410144329071,
      "logits/rejected": -0.6483398675918579,
      "logps/chosen": -0.721386730670929,
      "logps/rejected": -0.9058593511581421,
      "loss": 0.8456,
      "nll_loss": 0.7813476324081421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07209472358226776,
      "rewards/margins": 0.018512725830078125,
      "rewards/rejected": -0.09064941108226776,
      "step": 18900
    },
    {
      "epoch": 0.7086245339229169,
      "grad_norm": 2.5872681694742234,
      "learning_rate": 5.817604894236704e-07,
      "log_odds_chosen": 0.55853271484375,
      "log_odds_ratio": -0.5556640625,
      "logits/chosen": -0.735156238079071,
      "logits/rejected": -0.657421886920929,
      "logps/chosen": -0.665820300579071,
      "logps/rejected": -1.0070312023162842,
      "loss": 0.8371,
      "nll_loss": 0.7757812738418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06657715141773224,
      "rewards/margins": 0.03405914455652237,
      "rewards/rejected": -0.10063476860523224,
      "step": 18910
    },
    {
      "epoch": 0.7089992692660808,
      "grad_norm": 2.199087300767901,
      "learning_rate": 5.816067269023052e-07,
      "log_odds_chosen": 0.477783203125,
      "log_odds_ratio": -0.5899413824081421,
      "logits/chosen": -0.7154296636581421,
      "logits/rejected": -0.626953125,
      "logps/chosen": -0.654589831829071,
      "logps/rejected": -0.9537109136581421,
      "loss": 0.8329,
      "nll_loss": 0.7525390386581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06547851860523224,
      "rewards/margins": 0.0300445556640625,
      "rewards/rejected": -0.09544678032398224,
      "step": 18920
    },
    {
      "epoch": 0.7093740046092447,
      "grad_norm": 2.1308910999671125,
      "learning_rate": 5.814530862373863e-07,
      "log_odds_chosen": 0.522595226764679,
      "log_odds_ratio": -0.5794922113418579,
      "logits/chosen": -0.7127929925918579,
      "logits/rejected": -0.62060546875,
      "logps/chosen": -0.6766601800918579,
      "logps/rejected": -1.001562476158142,
      "loss": 0.8491,
      "nll_loss": 0.784960925579071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.067626953125,
      "rewards/margins": 0.03254089504480362,
      "rewards/rejected": -0.10025634616613388,
      "step": 18930
    },
    {
      "epoch": 0.7097487399524086,
      "grad_norm": 2.5582749949461765,
      "learning_rate": 5.812995672680471e-07,
      "log_odds_chosen": 0.36968994140625,
      "log_odds_ratio": -0.6416991949081421,
      "logits/chosen": -0.705078125,
      "logits/rejected": -0.6153320074081421,
      "logps/chosen": -0.699999988079071,
      "logps/rejected": -0.9447265863418579,
      "loss": 0.8486,
      "nll_loss": 0.7764648199081421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06995849311351776,
      "rewards/margins": 0.02445068396627903,
      "rewards/rejected": -0.09443359076976776,
      "step": 18940
    },
    {
      "epoch": 0.7101234752955725,
      "grad_norm": 2.0310959168741842,
      "learning_rate": 5.811461698337183e-07,
      "log_odds_chosen": 0.46375733613967896,
      "log_odds_ratio": -0.6151367425918579,
      "logits/chosen": -0.7689453363418579,
      "logits/rejected": -0.709179699420929,
      "logps/chosen": -0.67236328125,
      "logps/rejected": -0.9632812738418579,
      "loss": 0.8442,
      "nll_loss": 0.7808593511581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06719970703125,
      "rewards/margins": 0.02914886549115181,
      "rewards/rejected": -0.09639892727136612,
      "step": 18950
    },
    {
      "epoch": 0.7104982106387364,
      "grad_norm": 2.331296312913911,
      "learning_rate": 5.809928937741268e-07,
      "log_odds_chosen": 0.5328613519668579,
      "log_odds_ratio": -0.57470703125,
      "logits/chosen": -0.723437488079071,
      "logits/rejected": -0.625781238079071,
      "logps/chosen": -0.673535168170929,
      "logps/rejected": -0.9898437261581421,
      "loss": 0.8362,
      "nll_loss": 0.7662109136581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06741943210363388,
      "rewards/margins": 0.03162536770105362,
      "rewards/rejected": -0.09897460788488388,
      "step": 18960
    },
    {
      "epoch": 0.7108729459819003,
      "grad_norm": 2.4054483362041488,
      "learning_rate": 5.808397389292952e-07,
      "log_odds_chosen": 0.371551513671875,
      "log_odds_ratio": -0.6375976800918579,
      "logits/chosen": -0.7816406488418579,
      "logits/rejected": -0.664746105670929,
      "logps/chosen": -0.672656238079071,
      "logps/rejected": -0.911914050579071,
      "loss": 0.8312,
      "nll_loss": 0.788867175579071,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.06721191108226776,
      "rewards/margins": 0.02386474609375,
      "rewards/rejected": -0.09113769233226776,
      "step": 18970
    },
    {
      "epoch": 0.7112476813250642,
      "grad_norm": 2.50446588609306,
      "learning_rate": 5.806867051395413e-07,
      "log_odds_chosen": 0.584826648235321,
      "log_odds_ratio": -0.562695324420929,
      "logits/chosen": -0.724414050579071,
      "logits/rejected": -0.5970703363418579,
      "logps/chosen": -0.6317383050918579,
      "logps/rejected": -1.005468726158142,
      "loss": 0.8506,
      "nll_loss": 0.7593749761581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06318359076976776,
      "rewards/margins": 0.03738861158490181,
      "rewards/rejected": -0.10051269829273224,
      "step": 18980
    },
    {
      "epoch": 0.7116224166682281,
      "grad_norm": 2.3736840854735552,
      "learning_rate": 5.805337922454774e-07,
      "log_odds_chosen": 0.6011962890625,
      "log_odds_ratio": -0.5633789300918579,
      "logits/chosen": -0.69189453125,
      "logits/rejected": -0.605761706829071,
      "logps/chosen": -0.5863281488418579,
      "logps/rejected": -0.9564453363418579,
      "loss": 0.83,
      "nll_loss": 0.7525390386581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.05860595777630806,
      "rewards/margins": 0.03705291822552681,
      "rewards/rejected": -0.09569092094898224,
      "step": 18990
    },
    {
      "epoch": 0.7119971520113919,
      "grad_norm": 2.5313776899970186,
      "learning_rate": 5.803810000880093e-07,
      "log_odds_chosen": 0.6060791015625,
      "log_odds_ratio": -0.560546875,
      "logits/chosen": -0.771484375,
      "logits/rejected": -0.6631835699081421,
      "logps/chosen": -0.6680663824081421,
      "logps/rejected": -1.0457031726837158,
      "loss": 0.8327,
      "nll_loss": 0.746386706829071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06682129204273224,
      "rewards/margins": 0.03771362453699112,
      "rewards/rejected": -0.10458984225988388,
      "step": 19000
    },
    {
      "epoch": 0.7123718873545558,
      "grad_norm": 2.1143515475015873,
      "learning_rate": 5.802283285083356e-07,
      "log_odds_chosen": 0.626757800579071,
      "log_odds_ratio": -0.568066418170929,
      "logits/chosen": -0.7298828363418579,
      "logits/rejected": -0.634472668170929,
      "logps/chosen": -0.616503894329071,
      "logps/rejected": -1.0283203125,
      "loss": 0.8511,
      "nll_loss": 0.730175793170929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06171875074505806,
      "rewards/margins": 0.041229248046875,
      "rewards/rejected": -0.10286865383386612,
      "step": 19010
    },
    {
      "epoch": 0.7127466226977197,
      "grad_norm": 2.207491085765045,
      "learning_rate": 5.800757773479473e-07,
      "log_odds_chosen": 0.47386473417282104,
      "log_odds_ratio": -0.5982421636581421,
      "logits/chosen": -0.760937511920929,
      "logits/rejected": -0.632617175579071,
      "logps/chosen": -0.674023449420929,
      "logps/rejected": -0.984375,
      "loss": 0.8452,
      "nll_loss": 0.7757812738418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.0673828125,
      "rewards/margins": 0.03104858472943306,
      "rewards/rejected": -0.09833984076976776,
      "step": 19020
    },
    {
      "epoch": 0.7131213580408836,
      "grad_norm": 3.779966693060125,
      "learning_rate": 5.799233464486271e-07,
      "log_odds_chosen": 0.4858642518520355,
      "log_odds_ratio": -0.597363293170929,
      "logits/chosen": -0.768750011920929,
      "logits/rejected": -0.648632824420929,
      "logps/chosen": -0.6820312738418579,
      "logps/rejected": -1.008203148841858,
      "loss": 0.828,
      "nll_loss": 0.863476574420929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06820068508386612,
      "rewards/margins": 0.03267059475183487,
      "rewards/rejected": -0.100830078125,
      "step": 19030
    },
    {
      "epoch": 0.7134960933840475,
      "grad_norm": 2.3046109647821473,
      "learning_rate": 5.797710356524484e-07,
      "log_odds_chosen": 0.4922851622104645,
      "log_odds_ratio": -0.588183581829071,
      "logits/chosen": -0.755078136920929,
      "logits/rejected": -0.6368163824081421,
      "logps/chosen": -0.6742187738418579,
      "logps/rejected": -0.9857422113418579,
      "loss": 0.8411,
      "nll_loss": 0.8041015863418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06745605170726776,
      "rewards/margins": 0.03110961988568306,
      "rewards/rejected": -0.09858398139476776,
      "step": 19040
    },
    {
      "epoch": 0.7138708287272114,
      "grad_norm": 2.323825957525845,
      "learning_rate": 5.796188448017747e-07,
      "log_odds_chosen": 0.5102905035018921,
      "log_odds_ratio": -0.589160144329071,
      "logits/chosen": -0.754101574420929,
      "logits/rejected": -0.6622070074081421,
      "logps/chosen": -0.7095702886581421,
      "logps/rejected": -1.05078125,
      "loss": 0.8184,
      "nll_loss": 0.783398449420929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07087402045726776,
      "rewards/margins": 0.034072112292051315,
      "rewards/rejected": -0.10507812350988388,
      "step": 19050
    },
    {
      "epoch": 0.7142455640703753,
      "grad_norm": 2.5289225907191857,
      "learning_rate": 5.794667737392593e-07,
      "log_odds_chosen": 0.587158203125,
      "log_odds_ratio": -0.553906261920929,
      "logits/chosen": -0.7330077886581421,
      "logits/rejected": -0.62646484375,
      "logps/chosen": -0.639941394329071,
      "logps/rejected": -1.012109398841858,
      "loss": 0.833,
      "nll_loss": 0.7861328125,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06395263969898224,
      "rewards/margins": 0.03724823147058487,
      "rewards/rejected": -0.10128173977136612,
      "step": 19060
    },
    {
      "epoch": 0.7146202994135392,
      "grad_norm": 1.9987882411505045,
      "learning_rate": 5.793148223078442e-07,
      "log_odds_chosen": 0.595947265625,
      "log_odds_ratio": -0.556640625,
      "logits/chosen": -0.734179675579071,
      "logits/rejected": -0.644335925579071,
      "logps/chosen": -0.662109375,
      "logps/rejected": -1.022070288658142,
      "loss": 0.843,
      "nll_loss": 0.7733398675918579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06614990532398224,
      "rewards/margins": 0.035990141332149506,
      "rewards/rejected": -0.10219726711511612,
      "step": 19070
    },
    {
      "epoch": 0.7149950347567031,
      "grad_norm": 2.126556097906007,
      "learning_rate": 5.791629903507591e-07,
      "log_odds_chosen": 0.5648193359375,
      "log_odds_ratio": -0.566210925579071,
      "logits/chosen": -0.721875011920929,
      "logits/rejected": -0.616406261920929,
      "logps/chosen": -0.6661132574081421,
      "logps/rejected": -1.0154297351837158,
      "loss": 0.8529,
      "nll_loss": 0.761914074420929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06663818657398224,
      "rewards/margins": 0.03480834886431694,
      "rewards/rejected": -0.10139159858226776,
      "step": 19080
    },
    {
      "epoch": 0.715369770099867,
      "grad_norm": 2.088030693948063,
      "learning_rate": 5.79011277711522e-07,
      "log_odds_chosen": 0.3061767518520355,
      "log_odds_ratio": -0.691601574420929,
      "logits/chosen": -0.7398437261581421,
      "logits/rejected": -0.637011706829071,
      "logps/chosen": -0.7005859613418579,
      "logps/rejected": -0.9130859375,
      "loss": 0.8356,
      "nll_loss": 0.776562511920929,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07005615532398224,
      "rewards/margins": 0.02128448523581028,
      "rewards/rejected": -0.09138183295726776,
      "step": 19090
    },
    {
      "epoch": 0.7157445054430308,
      "grad_norm": 2.1853938768922228,
      "learning_rate": 5.788596842339373e-07,
      "log_odds_chosen": 0.46612548828125,
      "log_odds_ratio": -0.602343738079071,
      "logits/chosen": -0.756640613079071,
      "logits/rejected": -0.692187488079071,
      "logps/chosen": -0.657910168170929,
      "logps/rejected": -0.919921875,
      "loss": 0.823,
      "nll_loss": 0.7669922113418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06575927883386612,
      "rewards/margins": 0.026261139661073685,
      "rewards/rejected": -0.09199218451976776,
      "step": 19100
    },
    {
      "epoch": 0.7161192407861947,
      "grad_norm": 2.211962265178628,
      "learning_rate": 5.787082097620952e-07,
      "log_odds_chosen": 0.6033080816268921,
      "log_odds_ratio": -0.553417980670929,
      "logits/chosen": -0.784375011920929,
      "logits/rejected": -0.676953136920929,
      "logps/chosen": -0.6566406488418579,
      "logps/rejected": -1.03125,
      "loss": 0.8045,
      "nll_loss": 0.7611328363418579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06563720852136612,
      "rewards/margins": 0.03739471361041069,
      "rewards/rejected": -0.10307617485523224,
      "step": 19110
    },
    {
      "epoch": 0.7164939761293586,
      "grad_norm": 2.3974441323934688,
      "learning_rate": 5.785568541403717e-07,
      "log_odds_chosen": 0.26524657011032104,
      "log_odds_ratio": -0.6900390386581421,
      "logits/chosen": -0.7705078125,
      "logits/rejected": -0.653613269329071,
      "logps/chosen": -0.6783202886581421,
      "logps/rejected": -0.8658202886581421,
      "loss": 0.8548,
      "nll_loss": 0.7660156488418579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06784667819738388,
      "rewards/margins": 0.01873779296875,
      "rewards/rejected": -0.08653564751148224,
      "step": 19120
    },
    {
      "epoch": 0.7168687114725225,
      "grad_norm": 2.5341784024696365,
      "learning_rate": 5.784056172134274e-07,
      "log_odds_chosen": 0.4661498963832855,
      "log_odds_ratio": -0.59912109375,
      "logits/chosen": -0.8082031011581421,
      "logits/rejected": -0.678515613079071,
      "logps/chosen": -0.67578125,
      "logps/rejected": -0.988476574420929,
      "loss": 0.8376,
      "nll_loss": 0.750195324420929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06752929836511612,
      "rewards/margins": 0.03129119798541069,
      "rewards/rejected": -0.09885253757238388,
      "step": 19130
    },
    {
      "epoch": 0.7172434468156864,
      "grad_norm": 2.120865439425123,
      "learning_rate": 5.782544988262072e-07,
      "log_odds_chosen": 0.3650756776332855,
      "log_odds_ratio": -0.619335949420929,
      "logits/chosen": -0.7728515863418579,
      "logits/rejected": -0.6988281011581421,
      "logps/chosen": -0.662304699420929,
      "logps/rejected": -0.8929687738418579,
      "loss": 0.8369,
      "nll_loss": 0.8062499761581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06621094048023224,
      "rewards/margins": 0.02306976355612278,
      "rewards/rejected": -0.08927001804113388,
      "step": 19140
    },
    {
      "epoch": 0.7176181821588503,
      "grad_norm": 2.1102538723594026,
      "learning_rate": 5.781034988239392e-07,
      "log_odds_chosen": 0.44914549589157104,
      "log_odds_ratio": -0.5948241949081421,
      "logits/chosen": -0.704296886920929,
      "logits/rejected": -0.6551758050918579,
      "logps/chosen": -0.635546863079071,
      "logps/rejected": -0.9115234613418579,
      "loss": 0.8339,
      "nll_loss": 0.7548828125,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06351318210363388,
      "rewards/margins": 0.02759704552590847,
      "rewards/rejected": -0.09111328423023224,
      "step": 19150
    },
    {
      "epoch": 0.7179929175020142,
      "grad_norm": 2.6015439474262316,
      "learning_rate": 5.779526170521345e-07,
      "log_odds_chosen": 0.5486084222793579,
      "log_odds_ratio": -0.5791015625,
      "logits/chosen": -0.740234375,
      "logits/rejected": -0.656445324420929,
      "logps/chosen": -0.7417968511581421,
      "logps/rejected": -1.1064453125,
      "loss": 0.8675,
      "nll_loss": 0.798828125,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07418213039636612,
      "rewards/margins": 0.036529541015625,
      "rewards/rejected": -0.11064453423023224,
      "step": 19160
    },
    {
      "epoch": 0.7183676528451781,
      "grad_norm": 2.1641253168079513,
      "learning_rate": 5.778018533565859e-07,
      "log_odds_chosen": 0.532470703125,
      "log_odds_ratio": -0.583300769329071,
      "logits/chosen": -0.803515613079071,
      "logits/rejected": -0.665234386920929,
      "logps/chosen": -0.652636706829071,
      "logps/rejected": -0.986523449420929,
      "loss": 0.8081,
      "nll_loss": 0.7646484375,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06529541313648224,
      "rewards/margins": 0.03336029127240181,
      "rewards/rejected": -0.09858398139476776,
      "step": 19170
    },
    {
      "epoch": 0.718742388188342,
      "grad_norm": 2.2422776502638166,
      "learning_rate": 5.77651207583368e-07,
      "log_odds_chosen": 0.562792956829071,
      "log_odds_ratio": -0.575488269329071,
      "logits/chosen": -0.7310546636581421,
      "logits/rejected": -0.6454101800918579,
      "logps/chosen": -0.64599609375,
      "logps/rejected": -1.005859375,
      "loss": 0.8271,
      "nll_loss": 0.734375,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06458739936351776,
      "rewards/margins": 0.0360565185546875,
      "rewards/rejected": -0.10061035305261612,
      "step": 19180
    },
    {
      "epoch": 0.7191171235315059,
      "grad_norm": 3.7732143679221646,
      "learning_rate": 5.775006795788362e-07,
      "log_odds_chosen": 0.636474609375,
      "log_odds_ratio": -0.5513671636581421,
      "logits/chosen": -0.7705078125,
      "logits/rejected": -0.635546863079071,
      "logps/chosen": -0.6343749761581421,
      "logps/rejected": -1.0419921875,
      "loss": 0.8307,
      "nll_loss": 0.7451171875,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06348876655101776,
      "rewards/margins": 0.04061584547162056,
      "rewards/rejected": -0.1041259765625,
      "step": 19190
    },
    {
      "epoch": 0.7194918588746697,
      "grad_norm": 2.9210872168037536,
      "learning_rate": 5.773502691896258e-07,
      "log_odds_chosen": 0.4737792909145355,
      "log_odds_ratio": -0.5985351800918579,
      "logits/chosen": -0.6849609613418579,
      "logits/rejected": -0.619824230670929,
      "logps/chosen": -0.6656249761581421,
      "logps/rejected": -0.954882800579071,
      "loss": 0.8343,
      "nll_loss": 0.791210949420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06650390475988388,
      "rewards/margins": 0.02899169921875,
      "rewards/rejected": -0.09550781548023224,
      "step": 19200
    },
    {
      "epoch": 0.7198665942178336,
      "grad_norm": 2.296997561825158,
      "learning_rate": 5.771999762626513e-07,
      "log_odds_chosen": 0.52606201171875,
      "log_odds_ratio": -0.584277331829071,
      "logits/chosen": -0.7274414300918579,
      "logits/rejected": -0.599316418170929,
      "logps/chosen": -0.676562488079071,
      "logps/rejected": -1.0412108898162842,
      "loss": 0.837,
      "nll_loss": 0.813281238079071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06772460788488388,
      "rewards/margins": 0.03639068454504013,
      "rewards/rejected": -0.10408935695886612,
      "step": 19210
    },
    {
      "epoch": 0.7202413295609975,
      "grad_norm": 2.090444195100521,
      "learning_rate": 5.77049800645107e-07,
      "log_odds_chosen": 0.6409667730331421,
      "log_odds_ratio": -0.551562488079071,
      "logits/chosen": -0.7422851324081421,
      "logits/rejected": -0.628125011920929,
      "logps/chosen": -0.639453113079071,
      "logps/rejected": -1.042382836341858,
      "loss": 0.8273,
      "nll_loss": 0.798828125,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06395263969898224,
      "rewards/margins": 0.04034423828125,
      "rewards/rejected": -0.10422363132238388,
      "step": 19220
    },
    {
      "epoch": 0.7206160649041614,
      "grad_norm": 2.3485133781428194,
      "learning_rate": 5.768997421844641e-07,
      "log_odds_chosen": 0.518078625202179,
      "log_odds_ratio": -0.6048828363418579,
      "logits/chosen": -0.7572265863418579,
      "logits/rejected": -0.6543945074081421,
      "logps/chosen": -0.6763671636581421,
      "logps/rejected": -1.0146484375,
      "loss": 0.8434,
      "nll_loss": 0.810742199420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06766357272863388,
      "rewards/margins": 0.03390350192785263,
      "rewards/rejected": -0.10153808444738388,
      "step": 19230
    },
    {
      "epoch": 0.7209908002473253,
      "grad_norm": 2.6572238472666663,
      "learning_rate": 5.767498007284723e-07,
      "log_odds_chosen": 0.37141114473342896,
      "log_odds_ratio": -0.640625,
      "logits/chosen": -0.741992175579071,
      "logits/rejected": -0.6675781011581421,
      "logps/chosen": -0.74560546875,
      "logps/rejected": -0.9658203125,
      "loss": 0.8194,
      "nll_loss": 0.8099609613418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07456054538488388,
      "rewards/margins": 0.022064208984375,
      "rewards/rejected": -0.0965576171875,
      "step": 19240
    },
    {
      "epoch": 0.7213655355904892,
      "grad_norm": 3.3152478617524173,
      "learning_rate": 5.765999761251576e-07,
      "log_odds_chosen": 0.5404297113418579,
      "log_odds_ratio": -0.586718738079071,
      "logits/chosen": -0.7108398675918579,
      "logits/rejected": -0.602832019329071,
      "logps/chosen": -0.652539074420929,
      "logps/rejected": -0.996289074420929,
      "loss": 0.8243,
      "nll_loss": 0.760449230670929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06529541313648224,
      "rewards/margins": 0.034377288073301315,
      "rewards/rejected": -0.099609375,
      "step": 19250
    },
    {
      "epoch": 0.7217402709336531,
      "grad_norm": 2.320513639434486,
      "learning_rate": 5.764502682228225e-07,
      "log_odds_chosen": 0.42717283964157104,
      "log_odds_ratio": -0.614062488079071,
      "logits/chosen": -0.7255859375,
      "logits/rejected": -0.6214843988418579,
      "logps/chosen": -0.6949218511581421,
      "logps/rejected": -0.9593750238418579,
      "loss": 0.8339,
      "nll_loss": 0.773632824420929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06944580376148224,
      "rewards/margins": 0.02644653245806694,
      "rewards/rejected": -0.09591064602136612,
      "step": 19260
    },
    {
      "epoch": 0.722115006276817,
      "grad_norm": 2.8814192411878254,
      "learning_rate": 5.763006768700448e-07,
      "log_odds_chosen": 0.5081421136856079,
      "log_odds_ratio": -0.592089831829071,
      "logits/chosen": -0.6893554925918579,
      "logits/rejected": -0.6332031488418579,
      "logps/chosen": -0.7040039300918579,
      "logps/rejected": -1.03466796875,
      "loss": 0.835,
      "nll_loss": 0.812695324420929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07044677436351776,
      "rewards/margins": 0.03305358812212944,
      "rewards/rejected": -0.103515625,
      "step": 19270
    },
    {
      "epoch": 0.7224897416199809,
      "grad_norm": 2.3384047294207804,
      "learning_rate": 5.761512019156773e-07,
      "log_odds_chosen": 0.5480712652206421,
      "log_odds_ratio": -0.579296886920929,
      "logits/chosen": -0.748242199420929,
      "logits/rejected": -0.6141601800918579,
      "logps/chosen": -0.6827148199081421,
      "logps/rejected": -1.01171875,
      "loss": 0.8261,
      "nll_loss": 0.8111327886581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06827392429113388,
      "rewards/margins": 0.03289489820599556,
      "rewards/rejected": -0.10112304985523224,
      "step": 19280
    },
    {
      "epoch": 0.7228644769631448,
      "grad_norm": 2.787803006151359,
      "learning_rate": 5.760018432088474e-07,
      "log_odds_chosen": 0.60296630859375,
      "log_odds_ratio": -0.563671886920929,
      "logits/chosen": -0.7533203363418579,
      "logits/rejected": -0.641796886920929,
      "logps/chosen": -0.660937488079071,
      "logps/rejected": -0.9955078363418579,
      "loss": 0.8351,
      "nll_loss": 0.783984363079071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06602783501148224,
      "rewards/margins": 0.033457182347774506,
      "rewards/rejected": -0.09951172024011612,
      "step": 19290
    },
    {
      "epoch": 0.7232392123063087,
      "grad_norm": 2.1498022438935434,
      "learning_rate": 5.758526005989556e-07,
      "log_odds_chosen": 0.46807861328125,
      "log_odds_ratio": -0.5923827886581421,
      "logits/chosen": -0.6688476800918579,
      "logits/rejected": -0.59619140625,
      "logps/chosen": -0.69970703125,
      "logps/rejected": -0.9888671636581421,
      "loss": 0.8374,
      "nll_loss": 0.78173828125,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06997070461511612,
      "rewards/margins": 0.02887725830078125,
      "rewards/rejected": -0.09880371391773224,
      "step": 19300
    },
    {
      "epoch": 0.7236139476494725,
      "grad_norm": 2.244108663258082,
      "learning_rate": 5.757034739356758e-07,
      "log_odds_chosen": 0.39042967557907104,
      "log_odds_ratio": -0.622265636920929,
      "logits/chosen": -0.733593761920929,
      "logits/rejected": -0.6343749761581421,
      "logps/chosen": -0.6578124761581421,
      "logps/rejected": -0.928515613079071,
      "loss": 0.8585,
      "nll_loss": 0.7933593988418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06578369438648224,
      "rewards/margins": 0.02705993689596653,
      "rewards/rejected": -0.09279785305261612,
      "step": 19310
    },
    {
      "epoch": 0.7239886829926364,
      "grad_norm": 2.267663504201853,
      "learning_rate": 5.755544630689541e-07,
      "log_odds_chosen": 0.5384765863418579,
      "log_odds_ratio": -0.583300769329071,
      "logits/chosen": -0.7578125,
      "logits/rejected": -0.660351574420929,
      "logps/chosen": -0.68212890625,
      "logps/rejected": -1.004296898841858,
      "loss": 0.8458,
      "nll_loss": 0.7865234613418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06832275539636612,
      "rewards/margins": 0.03220214694738388,
      "rewards/rejected": -0.10043945163488388,
      "step": 19320
    },
    {
      "epoch": 0.7243634183358003,
      "grad_norm": 1.9894545752106454,
      "learning_rate": 5.754055678490085e-07,
      "log_odds_chosen": 0.532971203327179,
      "log_odds_ratio": -0.6006835699081421,
      "logits/chosen": -0.7476562261581421,
      "logits/rejected": -0.6219726800918579,
      "logps/chosen": -0.6751953363418579,
      "logps/rejected": -1.025976538658142,
      "loss": 0.8457,
      "nll_loss": 0.772265613079071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06752929836511612,
      "rewards/margins": 0.035161592066287994,
      "rewards/rejected": -0.10268554836511612,
      "step": 19330
    },
    {
      "epoch": 0.7247381536789642,
      "grad_norm": 2.1112450632754993,
      "learning_rate": 5.752567881263278e-07,
      "log_odds_chosen": 0.2709594666957855,
      "log_odds_ratio": -0.6566406488418579,
      "logits/chosen": -0.755859375,
      "logits/rejected": -0.6773437261581421,
      "logps/chosen": -0.6854492425918579,
      "logps/rejected": -0.848437488079071,
      "loss": 0.8207,
      "nll_loss": 0.834765613079071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06856689602136612,
      "rewards/margins": 0.01626892015337944,
      "rewards/rejected": -0.0848388671875,
      "step": 19340
    },
    {
      "epoch": 0.7251128890221281,
      "grad_norm": 2.593152174239538,
      "learning_rate": 5.751081237516715e-07,
      "log_odds_chosen": 0.483154296875,
      "log_odds_ratio": -0.600292980670929,
      "logits/chosen": -0.817187488079071,
      "logits/rejected": -0.68896484375,
      "logps/chosen": -0.662792980670929,
      "logps/rejected": -0.9781249761581421,
      "loss": 0.8189,
      "nll_loss": 0.787304699420929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06636963039636612,
      "rewards/margins": 0.03148040920495987,
      "rewards/rejected": -0.09780273586511612,
      "step": 19350
    },
    {
      "epoch": 0.725487624365292,
      "grad_norm": 2.188765293448177,
      "learning_rate": 5.74959574576069e-07,
      "log_odds_chosen": 0.4963012635707855,
      "log_odds_ratio": -0.6170898675918579,
      "logits/chosen": -0.7671874761581421,
      "logits/rejected": -0.6431640386581421,
      "logps/chosen": -0.630566418170929,
      "logps/rejected": -0.9361327886581421,
      "loss": 0.8446,
      "nll_loss": 0.8013671636581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06303711235523224,
      "rewards/margins": 0.03060150146484375,
      "rewards/rejected": -0.09364013373851776,
      "step": 19360
    },
    {
      "epoch": 0.7258623597084559,
      "grad_norm": 2.1251218295295446,
      "learning_rate": 5.748111404508186e-07,
      "log_odds_chosen": 0.5419921875,
      "log_odds_ratio": -0.6119140386581421,
      "logits/chosen": -0.7837890386581421,
      "logits/rejected": -0.6587890386581421,
      "logps/chosen": -0.711132824420929,
      "logps/rejected": -1.075585961341858,
      "loss": 0.8527,
      "nll_loss": 0.8232421875,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07113037258386612,
      "rewards/margins": 0.03635406494140625,
      "rewards/rejected": -0.10756836086511612,
      "step": 19370
    },
    {
      "epoch": 0.7262370950516198,
      "grad_norm": 2.3822854653266416,
      "learning_rate": 5.746628212274873e-07,
      "log_odds_chosen": 0.6114501953125,
      "log_odds_ratio": -0.5438476800918579,
      "logits/chosen": -0.793164074420929,
      "logits/rejected": -0.6678711175918579,
      "logps/chosen": -0.667187511920929,
      "logps/rejected": -1.0353515148162842,
      "loss": 0.8468,
      "nll_loss": 0.785937488079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06667480617761612,
      "rewards/margins": 0.03682861477136612,
      "rewards/rejected": -0.10349120944738388,
      "step": 19380
    },
    {
      "epoch": 0.7266118303947837,
      "grad_norm": 2.174207870394938,
      "learning_rate": 5.745146167579106e-07,
      "log_odds_chosen": 0.4980224668979645,
      "log_odds_ratio": -0.6073242425918579,
      "logits/chosen": -0.765429675579071,
      "logits/rejected": -0.7010742425918579,
      "logps/chosen": -0.6664062738418579,
      "logps/rejected": -0.9644531011581421,
      "loss": 0.8286,
      "nll_loss": 0.791015625,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06668701022863388,
      "rewards/margins": 0.02991638146340847,
      "rewards/rejected": -0.09653320163488388,
      "step": 19390
    },
    {
      "epoch": 0.7269865657379476,
      "grad_norm": 2.561256823370949,
      "learning_rate": 5.743665268941905e-07,
      "log_odds_chosen": 0.3842407166957855,
      "log_odds_ratio": -0.640917956829071,
      "logits/chosen": -0.745898425579071,
      "logits/rejected": -0.6259765625,
      "logps/chosen": -0.6983398199081421,
      "logps/rejected": -0.924609363079071,
      "loss": 0.8484,
      "nll_loss": 0.805859386920929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06981201469898224,
      "rewards/margins": 0.022613525390625,
      "rewards/rejected": -0.09246826171875,
      "step": 19400
    },
    {
      "epoch": 0.7273613010811114,
      "grad_norm": 2.622599237744655,
      "learning_rate": 5.742185514886961e-07,
      "log_odds_chosen": 0.656445324420929,
      "log_odds_ratio": -0.5263671875,
      "logits/chosen": -0.759570300579071,
      "logits/rejected": -0.640917956829071,
      "logps/chosen": -0.6641601324081421,
      "logps/rejected": -1.065039038658142,
      "loss": 0.837,
      "nll_loss": 0.788281261920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06645508110523224,
      "rewards/margins": 0.040026091039180756,
      "rewards/rejected": -0.10639648139476776,
      "step": 19410
    },
    {
      "epoch": 0.7277360364242753,
      "grad_norm": 2.5583035004642576,
      "learning_rate": 5.740706903940628e-07,
      "log_odds_chosen": 0.39299315214157104,
      "log_odds_ratio": -0.6322265863418579,
      "logits/chosen": -0.755859375,
      "logits/rejected": -0.6484375,
      "logps/chosen": -0.7206054925918579,
      "logps/rejected": -0.955078125,
      "loss": 0.8383,
      "nll_loss": 0.8277343511581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.072021484375,
      "rewards/margins": 0.02341156080365181,
      "rewards/rejected": -0.09548339992761612,
      "step": 19420
    },
    {
      "epoch": 0.7281107717674392,
      "grad_norm": 2.534723146080455,
      "learning_rate": 5.739229434631911e-07,
      "log_odds_chosen": 0.569531261920929,
      "log_odds_ratio": -0.56494140625,
      "logits/chosen": -0.800585925579071,
      "logits/rejected": -0.687695324420929,
      "logps/chosen": -0.6927734613418579,
      "logps/rejected": -1.067968726158142,
      "loss": 0.8523,
      "nll_loss": 0.748828113079071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06927490234375,
      "rewards/margins": 0.03753662109375,
      "rewards/rejected": -0.1068115234375,
      "step": 19430
    },
    {
      "epoch": 0.7284855071106031,
      "grad_norm": 2.5497502441413453,
      "learning_rate": 5.737753105492469e-07,
      "log_odds_chosen": 0.46319580078125,
      "log_odds_ratio": -0.5977538824081421,
      "logits/chosen": -0.7822265625,
      "logits/rejected": -0.670214831829071,
      "logps/chosen": -0.684374988079071,
      "logps/rejected": -1.0007812976837158,
      "loss": 0.8332,
      "nll_loss": 0.7730468511581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06839599460363388,
      "rewards/margins": 0.03165893629193306,
      "rewards/rejected": -0.10007324069738388,
      "step": 19440
    },
    {
      "epoch": 0.728860242453767,
      "grad_norm": 2.103437902843724,
      "learning_rate": 5.736277915056597e-07,
      "log_odds_chosen": 0.5653320550918579,
      "log_odds_ratio": -0.559521496295929,
      "logits/chosen": -0.790234386920929,
      "logits/rejected": -0.674511730670929,
      "logps/chosen": -0.7035156488418579,
      "logps/rejected": -1.057226538658142,
      "loss": 0.8545,
      "nll_loss": 0.7718750238418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.0703125,
      "rewards/margins": 0.03533325344324112,
      "rewards/rejected": -0.10573730617761612,
      "step": 19450
    },
    {
      "epoch": 0.7292349777969309,
      "grad_norm": 2.0863432355900486,
      "learning_rate": 5.734803861861232e-07,
      "log_odds_chosen": 0.593017578125,
      "log_odds_ratio": -0.5733398199081421,
      "logits/chosen": -0.7875000238418579,
      "logits/rejected": -0.6180664300918579,
      "logps/chosen": -0.7152343988418579,
      "logps/rejected": -1.110742211341858,
      "loss": 0.8482,
      "nll_loss": 0.8238281011581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07156982272863388,
      "rewards/margins": 0.03948974609375,
      "rewards/rejected": -0.11098632961511612,
      "step": 19460
    },
    {
      "epoch": 0.7296097131400948,
      "grad_norm": 2.341384452905344,
      "learning_rate": 5.733330944445938e-07,
      "log_odds_chosen": 0.4283203184604645,
      "log_odds_ratio": -0.6075195074081421,
      "logits/chosen": -0.7826172113418579,
      "logits/rejected": -0.669628918170929,
      "logps/chosen": -0.674609363079071,
      "logps/rejected": -0.95654296875,
      "loss": 0.8452,
      "nll_loss": 0.8310546875,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06744384765625,
      "rewards/margins": 0.028275299817323685,
      "rewards/rejected": -0.09571532905101776,
      "step": 19470
    },
    {
      "epoch": 0.7299844484832587,
      "grad_norm": 2.2318828345215276,
      "learning_rate": 5.731859161352903e-07,
      "log_odds_chosen": 0.42064207792282104,
      "log_odds_ratio": -0.639843761920929,
      "logits/chosen": -0.8197265863418579,
      "logits/rejected": -0.6787109375,
      "logps/chosen": -0.723437488079071,
      "logps/rejected": -1.030664086341858,
      "loss": 0.8306,
      "nll_loss": 0.804882824420929,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07231445610523224,
      "rewards/margins": 0.03067321702837944,
      "rewards/rejected": -0.10297851264476776,
      "step": 19480
    },
    {
      "epoch": 0.7303591838264226,
      "grad_norm": 2.2598661529574904,
      "learning_rate": 5.730388511126939e-07,
      "log_odds_chosen": 0.542041003704071,
      "log_odds_ratio": -0.5694335699081421,
      "logits/chosen": -0.7269531488418579,
      "logits/rejected": -0.625781238079071,
      "logps/chosen": -0.6744140386581421,
      "logps/rejected": -1.0222656726837158,
      "loss": 0.8312,
      "nll_loss": 0.7613281011581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06741943210363388,
      "rewards/margins": 0.03478698804974556,
      "rewards/rejected": -0.10224609076976776,
      "step": 19490
    },
    {
      "epoch": 0.7307339191695865,
      "grad_norm": 2.6250524626420098,
      "learning_rate": 5.728918992315463e-07,
      "log_odds_chosen": 0.6166626214981079,
      "log_odds_ratio": -0.571093738079071,
      "logits/chosen": -0.775585949420929,
      "logits/rejected": -0.652148425579071,
      "logps/chosen": -0.695996105670929,
      "logps/rejected": -1.108007788658142,
      "loss": 0.8192,
      "nll_loss": 0.759472668170929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06955566257238388,
      "rewards/margins": 0.04117431491613388,
      "rewards/rejected": -0.11074218899011612,
      "step": 19500
    },
    {
      "epoch": 0.7311086545127504,
      "grad_norm": 2.3582998148806693,
      "learning_rate": 5.727450603468501e-07,
      "log_odds_chosen": 0.509106457233429,
      "log_odds_ratio": -0.5977538824081421,
      "logits/chosen": -0.7015625238418579,
      "logits/rejected": -0.612500011920929,
      "logps/chosen": -0.692187488079071,
      "logps/rejected": -1.0203125476837158,
      "loss": 0.8227,
      "nll_loss": 0.7403320074081421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0692138671875,
      "rewards/margins": 0.0328826904296875,
      "rewards/rejected": -0.10214843600988388,
      "step": 19510
    },
    {
      "epoch": 0.7314833898559142,
      "grad_norm": 2.445702522919454,
      "learning_rate": 5.725983343138682e-07,
      "log_odds_chosen": 0.47407227754592896,
      "log_odds_ratio": -0.604785144329071,
      "logits/chosen": -0.7466796636581421,
      "logits/rejected": -0.637499988079071,
      "logps/chosen": -0.652148425579071,
      "logps/rejected": -0.966015636920929,
      "loss": 0.8246,
      "nll_loss": 0.792773425579071,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.06523437798023224,
      "rewards/margins": 0.031316377222537994,
      "rewards/rejected": -0.09660644829273224,
      "step": 19520
    },
    {
      "epoch": 0.7318581251990781,
      "grad_norm": 2.5146488883540057,
      "learning_rate": 5.724517209881224e-07,
      "log_odds_chosen": 0.46917724609375,
      "log_odds_ratio": -0.620312511920929,
      "logits/chosen": -0.8042968511581421,
      "logits/rejected": -0.6753906011581421,
      "logps/chosen": -0.7119140625,
      "logps/rejected": -1.0382812023162842,
      "loss": 0.848,
      "nll_loss": 0.8404296636581421,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07117919623851776,
      "rewards/margins": 0.03256683424115181,
      "rewards/rejected": -0.10373535007238388,
      "step": 19530
    },
    {
      "epoch": 0.732232860542242,
      "grad_norm": 2.275782792895197,
      "learning_rate": 5.723052202253938e-07,
      "log_odds_chosen": 0.5515381097793579,
      "log_odds_ratio": -0.6015625,
      "logits/chosen": -0.7572265863418579,
      "logits/rejected": -0.615429699420929,
      "logps/chosen": -0.630664050579071,
      "logps/rejected": -0.9994140863418579,
      "loss": 0.8475,
      "nll_loss": 0.798046886920929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06309814751148224,
      "rewards/margins": 0.03681640699505806,
      "rewards/rejected": -0.0999755859375,
      "step": 19540
    },
    {
      "epoch": 0.7326075958854059,
      "grad_norm": 2.6112782495078597,
      "learning_rate": 5.721588318817212e-07,
      "log_odds_chosen": 0.687939465045929,
      "log_odds_ratio": -0.5282226800918579,
      "logits/chosen": -0.729687511920929,
      "logits/rejected": -0.632617175579071,
      "logps/chosen": -0.6220703125,
      "logps/rejected": -1.0498046875,
      "loss": 0.8618,
      "nll_loss": 0.724609375,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06224365159869194,
      "rewards/margins": 0.04276733472943306,
      "rewards/rejected": -0.10502929985523224,
      "step": 19550
    },
    {
      "epoch": 0.7329823312285698,
      "grad_norm": 2.0986359971855193,
      "learning_rate": 5.720125558134017e-07,
      "log_odds_chosen": 0.512744128704071,
      "log_odds_ratio": -0.6015625,
      "logits/chosen": -0.820117175579071,
      "logits/rejected": -0.717578113079071,
      "logps/chosen": -0.703417956829071,
      "logps/rejected": -1.0476562976837158,
      "loss": 0.8278,
      "nll_loss": 0.789257824420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07038573920726776,
      "rewards/margins": 0.034332275390625,
      "rewards/rejected": -0.10471191257238388,
      "step": 19560
    },
    {
      "epoch": 0.7333570665717337,
      "grad_norm": 1.954462910817388,
      "learning_rate": 5.718663918769888e-07,
      "log_odds_chosen": 0.565997302532196,
      "log_odds_ratio": -0.565136730670929,
      "logits/chosen": -0.7876952886581421,
      "logits/rejected": -0.6888672113418579,
      "logps/chosen": -0.691210925579071,
      "logps/rejected": -1.0431640148162842,
      "loss": 0.84,
      "nll_loss": 0.8238281011581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06912841647863388,
      "rewards/margins": 0.03514404222369194,
      "rewards/rejected": -0.10429687798023224,
      "step": 19570
    },
    {
      "epoch": 0.7337318019148976,
      "grad_norm": 2.2325792223374883,
      "learning_rate": 5.717203399292928e-07,
      "log_odds_chosen": 0.6265624761581421,
      "log_odds_ratio": -0.557910144329071,
      "logits/chosen": -0.769238293170929,
      "logits/rejected": -0.6636718511581421,
      "logps/chosen": -0.674609363079071,
      "logps/rejected": -1.065820336341858,
      "loss": 0.8329,
      "nll_loss": 0.7759765386581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06752929836511612,
      "rewards/margins": 0.039093017578125,
      "rewards/rejected": -0.10659179836511612,
      "step": 19580
    },
    {
      "epoch": 0.7341065372580615,
      "grad_norm": 2.355354281716476,
      "learning_rate": 5.7157439982738e-07,
      "log_odds_chosen": 0.5952392816543579,
      "log_odds_ratio": -0.575390636920929,
      "logits/chosen": -0.749218761920929,
      "logits/rejected": -0.617871105670929,
      "logps/chosen": -0.6767578125,
      "logps/rejected": -1.063085913658142,
      "loss": 0.8318,
      "nll_loss": 0.751171886920929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06766357272863388,
      "rewards/margins": 0.03869018703699112,
      "rewards/rejected": -0.10640869289636612,
      "step": 19590
    },
    {
      "epoch": 0.7344812726012254,
      "grad_norm": 2.31810457759115,
      "learning_rate": 5.714285714285714e-07,
      "log_odds_chosen": 0.5916503667831421,
      "log_odds_ratio": -0.5704101324081421,
      "logits/chosen": -0.731249988079071,
      "logits/rejected": -0.6205078363418579,
      "logps/chosen": -0.6675781011581421,
      "logps/rejected": -1.0769531726837158,
      "loss": 0.8326,
      "nll_loss": 0.7222656011581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06674804538488388,
      "rewards/margins": 0.041016388684511185,
      "rewards/rejected": -0.10773925483226776,
      "step": 19600
    },
    {
      "epoch": 0.7348560079443893,
      "grad_norm": 2.4343106773975864,
      "learning_rate": 5.712828545904434e-07,
      "log_odds_chosen": 0.5052490234375,
      "log_odds_ratio": -0.60400390625,
      "logits/chosen": -0.7256835699081421,
      "logits/rejected": -0.645312488079071,
      "logps/chosen": -0.692187488079071,
      "logps/rejected": -1.0255858898162842,
      "loss": 0.8336,
      "nll_loss": 0.772656261920929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06918945163488388,
      "rewards/margins": 0.03330879285931587,
      "rewards/rejected": -0.10251464694738388,
      "step": 19610
    },
    {
      "epoch": 0.7352307432875531,
      "grad_norm": 2.4483787354633404,
      "learning_rate": 5.711372491708257e-07,
      "log_odds_chosen": 0.64080810546875,
      "log_odds_ratio": -0.548632800579071,
      "logits/chosen": -0.785351574420929,
      "logits/rejected": -0.6917968988418579,
      "logps/chosen": -0.6630859375,
      "logps/rejected": -1.0750000476837158,
      "loss": 0.8186,
      "nll_loss": 0.787109375,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06627197563648224,
      "rewards/margins": 0.04119415208697319,
      "rewards/rejected": -0.107421875,
      "step": 19620
    },
    {
      "epoch": 0.735605478630717,
      "grad_norm": 2.352742188034276,
      "learning_rate": 5.709917550278023e-07,
      "log_odds_chosen": 0.5583862066268921,
      "log_odds_ratio": -0.577343761920929,
      "logits/chosen": -0.726757824420929,
      "logits/rejected": -0.65234375,
      "logps/chosen": -0.657421886920929,
      "logps/rejected": -1.0285155773162842,
      "loss": 0.8463,
      "nll_loss": 0.782421886920929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06571044772863388,
      "rewards/margins": 0.03710021823644638,
      "rewards/rejected": -0.10283203423023224,
      "step": 19630
    },
    {
      "epoch": 0.7359802139738809,
      "grad_norm": 2.1463634534406313,
      "learning_rate": 5.7084637201971e-07,
      "log_odds_chosen": 0.42242431640625,
      "log_odds_ratio": -0.609179675579071,
      "logits/chosen": -0.7779296636581421,
      "logits/rejected": -0.671093761920929,
      "logps/chosen": -0.6817382574081421,
      "logps/rejected": -0.958984375,
      "loss": 0.8334,
      "nll_loss": 0.74267578125,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06815185397863388,
      "rewards/margins": 0.02780303917825222,
      "rewards/rejected": -0.095947265625,
      "step": 19640
    },
    {
      "epoch": 0.7363549493170448,
      "grad_norm": 2.5827980322763513,
      "learning_rate": 5.707011000051373e-07,
      "log_odds_chosen": 0.36016845703125,
      "log_odds_ratio": -0.6357421875,
      "logits/chosen": -0.770312488079071,
      "logits/rejected": -0.6927734613418579,
      "logps/chosen": -0.680859386920929,
      "logps/rejected": -0.9091796875,
      "loss": 0.835,
      "nll_loss": 0.743945300579071,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06804199516773224,
      "rewards/margins": 0.02282562293112278,
      "rewards/rejected": -0.09080810844898224,
      "step": 19650
    },
    {
      "epoch": 0.7367296846602087,
      "grad_norm": 2.3148003640893733,
      "learning_rate": 5.705559388429252e-07,
      "log_odds_chosen": 0.528369128704071,
      "log_odds_ratio": -0.5750976800918579,
      "logits/chosen": -0.7289062738418579,
      "logits/rejected": -0.6258789300918579,
      "logps/chosen": -0.6294921636581421,
      "logps/rejected": -0.948437511920929,
      "loss": 0.8406,
      "nll_loss": 0.777539074420929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06295166164636612,
      "rewards/margins": 0.031894683837890625,
      "rewards/rejected": -0.09483642876148224,
      "step": 19660
    },
    {
      "epoch": 0.7371044200033726,
      "grad_norm": 1.8994429162726747,
      "learning_rate": 5.704108883921655e-07,
      "log_odds_chosen": 0.4441894590854645,
      "log_odds_ratio": -0.6094726324081421,
      "logits/chosen": -0.723437488079071,
      "logits/rejected": -0.642285168170929,
      "logps/chosen": -0.6666015386581421,
      "logps/rejected": -0.932421863079071,
      "loss": 0.8351,
      "nll_loss": 0.7547851800918579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06658935546875,
      "rewards/margins": 0.02658233605325222,
      "rewards/rejected": -0.09317626804113388,
      "step": 19670
    },
    {
      "epoch": 0.7374791553465365,
      "grad_norm": 2.222881509831486,
      "learning_rate": 5.70265948512201e-07,
      "log_odds_chosen": 0.6218506097793579,
      "log_odds_ratio": -0.57080078125,
      "logits/chosen": -0.7235351800918579,
      "logits/rejected": -0.598339855670929,
      "logps/chosen": -0.6737304925918579,
      "logps/rejected": -1.1005859375,
      "loss": 0.8209,
      "nll_loss": 0.7763671875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06733398139476776,
      "rewards/margins": 0.04274139553308487,
      "rewards/rejected": -0.11013183742761612,
      "step": 19680
    },
    {
      "epoch": 0.7378538906897004,
      "grad_norm": 2.6470457012942816,
      "learning_rate": 5.701211190626241e-07,
      "log_odds_chosen": 0.4945434629917145,
      "log_odds_ratio": -0.6166015863418579,
      "logits/chosen": -0.758984386920929,
      "logits/rejected": -0.6722656488418579,
      "logps/chosen": -0.6654297113418579,
      "logps/rejected": -0.971875011920929,
      "loss": 0.8303,
      "nll_loss": 0.8115234375,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06660155951976776,
      "rewards/margins": 0.030558014288544655,
      "rewards/rejected": -0.09714355319738388,
      "step": 19690
    },
    {
      "epoch": 0.7382286260328643,
      "grad_norm": 2.13091883533283,
      "learning_rate": 5.699763999032772e-07,
      "log_odds_chosen": 0.598187267780304,
      "log_odds_ratio": -0.546582043170929,
      "logits/chosen": -0.7494140863418579,
      "logits/rejected": -0.685742199420929,
      "logps/chosen": -0.6744140386581421,
      "logps/rejected": -1.0537109375,
      "loss": 0.8256,
      "nll_loss": 0.7255859375,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06743164360523224,
      "rewards/margins": 0.037897489964962006,
      "rewards/rejected": -0.10549316555261612,
      "step": 19700
    },
    {
      "epoch": 0.7386033613760282,
      "grad_norm": 2.1132101472269693,
      "learning_rate": 5.69831790894251e-07,
      "log_odds_chosen": 0.4105224609375,
      "log_odds_ratio": -0.6407226324081421,
      "logits/chosen": -0.7417968511581421,
      "logits/rejected": -0.66064453125,
      "logps/chosen": -0.688281238079071,
      "logps/rejected": -0.9566406011581421,
      "loss": 0.8167,
      "nll_loss": 0.7767578363418579,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06878662109375,
      "rewards/margins": 0.02683868445456028,
      "rewards/rejected": -0.09553222358226776,
      "step": 19710
    },
    {
      "epoch": 0.7389780967191921,
      "grad_norm": 2.3027238164632804,
      "learning_rate": 5.69687291895885e-07,
      "log_odds_chosen": 0.592480480670929,
      "log_odds_ratio": -0.563281238079071,
      "logits/chosen": -0.7779296636581421,
      "logits/rejected": -0.6675781011581421,
      "logps/chosen": -0.6822265386581421,
      "logps/rejected": -1.054101586341858,
      "loss": 0.8466,
      "nll_loss": 0.8091796636581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06824950873851776,
      "rewards/margins": 0.03710632398724556,
      "rewards/rejected": -0.10537109524011612,
      "step": 19720
    },
    {
      "epoch": 0.7393528320623559,
      "grad_norm": 2.243800706636353,
      "learning_rate": 5.695429027687665e-07,
      "log_odds_chosen": 0.38837891817092896,
      "log_odds_ratio": -0.6114257574081421,
      "logits/chosen": -0.767382800579071,
      "logits/rejected": -0.69091796875,
      "logps/chosen": -0.700390636920929,
      "logps/rejected": -0.9574218988418579,
      "loss": 0.8277,
      "nll_loss": 0.8173828125,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07004394382238388,
      "rewards/margins": 0.02572936937212944,
      "rewards/rejected": -0.09572754055261612,
      "step": 19730
    },
    {
      "epoch": 0.7397275674055198,
      "grad_norm": 2.30162190529953,
      "learning_rate": 5.693986233737299e-07,
      "log_odds_chosen": 0.515380859375,
      "log_odds_ratio": -0.5791991949081421,
      "logits/chosen": -0.7855468988418579,
      "logits/rejected": -0.687207043170929,
      "logps/chosen": -0.673828125,
      "logps/rejected": -1.0173828601837158,
      "loss": 0.8504,
      "nll_loss": 0.771484375,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0673828125,
      "rewards/margins": 0.03439636155962944,
      "rewards/rejected": -0.10183105617761612,
      "step": 19740
    },
    {
      "epoch": 0.7401023027486837,
      "grad_norm": 2.5950873821595537,
      "learning_rate": 5.692544535718559e-07,
      "log_odds_chosen": 0.5582519769668579,
      "log_odds_ratio": -0.5640624761581421,
      "logits/chosen": -0.7376953363418579,
      "logits/rejected": -0.6495116949081421,
      "logps/chosen": -0.6478515863418579,
      "logps/rejected": -0.983203113079071,
      "loss": 0.827,
      "nll_loss": 0.7616211175918579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06474609673023224,
      "rewards/margins": 0.03358154371380806,
      "rewards/rejected": -0.09846191108226776,
      "step": 19750
    },
    {
      "epoch": 0.7404770380918476,
      "grad_norm": 2.4099096629828214,
      "learning_rate": 5.691103932244722e-07,
      "log_odds_chosen": 0.4074462950229645,
      "log_odds_ratio": -0.6387695074081421,
      "logits/chosen": -0.7646484375,
      "logits/rejected": -0.6664062738418579,
      "logps/chosen": -0.71875,
      "logps/rejected": -0.988085925579071,
      "loss": 0.85,
      "nll_loss": 0.8003906011581421,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07188721001148224,
      "rewards/margins": 0.02685394324362278,
      "rewards/rejected": -0.09869384765625,
      "step": 19760
    },
    {
      "epoch": 0.7408517734350115,
      "grad_norm": 2.2515638393985715,
      "learning_rate": 5.689664421931509e-07,
      "log_odds_chosen": 0.5337768793106079,
      "log_odds_ratio": -0.5907226800918579,
      "logits/chosen": -0.75,
      "logits/rejected": -0.644726574420929,
      "logps/chosen": -0.6919921636581421,
      "logps/rejected": -1.048242211341858,
      "loss": 0.8309,
      "nll_loss": 0.7216796875,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06926269829273224,
      "rewards/margins": 0.03562774509191513,
      "rewards/rejected": -0.10483398288488388,
      "step": 19770
    },
    {
      "epoch": 0.7412265087781754,
      "grad_norm": 2.314240802649811,
      "learning_rate": 5.6882260033971e-07,
      "log_odds_chosen": 0.5385986566543579,
      "log_odds_ratio": -0.557421863079071,
      "logits/chosen": -0.740429699420929,
      "logits/rejected": -0.6294921636581421,
      "logps/chosen": -0.677734375,
      "logps/rejected": -1.0125000476837158,
      "loss": 0.8366,
      "nll_loss": 0.767773449420929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06779785454273224,
      "rewards/margins": 0.03342285007238388,
      "rewards/rejected": -0.10115966945886612,
      "step": 19780
    },
    {
      "epoch": 0.7416012441213393,
      "grad_norm": 2.684228853127889,
      "learning_rate": 5.686788675262114e-07,
      "log_odds_chosen": 0.49028319120407104,
      "log_odds_ratio": -0.579052746295929,
      "logits/chosen": -0.746289074420929,
      "logits/rejected": -0.64501953125,
      "logps/chosen": -0.6327148675918579,
      "logps/rejected": -0.9130859375,
      "loss": 0.8402,
      "nll_loss": 0.7626953125,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06333007663488388,
      "rewards/margins": 0.027947235852479935,
      "rewards/rejected": -0.09128417819738388,
      "step": 19790
    },
    {
      "epoch": 0.7419759794645032,
      "grad_norm": 2.431592090487502,
      "learning_rate": 5.685352436149611e-07,
      "log_odds_chosen": 0.7330566644668579,
      "log_odds_ratio": -0.529541015625,
      "logits/chosen": -0.786914050579071,
      "logits/rejected": -0.6548827886581421,
      "logps/chosen": -0.664843738079071,
      "logps/rejected": -1.1416015625,
      "loss": 0.8194,
      "nll_loss": 0.7445312738418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06639404594898224,
      "rewards/margins": 0.0477447509765625,
      "rewards/rejected": -0.11418457329273224,
      "step": 19800
    },
    {
      "epoch": 0.7423507148076671,
      "grad_norm": 2.7973320792071625,
      "learning_rate": 5.683917284685082e-07,
      "log_odds_chosen": 0.5818725824356079,
      "log_odds_ratio": -0.564453125,
      "logits/chosen": -0.741992175579071,
      "logits/rejected": -0.6494140625,
      "logps/chosen": -0.680468738079071,
      "logps/rejected": -1.0537109375,
      "loss": 0.8276,
      "nll_loss": 0.791796863079071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06802978366613388,
      "rewards/margins": 0.03730926662683487,
      "rewards/rejected": -0.1053466796875,
      "step": 19810
    },
    {
      "epoch": 0.742725450150831,
      "grad_norm": 2.3736805602072946,
      "learning_rate": 5.682483219496449e-07,
      "log_odds_chosen": 0.73504638671875,
      "log_odds_ratio": -0.5296875238418579,
      "logits/chosen": -0.7826172113418579,
      "logits/rejected": -0.646191418170929,
      "logps/chosen": -0.6685546636581421,
      "logps/rejected": -1.158593773841858,
      "loss": 0.8448,
      "nll_loss": 0.781054675579071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06679687649011612,
      "rewards/margins": 0.04902191087603569,
      "rewards/rejected": -0.11591796576976776,
      "step": 19820
    },
    {
      "epoch": 0.7431001854939948,
      "grad_norm": 2.3947775448092465,
      "learning_rate": 5.681050239214051e-07,
      "log_odds_chosen": 0.679882824420929,
      "log_odds_ratio": -0.5306640863418579,
      "logits/chosen": -0.744921863079071,
      "logits/rejected": -0.6036132574081421,
      "logps/chosen": -0.6361328363418579,
      "logps/rejected": -1.0626952648162842,
      "loss": 0.8306,
      "nll_loss": 0.748046875,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06364746391773224,
      "rewards/margins": 0.04254150390625,
      "rewards/rejected": -0.10615234076976776,
      "step": 19830
    },
    {
      "epoch": 0.7434749208371587,
      "grad_norm": 2.5372841452790404,
      "learning_rate": 5.679618342470647e-07,
      "log_odds_chosen": 0.656054675579071,
      "log_odds_ratio": -0.529980480670929,
      "logits/chosen": -0.7398437261581421,
      "logits/rejected": -0.631542980670929,
      "logps/chosen": -0.6454101800918579,
      "logps/rejected": -1.063867211341858,
      "loss": 0.8277,
      "nll_loss": 0.7701171636581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06453857570886612,
      "rewards/margins": 0.04179992526769638,
      "rewards/rejected": -0.10635986179113388,
      "step": 19840
    },
    {
      "epoch": 0.7438496561803226,
      "grad_norm": 2.4199510764313317,
      "learning_rate": 5.678187527901407e-07,
      "log_odds_chosen": 0.4924682676792145,
      "log_odds_ratio": -0.59423828125,
      "logits/chosen": -0.7779296636581421,
      "logits/rejected": -0.6904296875,
      "logps/chosen": -0.671093761920929,
      "logps/rejected": -0.9916015863418579,
      "loss": 0.8153,
      "nll_loss": 0.78369140625,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06711425632238388,
      "rewards/margins": 0.03207588195800781,
      "rewards/rejected": -0.09919433295726776,
      "step": 19850
    },
    {
      "epoch": 0.7442243915234865,
      "grad_norm": 2.204858546900158,
      "learning_rate": 5.676757794143909e-07,
      "log_odds_chosen": 0.7062743902206421,
      "log_odds_ratio": -0.531054675579071,
      "logits/chosen": -0.775390625,
      "logits/rejected": -0.636914074420929,
      "logps/chosen": -0.633496105670929,
      "logps/rejected": -1.0968749523162842,
      "loss": 0.8183,
      "nll_loss": 0.741406261920929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06337890774011612,
      "rewards/margins": 0.046356201171875,
      "rewards/rejected": -0.10969237983226776,
      "step": 19860
    },
    {
      "epoch": 0.7445991268666504,
      "grad_norm": 2.119588913179103,
      "learning_rate": 5.675329139838122e-07,
      "log_odds_chosen": 0.602734386920929,
      "log_odds_ratio": -0.5794922113418579,
      "logits/chosen": -0.716015636920929,
      "logits/rejected": -0.6324218511581421,
      "logps/chosen": -0.63232421875,
      "logps/rejected": -0.9652343988418579,
      "loss": 0.8362,
      "nll_loss": 0.7665039300918579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06319580227136612,
      "rewards/margins": 0.03326110914349556,
      "rewards/rejected": -0.09645996242761612,
      "step": 19870
    },
    {
      "epoch": 0.7449738622098143,
      "grad_norm": 2.2370201470479922,
      "learning_rate": 5.673901563626419e-07,
      "log_odds_chosen": 0.6274780035018921,
      "log_odds_ratio": -0.55078125,
      "logits/chosen": -0.781445324420929,
      "logits/rejected": -0.626660168170929,
      "logps/chosen": -0.631640613079071,
      "logps/rejected": -1.0226562023162842,
      "loss": 0.8306,
      "nll_loss": 0.7607421875,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06314697116613388,
      "rewards/margins": 0.03909454494714737,
      "rewards/rejected": -0.102294921875,
      "step": 19880
    },
    {
      "epoch": 0.7453485975529782,
      "grad_norm": 2.1652272428316777,
      "learning_rate": 5.672475064153561e-07,
      "log_odds_chosen": 0.678466796875,
      "log_odds_ratio": -0.587109386920929,
      "logits/chosen": -0.7554687261581421,
      "logits/rejected": -0.632128894329071,
      "logps/chosen": -0.7255859375,
      "logps/rejected": -1.219140648841858,
      "loss": 0.8473,
      "nll_loss": 0.848828136920929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07261963188648224,
      "rewards/margins": 0.04936676099896431,
      "rewards/rejected": -0.12192382663488388,
      "step": 19890
    },
    {
      "epoch": 0.7457233328961421,
      "grad_norm": 2.533965479505338,
      "learning_rate": 5.671049640066686e-07,
      "log_odds_chosen": 0.678173840045929,
      "log_odds_ratio": -0.538769543170929,
      "logits/chosen": -0.760449230670929,
      "logits/rejected": -0.6053711175918579,
      "logps/chosen": -0.695117175579071,
      "logps/rejected": -1.1396484375,
      "loss": 0.8083,
      "nll_loss": 0.7611328363418579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06949462741613388,
      "rewards/margins": 0.04451293870806694,
      "rewards/rejected": -0.11391601711511612,
      "step": 19900
    },
    {
      "epoch": 0.746098068239306,
      "grad_norm": 2.4711920809122128,
      "learning_rate": 5.66962529001532e-07,
      "log_odds_chosen": 0.4913330078125,
      "log_odds_ratio": -0.6087890863418579,
      "logits/chosen": -0.6695312261581421,
      "logits/rejected": -0.5855468511581421,
      "logps/chosen": -0.6680663824081421,
      "logps/rejected": -0.973437488079071,
      "loss": 0.8514,
      "nll_loss": 0.8017578125,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06680908054113388,
      "rewards/margins": 0.030505752190947533,
      "rewards/rejected": -0.09726562350988388,
      "step": 19910
    },
    {
      "epoch": 0.7464728035824699,
      "grad_norm": 2.495310381683324,
      "learning_rate": 5.668202012651357e-07,
      "log_odds_chosen": 0.610156238079071,
      "log_odds_ratio": -0.568066418170929,
      "logits/chosen": -0.7734375,
      "logits/rejected": -0.63671875,
      "logps/chosen": -0.669726550579071,
      "logps/rejected": -1.051367163658142,
      "loss": 0.8504,
      "nll_loss": 0.76171875,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06700439751148224,
      "rewards/margins": 0.0381011962890625,
      "rewards/rejected": -0.10504150390625,
      "step": 19920
    },
    {
      "epoch": 0.7468475389256338,
      "grad_norm": 2.1203833942400743,
      "learning_rate": 5.666779806629058e-07,
      "log_odds_chosen": 0.5712890625,
      "log_odds_ratio": -0.5575195550918579,
      "logits/chosen": -0.761914074420929,
      "logits/rejected": -0.638476550579071,
      "logps/chosen": -0.6576172113418579,
      "logps/rejected": -0.99609375,
      "loss": 0.8422,
      "nll_loss": 0.779492199420929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06582031399011612,
      "rewards/margins": 0.03383178636431694,
      "rewards/rejected": -0.099609375,
      "step": 19930
    },
    {
      "epoch": 0.7472222742687976,
      "grad_norm": 2.4718917843111803,
      "learning_rate": 5.665358670605048e-07,
      "log_odds_chosen": 0.3967529237270355,
      "log_odds_ratio": -0.6263672113418579,
      "logits/chosen": -0.750781238079071,
      "logits/rejected": -0.628613293170929,
      "logps/chosen": -0.716601550579071,
      "logps/rejected": -0.9742187261581421,
      "loss": 0.8242,
      "nll_loss": 0.7660156488418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07160644233226776,
      "rewards/margins": 0.025757599622011185,
      "rewards/rejected": -0.09736327826976776,
      "step": 19940
    },
    {
      "epoch": 0.7475970096119615,
      "grad_norm": 2.02567587237601,
      "learning_rate": 5.663938603238308e-07,
      "log_odds_chosen": 0.4911254942417145,
      "log_odds_ratio": -0.5986328125,
      "logits/chosen": -0.7578125,
      "logits/rejected": -0.6572265625,
      "logps/chosen": -0.680371105670929,
      "logps/rejected": -0.9853515625,
      "loss": 0.838,
      "nll_loss": 0.791015625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06804199516773224,
      "rewards/margins": 0.03056640550494194,
      "rewards/rejected": -0.09860839694738388,
      "step": 19950
    },
    {
      "epoch": 0.7479717449551254,
      "grad_norm": 2.2345911717018407,
      "learning_rate": 5.662519603190176e-07,
      "log_odds_chosen": 0.6177978515625,
      "log_odds_ratio": -0.57080078125,
      "logits/chosen": -0.783398449420929,
      "logits/rejected": -0.698925793170929,
      "logps/chosen": -0.6820312738418579,
      "logps/rejected": -1.115234375,
      "loss": 0.843,
      "nll_loss": 0.75732421875,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06822510063648224,
      "rewards/margins": 0.04322509840130806,
      "rewards/rejected": -0.11151123046875,
      "step": 19960
    },
    {
      "epoch": 0.7483464802982893,
      "grad_norm": 2.4715242801279156,
      "learning_rate": 5.661101669124328e-07,
      "log_odds_chosen": 0.5926758050918579,
      "log_odds_ratio": -0.55908203125,
      "logits/chosen": -0.7523437738418579,
      "logits/rejected": -0.6490234136581421,
      "logps/chosen": -0.70849609375,
      "logps/rejected": -1.08984375,
      "loss": 0.8321,
      "nll_loss": 0.792187511920929,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07082519680261612,
      "rewards/margins": 0.038086701184511185,
      "rewards/rejected": -0.10895995795726776,
      "step": 19970
    },
    {
      "epoch": 0.7487212156414532,
      "grad_norm": 2.211285292365349,
      "learning_rate": 5.659684799706784e-07,
      "log_odds_chosen": 0.45341795682907104,
      "log_odds_ratio": -0.623242199420929,
      "logits/chosen": -0.7542968988418579,
      "logits/rejected": -0.66455078125,
      "logps/chosen": -0.6797851324081421,
      "logps/rejected": -0.9859374761581421,
      "loss": 0.8242,
      "nll_loss": 0.797558605670929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06796874850988388,
      "rewards/margins": 0.030532073229551315,
      "rewards/rejected": -0.09843750298023224,
      "step": 19980
    },
    {
      "epoch": 0.7490959509846171,
      "grad_norm": 2.0175268817822483,
      "learning_rate": 5.658268993605907e-07,
      "log_odds_chosen": 0.7154175043106079,
      "log_odds_ratio": -0.52880859375,
      "logits/chosen": -0.7530273199081421,
      "logits/rejected": -0.6451171636581421,
      "logps/chosen": -0.6104491949081421,
      "logps/rejected": -1.0634765625,
      "loss": 0.8485,
      "nll_loss": 0.719433605670929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06109619140625,
      "rewards/margins": 0.0452423095703125,
      "rewards/rejected": -0.1063232421875,
      "step": 19990
    },
    {
      "epoch": 0.749470686327781,
      "grad_norm": 2.2959736236101635,
      "learning_rate": 5.65685424949238e-07,
      "log_odds_chosen": 0.48515623807907104,
      "log_odds_ratio": -0.5980468988418579,
      "logits/chosen": -0.773242175579071,
      "logits/rejected": -0.665722668170929,
      "logps/chosen": -0.6971679925918579,
      "logps/rejected": -0.994140625,
      "loss": 0.8173,
      "nll_loss": 0.8197265863418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06965331733226776,
      "rewards/margins": 0.029790496453642845,
      "rewards/rejected": -0.09941406548023224,
      "step": 20000
    },
    {
      "epoch": 0.749470686327781,
      "eval_log_odds_chosen": 0.412467896938324,
      "eval_log_odds_ratio": -0.6322020888328552,
      "eval_logits/chosen": -0.7044387459754944,
      "eval_logits/rejected": -0.6075789332389832,
      "eval_logps/chosen": -0.743389904499054,
      "eval_logps/rejected": -1.0265930891036987,
      "eval_loss": 1.1105307340621948,
      "eval_nll_loss": 1.0498415231704712,
      "eval_rewards/accuracies": 0.5907366871833801,
      "eval_rewards/chosen": -0.07433958351612091,
      "eval_rewards/margins": 0.028321433812379837,
      "eval_rewards/rejected": -0.10265849530696869,
      "eval_runtime": 2433.7879,
      "eval_samples_per_second": 77.993,
      "eval_steps_per_second": 1.219,
      "step": 20000
    },
    {
      "epoch": 0.7498454216709449,
      "grad_norm": 2.33625130919939,
      "learning_rate": 5.655440566039219e-07,
      "log_odds_chosen": 0.7818359136581421,
      "log_odds_ratio": -0.48261719942092896,
      "logits/chosen": -0.765429675579071,
      "logits/rejected": -0.6712890863418579,
      "logps/chosen": -0.586718738079071,
      "logps/rejected": -1.048242211341858,
      "loss": 0.822,
      "nll_loss": 0.6884765625,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.05866699293255806,
      "rewards/margins": 0.04620361328125,
      "rewards/rejected": -0.1048583984375,
      "step": 20010
    },
    {
      "epoch": 0.7502201570141088,
      "grad_norm": 2.201370797632621,
      "learning_rate": 5.654027941921755e-07,
      "log_odds_chosen": 0.43950194120407104,
      "log_odds_ratio": -0.61376953125,
      "logits/chosen": -0.7623046636581421,
      "logits/rejected": -0.648144543170929,
      "logps/chosen": -0.693164050579071,
      "logps/rejected": -0.9810546636581421,
      "loss": 0.8288,
      "nll_loss": 0.7743164300918579,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06925048679113388,
      "rewards/margins": 0.02883758582174778,
      "rewards/rejected": -0.09814453125,
      "step": 20020
    },
    {
      "epoch": 0.7505948923572727,
      "grad_norm": 2.218386809906093,
      "learning_rate": 5.65261637581764e-07,
      "log_odds_chosen": 0.41645509004592896,
      "log_odds_ratio": -0.6187499761581421,
      "logits/chosen": -0.768261730670929,
      "logits/rejected": -0.681835949420929,
      "logps/chosen": -0.66357421875,
      "logps/rejected": -0.9306640625,
      "loss": 0.8255,
      "nll_loss": 0.767773449420929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06639404594898224,
      "rewards/margins": 0.026702880859375,
      "rewards/rejected": -0.09307861328125,
      "step": 20030
    },
    {
      "epoch": 0.7509696277004365,
      "grad_norm": 2.5307340836894636,
      "learning_rate": 5.65120586640683e-07,
      "log_odds_chosen": 0.49229735136032104,
      "log_odds_ratio": -0.6065429449081421,
      "logits/chosen": -0.7720702886581421,
      "logits/rejected": -0.684863269329071,
      "logps/chosen": -0.6998046636581421,
      "logps/rejected": -1.0421874523162842,
      "loss": 0.8535,
      "nll_loss": 0.804492175579071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.070068359375,
      "rewards/margins": 0.03422851487994194,
      "rewards/rejected": -0.10423584282398224,
      "step": 20040
    },
    {
      "epoch": 0.7513443630436004,
      "grad_norm": 2.2130511780570927,
      "learning_rate": 5.649796412371589e-07,
      "log_odds_chosen": 0.42680662870407104,
      "log_odds_ratio": -0.6136718988418579,
      "logits/chosen": -0.709765613079071,
      "logits/rejected": -0.6366211175918579,
      "logps/chosen": -0.6641601324081421,
      "logps/rejected": -0.9136718511581421,
      "loss": 0.8196,
      "nll_loss": 0.7689453363418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06640625,
      "rewards/margins": 0.024970244616270065,
      "rewards/rejected": -0.09130859375,
      "step": 20050
    },
    {
      "epoch": 0.7517190983867643,
      "grad_norm": 2.202588868322955,
      "learning_rate": 5.648388012396479e-07,
      "log_odds_chosen": 0.516430675983429,
      "log_odds_ratio": -0.58203125,
      "logits/chosen": -0.7914062738418579,
      "logits/rejected": -0.6504882574081421,
      "logps/chosen": -0.623242199420929,
      "logps/rejected": -0.94921875,
      "loss": 0.8345,
      "nll_loss": 0.7109375,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06232910230755806,
      "rewards/margins": 0.032657623291015625,
      "rewards/rejected": -0.0948486328125,
      "step": 20060
    },
    {
      "epoch": 0.7520938337299282,
      "grad_norm": 2.421703391345266,
      "learning_rate": 5.646980665168356e-07,
      "log_odds_chosen": 0.5287841558456421,
      "log_odds_ratio": -0.621142566204071,
      "logits/chosen": -0.719921886920929,
      "logits/rejected": -0.634082019329071,
      "logps/chosen": -0.708984375,
      "logps/rejected": -1.0535156726837158,
      "loss": 0.8178,
      "nll_loss": 0.7640625238418579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07083740085363388,
      "rewards/margins": 0.034523773938417435,
      "rewards/rejected": -0.10539551079273224,
      "step": 20070
    },
    {
      "epoch": 0.7524685690730921,
      "grad_norm": 2.0853300661747256,
      "learning_rate": 5.645574369376366e-07,
      "log_odds_chosen": 0.5476318597793579,
      "log_odds_ratio": -0.552734375,
      "logits/chosen": -0.749804675579071,
      "logits/rejected": -0.6595703363418579,
      "logps/chosen": -0.6826171875,
      "logps/rejected": -1.010156273841858,
      "loss": 0.8079,
      "nll_loss": 0.770703136920929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06834717094898224,
      "rewards/margins": 0.032779693603515625,
      "rewards/rejected": -0.10104980319738388,
      "step": 20080
    },
    {
      "epoch": 0.752843304416256,
      "grad_norm": 2.06626406928798,
      "learning_rate": 5.644169123711941e-07,
      "log_odds_chosen": 0.48383790254592896,
      "log_odds_ratio": -0.6060546636581421,
      "logits/chosen": -0.752636730670929,
      "logits/rejected": -0.669238269329071,
      "logps/chosen": -0.660839855670929,
      "logps/rejected": -0.9789062738418579,
      "loss": 0.8193,
      "nll_loss": 0.7289062738418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06605224311351776,
      "rewards/margins": 0.03187408298254013,
      "rewards/rejected": -0.09793701022863388,
      "step": 20090
    },
    {
      "epoch": 0.7532180397594199,
      "grad_norm": 2.771707602713086,
      "learning_rate": 5.642764926868786e-07,
      "log_odds_chosen": 0.44952392578125,
      "log_odds_ratio": -0.6258789300918579,
      "logits/chosen": -0.73681640625,
      "logits/rejected": -0.6396484375,
      "logps/chosen": -0.685351550579071,
      "logps/rejected": -0.93359375,
      "loss": 0.8233,
      "nll_loss": 0.7347656488418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06854248046875,
      "rewards/margins": 0.024810791015625,
      "rewards/rejected": -0.09332275390625,
      "step": 20100
    },
    {
      "epoch": 0.7535927751025838,
      "grad_norm": 2.7529420306457064,
      "learning_rate": 5.641361777542885e-07,
      "log_odds_chosen": 0.5503906011581421,
      "log_odds_ratio": -0.5845702886581421,
      "logits/chosen": -0.729687511920929,
      "logits/rejected": -0.6161133050918579,
      "logps/chosen": -0.6729491949081421,
      "logps/rejected": -1.006250023841858,
      "loss": 0.8219,
      "nll_loss": 0.750195324420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06734619289636612,
      "rewards/margins": 0.03325500339269638,
      "rewards/rejected": -0.10057373344898224,
      "step": 20110
    },
    {
      "epoch": 0.7539675104457477,
      "grad_norm": 2.3143844781342096,
      "learning_rate": 5.639959674432491e-07,
      "log_odds_chosen": 0.5677856206893921,
      "log_odds_ratio": -0.566699206829071,
      "logits/chosen": -0.7183593511581421,
      "logits/rejected": -0.602832019329071,
      "logps/chosen": -0.62646484375,
      "logps/rejected": -0.975390613079071,
      "loss": 0.8395,
      "nll_loss": 0.750292956829071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06265868991613388,
      "rewards/margins": 0.034894563257694244,
      "rewards/rejected": -0.0975341796875,
      "step": 20120
    },
    {
      "epoch": 0.7543422457889116,
      "grad_norm": 2.2127006915217633,
      "learning_rate": 5.638558616238116e-07,
      "log_odds_chosen": 0.6480712890625,
      "log_odds_ratio": -0.5616699457168579,
      "logits/chosen": -0.76171875,
      "logits/rejected": -0.651660144329071,
      "logps/chosen": -0.640429675579071,
      "logps/rejected": -1.069726586341858,
      "loss": 0.8151,
      "nll_loss": 0.7857421636581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06405029445886612,
      "rewards/margins": 0.043013762682676315,
      "rewards/rejected": -0.10703124850988388,
      "step": 20130
    },
    {
      "epoch": 0.7547169811320755,
      "grad_norm": 2.2221659290025984,
      "learning_rate": 5.637158601662535e-07,
      "log_odds_chosen": 0.49165040254592896,
      "log_odds_ratio": -0.60400390625,
      "logits/chosen": -0.7744140625,
      "logits/rejected": -0.663378894329071,
      "logps/chosen": -0.682812511920929,
      "logps/rejected": -1.031640648841858,
      "loss": 0.8399,
      "nll_loss": 0.7720702886581421,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06828613579273224,
      "rewards/margins": 0.03491096571087837,
      "rewards/rejected": -0.10317382961511612,
      "step": 20140
    },
    {
      "epoch": 0.7550917164752393,
      "grad_norm": 2.753410511378416,
      "learning_rate": 5.635759629410775e-07,
      "log_odds_chosen": 0.2966064512729645,
      "log_odds_ratio": -0.6895507574081421,
      "logits/chosen": -0.73583984375,
      "logits/rejected": -0.651562511920929,
      "logps/chosen": -0.73388671875,
      "logps/rejected": -0.940625011920929,
      "loss": 0.8318,
      "nll_loss": 0.8421875238418579,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07338867336511612,
      "rewards/margins": 0.02068328857421875,
      "rewards/rejected": -0.09403076022863388,
      "step": 20150
    },
    {
      "epoch": 0.7554664518184032,
      "grad_norm": 2.4614486518349357,
      "learning_rate": 5.63436169819011e-07,
      "log_odds_chosen": 0.652539074420929,
      "log_odds_ratio": -0.554882824420929,
      "logits/chosen": -0.716601550579071,
      "logits/rejected": -0.608203113079071,
      "logps/chosen": -0.7352539300918579,
      "logps/rejected": -1.1521484851837158,
      "loss": 0.821,
      "nll_loss": 0.833789050579071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07357177883386612,
      "rewards/margins": 0.041574858129024506,
      "rewards/rejected": -0.1151123046875,
      "step": 20160
    },
    {
      "epoch": 0.7558411871615671,
      "grad_norm": 2.3662123676458346,
      "learning_rate": 5.63296480671006e-07,
      "log_odds_chosen": 0.5281982421875,
      "log_odds_ratio": -0.599414050579071,
      "logits/chosen": -0.7425781488418579,
      "logits/rejected": -0.625195324420929,
      "logps/chosen": -0.67578125,
      "logps/rejected": -1.0261719226837158,
      "loss": 0.8284,
      "nll_loss": 0.75830078125,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06756591796875,
      "rewards/margins": 0.03494720533490181,
      "rewards/rejected": -0.10250244289636612,
      "step": 20170
    },
    {
      "epoch": 0.756215922504731,
      "grad_norm": 2.3378149268584316,
      "learning_rate": 5.631568953682381e-07,
      "log_odds_chosen": 0.605664074420929,
      "log_odds_ratio": -0.560546875,
      "logits/chosen": -0.7276366949081421,
      "logits/rejected": -0.6112304925918579,
      "logps/chosen": -0.675000011920929,
      "logps/rejected": -1.065039038658142,
      "loss": 0.81,
      "nll_loss": 0.732128918170929,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06751708686351776,
      "rewards/margins": 0.03899841383099556,
      "rewards/rejected": -0.10654296725988388,
      "step": 20180
    },
    {
      "epoch": 0.7565906578478949,
      "grad_norm": 2.080458548036959,
      "learning_rate": 5.630174137821066e-07,
      "log_odds_chosen": 0.7662109136581421,
      "log_odds_ratio": -0.49560546875,
      "logits/chosen": -0.7328125238418579,
      "logits/rejected": -0.616503894329071,
      "logps/chosen": -0.63671875,
      "logps/rejected": -1.1179687976837158,
      "loss": 0.8346,
      "nll_loss": 0.754687488079071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06370849907398224,
      "rewards/margins": 0.04808960109949112,
      "rewards/rejected": -0.11184082180261612,
      "step": 20190
    },
    {
      "epoch": 0.7569653931910588,
      "grad_norm": 2.4814124595866,
      "learning_rate": 5.628780357842334e-07,
      "log_odds_chosen": 0.44000244140625,
      "log_odds_ratio": -0.615039050579071,
      "logits/chosen": -0.6859375238418579,
      "logits/rejected": -0.59228515625,
      "logps/chosen": -0.6900390386581421,
      "logps/rejected": -0.959179699420929,
      "loss": 0.8236,
      "nll_loss": 0.824999988079071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06899414211511612,
      "rewards/margins": 0.0269927978515625,
      "rewards/rejected": -0.09603271633386612,
      "step": 20200
    },
    {
      "epoch": 0.7573401285342227,
      "grad_norm": 2.23789564258879,
      "learning_rate": 5.627387612464627e-07,
      "log_odds_chosen": 0.41209715604782104,
      "log_odds_ratio": -0.61572265625,
      "logits/chosen": -0.67138671875,
      "logits/rejected": -0.582812488079071,
      "logps/chosen": -0.711132824420929,
      "logps/rejected": -0.956347644329071,
      "loss": 0.8363,
      "nll_loss": 0.7274414300918579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07111816108226776,
      "rewards/margins": 0.02457122877240181,
      "rewards/rejected": -0.0955810546875,
      "step": 20210
    },
    {
      "epoch": 0.7577148638773866,
      "grad_norm": 2.3291084510392057,
      "learning_rate": 5.625995900408606e-07,
      "log_odds_chosen": 0.47150880098342896,
      "log_odds_ratio": -0.600292980670929,
      "logits/chosen": -0.731249988079071,
      "logits/rejected": -0.6163085699081421,
      "logps/chosen": -0.6728515625,
      "logps/rejected": -0.962109386920929,
      "loss": 0.8015,
      "nll_loss": 0.74853515625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06723632663488388,
      "rewards/margins": 0.02900085411965847,
      "rewards/rejected": -0.09627685695886612,
      "step": 20220
    },
    {
      "epoch": 0.7580895992205505,
      "grad_norm": 2.5293945354298444,
      "learning_rate": 5.624605220397146e-07,
      "log_odds_chosen": 0.3790039122104645,
      "log_odds_ratio": -0.640625,
      "logits/chosen": -0.733203113079071,
      "logits/rejected": -0.6473633050918579,
      "logps/chosen": -0.67041015625,
      "logps/rejected": -0.912890613079071,
      "loss": 0.8411,
      "nll_loss": 0.776562511920929,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06700439751148224,
      "rewards/margins": 0.024268340319395065,
      "rewards/rejected": -0.09125976264476776,
      "step": 20230
    },
    {
      "epoch": 0.7584643345637144,
      "grad_norm": 2.611383372624855,
      "learning_rate": 5.623215571155332e-07,
      "log_odds_chosen": 0.5127929449081421,
      "log_odds_ratio": -0.5835937261581421,
      "logits/chosen": -0.7300781011581421,
      "logits/rejected": -0.644726574420929,
      "logps/chosen": -0.6644531488418579,
      "logps/rejected": -0.990429699420929,
      "loss": 0.8225,
      "nll_loss": 0.7769531011581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06643066555261612,
      "rewards/margins": 0.03251495212316513,
      "rewards/rejected": -0.0989990234375,
      "step": 20240
    },
    {
      "epoch": 0.7588390699068782,
      "grad_norm": 2.3151469292902203,
      "learning_rate": 5.621826951410452e-07,
      "log_odds_chosen": 0.4639648497104645,
      "log_odds_ratio": -0.5982421636581421,
      "logits/chosen": -0.7933593988418579,
      "logits/rejected": -0.643847644329071,
      "logps/chosen": -0.6871093511581421,
      "logps/rejected": -0.9808593988418579,
      "loss": 0.8282,
      "nll_loss": 0.7593749761581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06871338188648224,
      "rewards/margins": 0.02943878248333931,
      "rewards/rejected": -0.09813232719898224,
      "step": 20250
    },
    {
      "epoch": 0.7592138052500421,
      "grad_norm": 2.4199913440832206,
      "learning_rate": 5.620439359891992e-07,
      "log_odds_chosen": 0.601855456829071,
      "log_odds_ratio": -0.565722644329071,
      "logits/chosen": -0.728222668170929,
      "logits/rejected": -0.6131836175918579,
      "logps/chosen": -0.668749988079071,
      "logps/rejected": -1.057226538658142,
      "loss": 0.8355,
      "nll_loss": 0.800585925579071,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06687011569738388,
      "rewards/margins": 0.03891754150390625,
      "rewards/rejected": -0.10573730617761612,
      "step": 20260
    },
    {
      "epoch": 0.759588540593206,
      "grad_norm": 2.296333300895554,
      "learning_rate": 5.619052795331631e-07,
      "log_odds_chosen": 0.59381103515625,
      "log_odds_ratio": -0.61279296875,
      "logits/chosen": -0.744921863079071,
      "logits/rejected": -0.638476550579071,
      "logps/chosen": -0.703906238079071,
      "logps/rejected": -1.108007788658142,
      "loss": 0.8189,
      "nll_loss": 0.794921875,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07042236626148224,
      "rewards/margins": 0.04037780687212944,
      "rewards/rejected": -0.11075439304113388,
      "step": 20270
    },
    {
      "epoch": 0.7599632759363699,
      "grad_norm": 2.100389585088275,
      "learning_rate": 5.617667256463242e-07,
      "log_odds_chosen": 0.625683605670929,
      "log_odds_ratio": -0.5536133050918579,
      "logits/chosen": -0.748828113079071,
      "logits/rejected": -0.6375976800918579,
      "logps/chosen": -0.66015625,
      "logps/rejected": -1.0431640148162842,
      "loss": 0.8156,
      "nll_loss": 0.753222644329071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06600341945886612,
      "rewards/margins": 0.03824768215417862,
      "rewards/rejected": -0.10422363132238388,
      "step": 20280
    },
    {
      "epoch": 0.7603380112795338,
      "grad_norm": 2.2867244008460217,
      "learning_rate": 5.616282742022878e-07,
      "log_odds_chosen": 0.47825926542282104,
      "log_odds_ratio": -0.586132824420929,
      "logits/chosen": -0.7515624761581421,
      "logits/rejected": -0.65625,
      "logps/chosen": -0.6162109375,
      "logps/rejected": -0.910937488079071,
      "loss": 0.8473,
      "nll_loss": 0.724902331829071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06166992336511612,
      "rewards/margins": 0.029499053955078125,
      "rewards/rejected": -0.09107665717601776,
      "step": 20290
    },
    {
      "epoch": 0.7607127466226977,
      "grad_norm": 2.746524771847545,
      "learning_rate": 5.614899250748771e-07,
      "log_odds_chosen": 0.4942077696323395,
      "log_odds_ratio": -0.611132800579071,
      "logits/chosen": -0.745898425579071,
      "logits/rejected": -0.6382812261581421,
      "logps/chosen": -0.6922851800918579,
      "logps/rejected": -1.0011718273162842,
      "loss": 0.8134,
      "nll_loss": 0.7398437261581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0692138671875,
      "rewards/margins": 0.030908966436982155,
      "rewards/rejected": -0.10007324069738388,
      "step": 20300
    },
    {
      "epoch": 0.7610874819658616,
      "grad_norm": 2.3958531461079335,
      "learning_rate": 5.613516781381333e-07,
      "log_odds_chosen": 0.5583251714706421,
      "log_odds_ratio": -0.5760742425918579,
      "logits/chosen": -0.7763671875,
      "logits/rejected": -0.674609363079071,
      "logps/chosen": -0.674511730670929,
      "logps/rejected": -1.0310547351837158,
      "loss": 0.8368,
      "nll_loss": 0.7880859375,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06740722805261612,
      "rewards/margins": 0.03567809984087944,
      "rewards/rejected": -0.10312499850988388,
      "step": 20310
    },
    {
      "epoch": 0.7614622173090255,
      "grad_norm": 2.574756645058044,
      "learning_rate": 5.612135332663137e-07,
      "log_odds_chosen": 0.4151855409145355,
      "log_odds_ratio": -0.624804675579071,
      "logits/chosen": -0.736132800579071,
      "logits/rejected": -0.6700195074081421,
      "logps/chosen": -0.669140636920929,
      "logps/rejected": -0.942187488079071,
      "loss": 0.8312,
      "nll_loss": 0.786328136920929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06689453125,
      "rewards/margins": 0.02724609337747097,
      "rewards/rejected": -0.09417724609375,
      "step": 20320
    },
    {
      "epoch": 0.7618369526521894,
      "grad_norm": 2.503742767202206,
      "learning_rate": 5.61075490333893e-07,
      "log_odds_chosen": 0.551196277141571,
      "log_odds_ratio": -0.5694335699081421,
      "logits/chosen": -0.7939453125,
      "logits/rejected": -0.687207043170929,
      "logps/chosen": -0.663281261920929,
      "logps/rejected": -1.0144531726837158,
      "loss": 0.8341,
      "nll_loss": 0.7601562738418579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06635741889476776,
      "rewards/margins": 0.03508758544921875,
      "rewards/rejected": -0.1014404296875,
      "step": 20330
    },
    {
      "epoch": 0.7622116879953533,
      "grad_norm": 2.315421358360028,
      "learning_rate": 5.609375492155617e-07,
      "log_odds_chosen": 0.609912097454071,
      "log_odds_ratio": -0.5679687261581421,
      "logits/chosen": -0.754101574420929,
      "logits/rejected": -0.66796875,
      "logps/chosen": -0.6929687261581421,
      "logps/rejected": -1.0529296398162842,
      "loss": 0.8202,
      "nll_loss": 0.763476550579071,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06929931789636612,
      "rewards/margins": 0.03607024997472763,
      "rewards/rejected": -0.10529784858226776,
      "step": 20340
    },
    {
      "epoch": 0.7625864233385172,
      "grad_norm": 2.3262849265998504,
      "learning_rate": 5.607997097862253e-07,
      "log_odds_chosen": 0.652636706829071,
      "log_odds_ratio": -0.5418945550918579,
      "logits/chosen": -0.7734375,
      "logits/rejected": -0.6285156011581421,
      "logps/chosen": -0.6410156488418579,
      "logps/rejected": -1.050195336341858,
      "loss": 0.822,
      "nll_loss": 0.726367175579071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0640869140625,
      "rewards/margins": 0.04079895094037056,
      "rewards/rejected": -0.10502929985523224,
      "step": 20350
    },
    {
      "epoch": 0.762961158681681,
      "grad_norm": 2.083861339176369,
      "learning_rate": 5.60661971921005e-07,
      "log_odds_chosen": 0.4437499940395355,
      "log_odds_ratio": -0.634228527545929,
      "logits/chosen": -0.786914050579071,
      "logits/rejected": -0.720507800579071,
      "logps/chosen": -0.6802734136581421,
      "logps/rejected": -0.958203136920929,
      "loss": 0.837,
      "nll_loss": 0.758007824420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06802978366613388,
      "rewards/margins": 0.027803802862763405,
      "rewards/rejected": -0.09575195610523224,
      "step": 20360
    },
    {
      "epoch": 0.7633358940248449,
      "grad_norm": 2.182499618065464,
      "learning_rate": 5.605243354952362e-07,
      "log_odds_chosen": 0.585009753704071,
      "log_odds_ratio": -0.56494140625,
      "logits/chosen": -0.770312488079071,
      "logits/rejected": -0.627636730670929,
      "logps/chosen": -0.6668945550918579,
      "logps/rejected": -1.042578101158142,
      "loss": 0.8291,
      "nll_loss": 0.7744140625,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06676025688648224,
      "rewards/margins": 0.03755340725183487,
      "rewards/rejected": -0.10432128608226776,
      "step": 20370
    },
    {
      "epoch": 0.7637106293680088,
      "grad_norm": 2.311475900969715,
      "learning_rate": 5.603868003844686e-07,
      "log_odds_chosen": 0.443359375,
      "log_odds_ratio": -0.610644519329071,
      "logits/chosen": -0.721972644329071,
      "logits/rejected": -0.6465820074081421,
      "logps/chosen": -0.734667956829071,
      "logps/rejected": -1.025781273841858,
      "loss": 0.846,
      "nll_loss": 0.804882824420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07343749701976776,
      "rewards/margins": 0.02913818322122097,
      "rewards/rejected": -0.10261230170726776,
      "step": 20380
    },
    {
      "epoch": 0.7640853647111727,
      "grad_norm": 2.1326404348275005,
      "learning_rate": 5.602493664644657e-07,
      "log_odds_chosen": 0.5218261480331421,
      "log_odds_ratio": -0.593457043170929,
      "logits/chosen": -0.7486327886581421,
      "logits/rejected": -0.65478515625,
      "logps/chosen": -0.6380859613418579,
      "logps/rejected": -0.983593761920929,
      "loss": 0.8341,
      "nll_loss": 0.7953125238418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06380615383386612,
      "rewards/margins": 0.03454742580652237,
      "rewards/rejected": -0.09827880561351776,
      "step": 20390
    },
    {
      "epoch": 0.7644601000543366,
      "grad_norm": 2.334502632708404,
      "learning_rate": 5.601120336112038e-07,
      "log_odds_chosen": 0.570080578327179,
      "log_odds_ratio": -0.5777343511581421,
      "logits/chosen": -0.7593749761581421,
      "logits/rejected": -0.6382812261581421,
      "logps/chosen": -0.666796863079071,
      "logps/rejected": -1.0255858898162842,
      "loss": 0.842,
      "nll_loss": 0.7705078125,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06671142578125,
      "rewards/margins": 0.03589935228228569,
      "rewards/rejected": -0.1025390625,
      "step": 20400
    },
    {
      "epoch": 0.7648348353975005,
      "grad_norm": 2.577453984247608,
      "learning_rate": 5.599748017008724e-07,
      "log_odds_chosen": 0.6284424066543579,
      "log_odds_ratio": -0.550000011920929,
      "logits/chosen": -0.765429675579071,
      "logits/rejected": -0.66552734375,
      "logps/chosen": -0.657421886920929,
      "logps/rejected": -1.044921875,
      "loss": 0.8385,
      "nll_loss": 0.768750011920929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06571044772863388,
      "rewards/margins": 0.0388031005859375,
      "rewards/rejected": -0.10451660305261612,
      "step": 20410
    },
    {
      "epoch": 0.7652095707406644,
      "grad_norm": 2.4044790633242754,
      "learning_rate": 5.598376706098727e-07,
      "log_odds_chosen": 0.43059080839157104,
      "log_odds_ratio": -0.619433581829071,
      "logits/chosen": -0.781542956829071,
      "logits/rejected": -0.647265613079071,
      "logps/chosen": -0.70458984375,
      "logps/rejected": -0.989453136920929,
      "loss": 0.8294,
      "nll_loss": 0.739453136920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07038573920726776,
      "rewards/margins": 0.02850341796875,
      "rewards/rejected": -0.098876953125,
      "step": 20420
    },
    {
      "epoch": 0.7655843060838283,
      "grad_norm": 2.980488049957178,
      "learning_rate": 5.59700640214818e-07,
      "log_odds_chosen": 0.652587890625,
      "log_odds_ratio": -0.543164074420929,
      "logits/chosen": -0.690722644329071,
      "logits/rejected": -0.5821288824081421,
      "logps/chosen": -0.624218761920929,
      "logps/rejected": -1.024999976158142,
      "loss": 0.8037,
      "nll_loss": 0.714062511920929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06246338039636612,
      "rewards/margins": 0.03999633714556694,
      "rewards/rejected": -0.1025390625,
      "step": 20430
    },
    {
      "epoch": 0.7659590414269922,
      "grad_norm": 2.1678114471372765,
      "learning_rate": 5.595637103925327e-07,
      "log_odds_chosen": 0.546557605266571,
      "log_odds_ratio": -0.5826171636581421,
      "logits/chosen": -0.796875,
      "logits/rejected": -0.6600586175918579,
      "logps/chosen": -0.673144519329071,
      "logps/rejected": -1.033203125,
      "loss": 0.8252,
      "nll_loss": 0.7710937261581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06732177734375,
      "rewards/margins": 0.03600006178021431,
      "rewards/rejected": -0.10344238579273224,
      "step": 20440
    },
    {
      "epoch": 0.7663337767701561,
      "grad_norm": 2.1392692598532856,
      "learning_rate": 5.594268810200525e-07,
      "log_odds_chosen": 0.674877941608429,
      "log_odds_ratio": -0.529589831829071,
      "logits/chosen": -0.7681640386581421,
      "logits/rejected": -0.6390625238418579,
      "logps/chosen": -0.639941394329071,
      "logps/rejected": -1.044335961341858,
      "loss": 0.837,
      "nll_loss": 0.7523437738418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06400146335363388,
      "rewards/margins": 0.04042205959558487,
      "rewards/rejected": -0.1043701171875,
      "step": 20450
    },
    {
      "epoch": 0.7667085121133199,
      "grad_norm": 2.1541301924843665,
      "learning_rate": 5.592901519746228e-07,
      "log_odds_chosen": 0.543212890625,
      "log_odds_ratio": -0.593066394329071,
      "logits/chosen": -0.78515625,
      "logits/rejected": -0.676074206829071,
      "logps/chosen": -0.650195300579071,
      "logps/rejected": -0.990039050579071,
      "loss": 0.8347,
      "nll_loss": 0.754589855670929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06497802585363388,
      "rewards/margins": 0.03395385667681694,
      "rewards/rejected": -0.09903564304113388,
      "step": 20460
    },
    {
      "epoch": 0.7670832474564838,
      "grad_norm": 2.2123763762387614,
      "learning_rate": 5.591535231336994e-07,
      "log_odds_chosen": 0.5450073480606079,
      "log_odds_ratio": -0.6021484136581421,
      "logits/chosen": -0.72412109375,
      "logits/rejected": -0.62158203125,
      "logps/chosen": -0.666015625,
      "logps/rejected": -1.0080077648162842,
      "loss": 0.8156,
      "nll_loss": 0.7904297113418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.066650390625,
      "rewards/margins": 0.03418578952550888,
      "rewards/rejected": -0.10078124701976776,
      "step": 20470
    },
    {
      "epoch": 0.7674579827996477,
      "grad_norm": 2.4395399017464827,
      "learning_rate": 5.590169943749474e-07,
      "log_odds_chosen": 0.45159912109375,
      "log_odds_ratio": -0.610546886920929,
      "logits/chosen": -0.7613281011581421,
      "logits/rejected": -0.6597656011581421,
      "logps/chosen": -0.6786133050918579,
      "logps/rejected": -0.97265625,
      "loss": 0.8166,
      "nll_loss": 0.7691406011581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06784667819738388,
      "rewards/margins": 0.02943267859518528,
      "rewards/rejected": -0.09726562350988388,
      "step": 20480
    },
    {
      "epoch": 0.7678327181428116,
      "grad_norm": 2.545331847094258,
      "learning_rate": 5.588805655762408e-07,
      "log_odds_chosen": 0.560864269733429,
      "log_odds_ratio": -0.55615234375,
      "logits/chosen": -0.7587890625,
      "logits/rejected": -0.651562511920929,
      "logps/chosen": -0.63427734375,
      "logps/rejected": -0.977734386920929,
      "loss": 0.8055,
      "nll_loss": 0.7298828363418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06340332329273224,
      "rewards/margins": 0.03443298488855362,
      "rewards/rejected": -0.0977783203125,
      "step": 20490
    },
    {
      "epoch": 0.7682074534859755,
      "grad_norm": 2.6391891756506984,
      "learning_rate": 5.587442366156625e-07,
      "log_odds_chosen": 0.5179687738418579,
      "log_odds_ratio": -0.5889648199081421,
      "logits/chosen": -0.705859363079071,
      "logits/rejected": -0.6273437738418579,
      "logps/chosen": -0.7074218988418579,
      "logps/rejected": -1.030859351158142,
      "loss": 0.8464,
      "nll_loss": 0.801074206829071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07070312649011612,
      "rewards/margins": 0.03231964260339737,
      "rewards/rejected": -0.10307617485523224,
      "step": 20500
    },
    {
      "epoch": 0.7685821888291394,
      "grad_norm": 2.771429668715044,
      "learning_rate": 5.58608007371503e-07,
      "log_odds_chosen": 0.626782238483429,
      "log_odds_ratio": -0.5794922113418579,
      "logits/chosen": -0.7186523675918579,
      "logits/rejected": -0.619140625,
      "logps/chosen": -0.68212890625,
      "logps/rejected": -1.091210961341858,
      "loss": 0.8311,
      "nll_loss": 0.792187511920929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.0682373046875,
      "rewards/margins": 0.040924072265625,
      "rewards/rejected": -0.10920409858226776,
      "step": 20510
    },
    {
      "epoch": 0.7689569241723033,
      "grad_norm": 2.632854518697029,
      "learning_rate": 5.584718777222606e-07,
      "log_odds_chosen": 0.6683593988418579,
      "log_odds_ratio": -0.551074206829071,
      "logits/chosen": -0.7455078363418579,
      "logits/rejected": -0.6329101324081421,
      "logps/chosen": -0.669238269329071,
      "logps/rejected": -1.0978515148162842,
      "loss": 0.8122,
      "nll_loss": 0.7630859613418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06695556640625,
      "rewards/margins": 0.0427398681640625,
      "rewards/rejected": -0.10969237983226776,
      "step": 20520
    },
    {
      "epoch": 0.7693316595154672,
      "grad_norm": 2.376233548615545,
      "learning_rate": 5.58335847546641e-07,
      "log_odds_chosen": 0.3864990174770355,
      "log_odds_ratio": -0.646484375,
      "logits/chosen": -0.7671874761581421,
      "logits/rejected": -0.6297851800918579,
      "logps/chosen": -0.66455078125,
      "logps/rejected": -0.916210949420929,
      "loss": 0.8316,
      "nll_loss": 0.706347644329071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06643066555261612,
      "rewards/margins": 0.02525634691119194,
      "rewards/rejected": -0.09169922024011612,
      "step": 20530
    },
    {
      "epoch": 0.7697063948586311,
      "grad_norm": 2.2118180273477464,
      "learning_rate": 5.58199916723556e-07,
      "log_odds_chosen": 0.5816894769668579,
      "log_odds_ratio": -0.57177734375,
      "logits/chosen": -0.7769531011581421,
      "logits/rejected": -0.66552734375,
      "logps/chosen": -0.684765636920929,
      "logps/rejected": -1.0216796398162842,
      "loss": 0.8258,
      "nll_loss": 0.7627929449081421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06849364936351776,
      "rewards/margins": 0.03367156907916069,
      "rewards/rejected": -0.10209961235523224,
      "step": 20540
    },
    {
      "epoch": 0.770081130201795,
      "grad_norm": 2.037836619430893,
      "learning_rate": 5.580640851321247e-07,
      "log_odds_chosen": 0.6194397211074829,
      "log_odds_ratio": -0.56201171875,
      "logits/chosen": -0.7880859375,
      "logits/rejected": -0.6280273199081421,
      "logps/chosen": -0.649609386920929,
      "logps/rejected": -1.0476562976837158,
      "loss": 0.8244,
      "nll_loss": 0.7474609613418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06494140625,
      "rewards/margins": 0.03984527662396431,
      "rewards/rejected": -0.10478515923023224,
      "step": 20550
    },
    {
      "epoch": 0.7704558655449588,
      "grad_norm": 2.297582222246634,
      "learning_rate": 5.579283526516706e-07,
      "log_odds_chosen": 0.42656248807907104,
      "log_odds_ratio": -0.640820324420929,
      "logits/chosen": -0.7406250238418579,
      "logits/rejected": -0.643847644329071,
      "logps/chosen": -0.6552734375,
      "logps/rejected": -0.899121105670929,
      "loss": 0.8022,
      "nll_loss": 0.7132812738418579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06549072265625,
      "rewards/margins": 0.02439422532916069,
      "rewards/rejected": -0.08984375,
      "step": 20560
    },
    {
      "epoch": 0.7708306008881227,
      "grad_norm": 2.216150606909307,
      "learning_rate": 5.577927191617239e-07,
      "log_odds_chosen": 0.45294189453125,
      "log_odds_ratio": -0.5918945074081421,
      "logits/chosen": -0.760058581829071,
      "logits/rejected": -0.645800769329071,
      "logps/chosen": -0.683300793170929,
      "logps/rejected": -0.960156261920929,
      "loss": 0.8333,
      "nll_loss": 0.7417968511581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06832275539636612,
      "rewards/margins": 0.027680207043886185,
      "rewards/rejected": -0.095947265625,
      "step": 20570
    },
    {
      "epoch": 0.7712053362312866,
      "grad_norm": 2.5332089823167205,
      "learning_rate": 5.576571845420189e-07,
      "log_odds_chosen": 0.6060546636581421,
      "log_odds_ratio": -0.572558581829071,
      "logits/chosen": -0.6919921636581421,
      "logits/rejected": -0.5833495855331421,
      "logps/chosen": -0.711621105670929,
      "logps/rejected": -1.108789086341858,
      "loss": 0.8304,
      "nll_loss": 0.8017578125,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07124023139476776,
      "rewards/margins": 0.03972472995519638,
      "rewards/rejected": -0.11091308295726776,
      "step": 20580
    },
    {
      "epoch": 0.7715800715744505,
      "grad_norm": 2.3529690842756916,
      "learning_rate": 5.575217486724946e-07,
      "log_odds_chosen": 0.29534912109375,
      "log_odds_ratio": -0.676074206829071,
      "logits/chosen": -0.7232421636581421,
      "logits/rejected": -0.7025390863418579,
      "logps/chosen": -0.693359375,
      "logps/rejected": -0.8423827886581421,
      "loss": 0.8305,
      "nll_loss": 0.777148425579071,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.0693359375,
      "rewards/margins": 0.01489105261862278,
      "rewards/rejected": -0.084228515625,
      "step": 20590
    },
    {
      "epoch": 0.7719548069176144,
      "grad_norm": 3.211800876549038,
      "learning_rate": 5.573864114332941e-07,
      "log_odds_chosen": 0.556445300579071,
      "log_odds_ratio": -0.563183605670929,
      "logits/chosen": -0.7349609136581421,
      "logits/rejected": -0.62548828125,
      "logps/chosen": -0.663867175579071,
      "logps/rejected": -0.9898437261581421,
      "loss": 0.8318,
      "nll_loss": 0.799609363079071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06632079929113388,
      "rewards/margins": 0.03258972242474556,
      "rewards/rejected": -0.09895019233226776,
      "step": 20600
    },
    {
      "epoch": 0.7723295422607783,
      "grad_norm": 2.2599458460704707,
      "learning_rate": 5.572511727047639e-07,
      "log_odds_chosen": 0.589917004108429,
      "log_odds_ratio": -0.544238269329071,
      "logits/chosen": -0.733593761920929,
      "logits/rejected": -0.6131836175918579,
      "logps/chosen": -0.664257824420929,
      "logps/rejected": -1.0361328125,
      "loss": 0.8221,
      "nll_loss": 0.775195300579071,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06636963039636612,
      "rewards/margins": 0.03726654127240181,
      "rewards/rejected": -0.10362549126148224,
      "step": 20610
    },
    {
      "epoch": 0.7727042776039422,
      "grad_norm": 2.6234527423910614,
      "learning_rate": 5.571160323674535e-07,
      "log_odds_chosen": 0.4229492247104645,
      "log_odds_ratio": -0.628125011920929,
      "logits/chosen": -0.742968738079071,
      "logits/rejected": -0.65673828125,
      "logps/chosen": -0.6396484375,
      "logps/rejected": -0.8916015625,
      "loss": 0.8346,
      "nll_loss": 0.8529297113418579,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06401367485523224,
      "rewards/margins": 0.02518768236041069,
      "rewards/rejected": -0.08928222954273224,
      "step": 20620
    },
    {
      "epoch": 0.7730790129471061,
      "grad_norm": 2.5835352867190635,
      "learning_rate": 5.569809903021156e-07,
      "log_odds_chosen": 0.39738768339157104,
      "log_odds_ratio": -0.6312500238418579,
      "logits/chosen": -0.766406238079071,
      "logits/rejected": -0.675488293170929,
      "logps/chosen": -0.7152343988418579,
      "logps/rejected": -0.9742187261581421,
      "loss": 0.8298,
      "nll_loss": 0.768359363079071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07154540717601776,
      "rewards/margins": 0.025847624987363815,
      "rewards/rejected": -0.09738769382238388,
      "step": 20630
    },
    {
      "epoch": 0.77345374829027,
      "grad_norm": 2.664835115217778,
      "learning_rate": 5.568460463897046e-07,
      "log_odds_chosen": 0.56103515625,
      "log_odds_ratio": -0.5762695074081421,
      "logits/chosen": -0.7569335699081421,
      "logits/rejected": -0.6698242425918579,
      "logps/chosen": -0.696093738079071,
      "logps/rejected": -1.0476562976837158,
      "loss": 0.8275,
      "nll_loss": 0.747851550579071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06961669772863388,
      "rewards/margins": 0.03511963039636612,
      "rewards/rejected": -0.10474853217601776,
      "step": 20640
    },
    {
      "epoch": 0.7738284836334339,
      "grad_norm": 2.4598044473706553,
      "learning_rate": 5.567112005113767e-07,
      "log_odds_chosen": 0.462646484375,
      "log_odds_ratio": -0.5870116949081421,
      "logits/chosen": -0.794140636920929,
      "logits/rejected": -0.663867175579071,
      "logps/chosen": -0.650585949420929,
      "logps/rejected": -0.9486328363418579,
      "loss": 0.8135,
      "nll_loss": 0.7171875238418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06505127251148224,
      "rewards/margins": 0.0298004150390625,
      "rewards/rejected": -0.09482421725988388,
      "step": 20650
    },
    {
      "epoch": 0.7742032189765978,
      "grad_norm": 2.190456796119735,
      "learning_rate": 5.565764525484902e-07,
      "log_odds_chosen": 0.604321300983429,
      "log_odds_ratio": -0.56494140625,
      "logits/chosen": -0.771484375,
      "logits/rejected": -0.658007800579071,
      "logps/chosen": -0.6529296636581421,
      "logps/rejected": -1.025781273841858,
      "loss": 0.8122,
      "nll_loss": 0.7769531011581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0653076171875,
      "rewards/margins": 0.037305451929569244,
      "rewards/rejected": -0.10256347805261612,
      "step": 20660
    },
    {
      "epoch": 0.7745779543197616,
      "grad_norm": 2.7492090771403492,
      "learning_rate": 5.564418023826033e-07,
      "log_odds_chosen": 0.48881834745407104,
      "log_odds_ratio": -0.595507800579071,
      "logits/chosen": -0.7718750238418579,
      "logits/rejected": -0.6841796636581421,
      "logps/chosen": -0.6553710699081421,
      "logps/rejected": -0.952929675579071,
      "loss": 0.838,
      "nll_loss": 0.77587890625,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06560058891773224,
      "rewards/margins": 0.02972869947552681,
      "rewards/rejected": -0.09521484375,
      "step": 20670
    },
    {
      "epoch": 0.7749526896629255,
      "grad_norm": 2.551970350775998,
      "learning_rate": 5.563072498954754e-07,
      "log_odds_chosen": 0.5089477300643921,
      "log_odds_ratio": -0.595898449420929,
      "logits/chosen": -0.7564452886581421,
      "logits/rejected": -0.631542980670929,
      "logps/chosen": -0.695507824420929,
      "logps/rejected": -1.031835913658142,
      "loss": 0.8389,
      "nll_loss": 0.8134765625,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06954345852136612,
      "rewards/margins": 0.03366241604089737,
      "rewards/rejected": -0.10324706882238388,
      "step": 20680
    },
    {
      "epoch": 0.7753274250060894,
      "grad_norm": 2.3345888576850884,
      "learning_rate": 5.561727949690656e-07,
      "log_odds_chosen": 0.54046630859375,
      "log_odds_ratio": -0.5718749761581421,
      "logits/chosen": -0.7046874761581421,
      "logits/rejected": -0.60791015625,
      "logps/chosen": -0.651171863079071,
      "logps/rejected": -0.9761718511581421,
      "loss": 0.8309,
      "nll_loss": 0.706835925579071,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06516113132238388,
      "rewards/margins": 0.032462310045957565,
      "rewards/rejected": -0.09758301079273224,
      "step": 20690
    },
    {
      "epoch": 0.7757021603492533,
      "grad_norm": 2.3490194464085588,
      "learning_rate": 5.560384374855327e-07,
      "log_odds_chosen": 0.679394543170929,
      "log_odds_ratio": -0.544726550579071,
      "logits/chosen": -0.7085937261581421,
      "logits/rejected": -0.6064453125,
      "logps/chosen": -0.586718738079071,
      "logps/rejected": -1.0187499523162842,
      "loss": 0.8281,
      "nll_loss": 0.782031238079071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.05869140475988388,
      "rewards/margins": 0.04319000244140625,
      "rewards/rejected": -0.10183105617761612,
      "step": 20700
    },
    {
      "epoch": 0.7760768956924172,
      "grad_norm": 2.5079808620548554,
      "learning_rate": 5.559041773272347e-07,
      "log_odds_chosen": 0.4785522520542145,
      "log_odds_ratio": -0.5787109136581421,
      "logits/chosen": -0.7875000238418579,
      "logits/rejected": -0.703320324420929,
      "logps/chosen": -0.6434570550918579,
      "logps/rejected": -0.925000011920929,
      "loss": 0.8181,
      "nll_loss": 0.732714831829071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06430663913488388,
      "rewards/margins": 0.02815551683306694,
      "rewards/rejected": -0.09249267727136612,
      "step": 20710
    },
    {
      "epoch": 0.7764516310355811,
      "grad_norm": 2.699923620924898,
      "learning_rate": 5.557700143767284e-07,
      "log_odds_chosen": 0.4983154237270355,
      "log_odds_ratio": -0.573925793170929,
      "logits/chosen": -0.7783203125,
      "logits/rejected": -0.690625011920929,
      "logps/chosen": -0.643750011920929,
      "logps/rejected": -0.921093761920929,
      "loss": 0.8497,
      "nll_loss": 0.782421886920929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06437988579273224,
      "rewards/margins": 0.027779389172792435,
      "rewards/rejected": -0.09215088188648224,
      "step": 20720
    },
    {
      "epoch": 0.776826366378745,
      "grad_norm": 2.8734292706996816,
      "learning_rate": 5.556359485167687e-07,
      "log_odds_chosen": 0.526074230670929,
      "log_odds_ratio": -0.62060546875,
      "logits/chosen": -0.6880859136581421,
      "logits/rejected": -0.616406261920929,
      "logps/chosen": -0.641406238079071,
      "logps/rejected": -0.989453136920929,
      "loss": 0.8164,
      "nll_loss": 0.787304699420929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0640869140625,
      "rewards/margins": 0.03487396240234375,
      "rewards/rejected": -0.09902343899011612,
      "step": 20730
    },
    {
      "epoch": 0.7772011017219089,
      "grad_norm": 2.673989473775602,
      "learning_rate": 5.555019796303087e-07,
      "log_odds_chosen": 0.23157958686351776,
      "log_odds_ratio": -0.6875976324081421,
      "logits/chosen": -0.7417968511581421,
      "logits/rejected": -0.6400390863418579,
      "logps/chosen": -0.76171875,
      "logps/rejected": -0.9388672113418579,
      "loss": 0.8163,
      "nll_loss": 0.781054675579071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07618407905101776,
      "rewards/margins": 0.01772613450884819,
      "rewards/rejected": -0.09384765475988388,
      "step": 20740
    },
    {
      "epoch": 0.7775758370650728,
      "grad_norm": 2.391041888188658,
      "learning_rate": 5.553681076004985e-07,
      "log_odds_chosen": 0.45001220703125,
      "log_odds_ratio": -0.6064453125,
      "logits/chosen": -0.6962890625,
      "logits/rejected": -0.61328125,
      "logps/chosen": -0.7162109613418579,
      "logps/rejected": -0.983593761920929,
      "loss": 0.8241,
      "nll_loss": 0.7818359136581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07160644233226776,
      "rewards/margins": 0.02684936486184597,
      "rewards/rejected": -0.09836425632238388,
      "step": 20750
    },
    {
      "epoch": 0.7779505724082367,
      "grad_norm": 2.0364003309246304,
      "learning_rate": 5.55234332310686e-07,
      "log_odds_chosen": 0.5714355707168579,
      "log_odds_ratio": -0.5621093511581421,
      "logits/chosen": -0.763867199420929,
      "logits/rejected": -0.6705077886581421,
      "logps/chosen": -0.6654297113418579,
      "logps/rejected": -1.0283203125,
      "loss": 0.825,
      "nll_loss": 0.7740234136581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06660155951976776,
      "rewards/margins": 0.03619079664349556,
      "rewards/rejected": -0.10277099907398224,
      "step": 20760
    },
    {
      "epoch": 0.7783253077514005,
      "grad_norm": 2.484711369015114,
      "learning_rate": 5.551006536444146e-07,
      "log_odds_chosen": 0.44343262910842896,
      "log_odds_ratio": -0.6297851800918579,
      "logits/chosen": -0.6952148675918579,
      "logits/rejected": -0.642578125,
      "logps/chosen": -0.6844726800918579,
      "logps/rejected": -0.987500011920929,
      "loss": 0.8234,
      "nll_loss": 0.743457019329071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06843261420726776,
      "rewards/margins": 0.030303955078125,
      "rewards/rejected": -0.09876708686351776,
      "step": 20770
    },
    {
      "epoch": 0.7787000430945644,
      "grad_norm": 2.623395002223735,
      "learning_rate": 5.549670714854249e-07,
      "log_odds_chosen": 0.672497570514679,
      "log_odds_ratio": -0.525683581829071,
      "logits/chosen": -0.769726574420929,
      "logits/rejected": -0.647753894329071,
      "logps/chosen": -0.606152355670929,
      "logps/rejected": -1.012304663658142,
      "loss": 0.8134,
      "nll_loss": 0.7601562738418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06060791015625,
      "rewards/margins": 0.04074859619140625,
      "rewards/rejected": -0.10123290866613388,
      "step": 20780
    },
    {
      "epoch": 0.7790747784377283,
      "grad_norm": 2.528288036643753,
      "learning_rate": 5.548335857176527e-07,
      "log_odds_chosen": 0.6805664300918579,
      "log_odds_ratio": -0.5487304925918579,
      "logits/chosen": -0.803906261920929,
      "logits/rejected": -0.672167956829071,
      "logps/chosen": -0.691601574420929,
      "logps/rejected": -1.146875023841858,
      "loss": 0.8166,
      "nll_loss": 0.755078136920929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06917724758386612,
      "rewards/margins": 0.04554595798254013,
      "rewards/rejected": -0.11459960788488388,
      "step": 20790
    },
    {
      "epoch": 0.7794495137808922,
      "grad_norm": 3.161498105873404,
      "learning_rate": 5.547001962252292e-07,
      "log_odds_chosen": 0.5967773199081421,
      "log_odds_ratio": -0.5743163824081421,
      "logits/chosen": -0.77734375,
      "logits/rejected": -0.678417980670929,
      "logps/chosen": -0.6475585699081421,
      "logps/rejected": -1.029687523841858,
      "loss": 0.8371,
      "nll_loss": 0.779296875,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06477050483226776,
      "rewards/margins": 0.03815612941980362,
      "rewards/rejected": -0.10292968899011612,
      "step": 20800
    },
    {
      "epoch": 0.7798242491240561,
      "grad_norm": 2.6123665887984533,
      "learning_rate": 5.545669028924805e-07,
      "log_odds_chosen": 0.49443358182907104,
      "log_odds_ratio": -0.591015636920929,
      "logits/chosen": -0.749707043170929,
      "logits/rejected": -0.6421874761581421,
      "logps/chosen": -0.6695312261581421,
      "logps/rejected": -0.972460925579071,
      "loss": 0.8306,
      "nll_loss": 0.7474609613418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06691894680261612,
      "rewards/margins": 0.03028717078268528,
      "rewards/rejected": -0.09724120795726776,
      "step": 20810
    },
    {
      "epoch": 0.78019898446722,
      "grad_norm": 2.690979054771548,
      "learning_rate": 5.544337056039272e-07,
      "log_odds_chosen": 0.552380383014679,
      "log_odds_ratio": -0.564746081829071,
      "logits/chosen": -0.761523425579071,
      "logits/rejected": -0.6456054449081421,
      "logps/chosen": -0.737500011920929,
      "logps/rejected": -1.089453101158142,
      "loss": 0.8209,
      "nll_loss": 0.794726550579071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07363281399011612,
      "rewards/margins": 0.03527374193072319,
      "rewards/rejected": -0.10895995795726776,
      "step": 20820
    },
    {
      "epoch": 0.7805737198103839,
      "grad_norm": 2.522831973502504,
      "learning_rate": 5.543006042442842e-07,
      "log_odds_chosen": 0.3934082090854645,
      "log_odds_ratio": -0.634472668170929,
      "logits/chosen": -0.7490234375,
      "logits/rejected": -0.651562511920929,
      "logps/chosen": -0.65576171875,
      "logps/rejected": -0.9228515625,
      "loss": 0.8353,
      "nll_loss": 0.7701171636581421,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.06563720852136612,
      "rewards/margins": 0.02671051025390625,
      "rewards/rejected": -0.09230957180261612,
      "step": 20830
    },
    {
      "epoch": 0.7809484551535478,
      "grad_norm": 2.4496628419125073,
      "learning_rate": 5.541675986984596e-07,
      "log_odds_chosen": 0.35297852754592896,
      "log_odds_ratio": -0.6675781011581421,
      "logits/chosen": -0.724316418170929,
      "logits/rejected": -0.6341797113418579,
      "logps/chosen": -0.7265625,
      "logps/rejected": -0.9546874761581421,
      "loss": 0.8269,
      "nll_loss": 0.730175793170929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07269287109375,
      "rewards/margins": 0.02279815636575222,
      "rewards/rejected": -0.09539794921875,
      "step": 20840
    },
    {
      "epoch": 0.7813231904967117,
      "grad_norm": 2.5977574933763736,
      "learning_rate": 5.540346888515549e-07,
      "log_odds_chosen": 0.4308105409145355,
      "log_odds_ratio": -0.605175793170929,
      "logits/chosen": -0.7757812738418579,
      "logits/rejected": -0.653124988079071,
      "logps/chosen": -0.6690429449081421,
      "logps/rejected": -0.9136718511581421,
      "loss": 0.7949,
      "nll_loss": 0.73681640625,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06687011569738388,
      "rewards/margins": 0.02445678785443306,
      "rewards/rejected": -0.09139404445886612,
      "step": 20850
    },
    {
      "epoch": 0.7816979258398756,
      "grad_norm": 2.418621200627784,
      "learning_rate": 5.539018745888647e-07,
      "log_odds_chosen": 0.589160144329071,
      "log_odds_ratio": -0.552539050579071,
      "logits/chosen": -0.758007824420929,
      "logits/rejected": -0.6376953125,
      "logps/chosen": -0.6797851324081421,
      "logps/rejected": -1.0417969226837158,
      "loss": 0.8096,
      "nll_loss": 0.773242175579071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06800536811351776,
      "rewards/margins": 0.03613586351275444,
      "rewards/rejected": -0.1041259765625,
      "step": 20860
    },
    {
      "epoch": 0.7820726611830395,
      "grad_norm": 2.3769504701189703,
      "learning_rate": 5.537691557958757e-07,
      "log_odds_chosen": 0.7259765863418579,
      "log_odds_ratio": -0.5057617425918579,
      "logits/chosen": -0.7574218511581421,
      "logits/rejected": -0.641796886920929,
      "logps/chosen": -0.65185546875,
      "logps/rejected": -1.0888671875,
      "loss": 0.7999,
      "nll_loss": 0.747851550579071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06527099758386612,
      "rewards/margins": 0.043609619140625,
      "rewards/rejected": -0.10881347954273224,
      "step": 20870
    },
    {
      "epoch": 0.7824473965262033,
      "grad_norm": 2.4484472580516146,
      "learning_rate": 5.536365323582665e-07,
      "log_odds_chosen": 0.71923828125,
      "log_odds_ratio": -0.513378918170929,
      "logits/chosen": -0.7352539300918579,
      "logits/rejected": -0.653613269329071,
      "logps/chosen": -0.655078113079071,
      "logps/rejected": -1.097265601158142,
      "loss": 0.8237,
      "nll_loss": 0.757031261920929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.0655517578125,
      "rewards/margins": 0.04418640211224556,
      "rewards/rejected": -0.10964355617761612,
      "step": 20880
    },
    {
      "epoch": 0.7828221318693672,
      "grad_norm": 2.3008915165698083,
      "learning_rate": 5.535040041619073e-07,
      "log_odds_chosen": 0.4607788026332855,
      "log_odds_ratio": -0.5970703363418579,
      "logits/chosen": -0.749804675579071,
      "logits/rejected": -0.668261706829071,
      "logps/chosen": -0.628710925579071,
      "logps/rejected": -0.9061523675918579,
      "loss": 0.8108,
      "nll_loss": 0.734570324420929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06284179538488388,
      "rewards/margins": 0.02771911583840847,
      "rewards/rejected": -0.09062500298023224,
      "step": 20890
    },
    {
      "epoch": 0.7831968672125311,
      "grad_norm": 2.4917549790426645,
      "learning_rate": 5.533715710928597e-07,
      "log_odds_chosen": 0.3794189393520355,
      "log_odds_ratio": -0.6449218988418579,
      "logits/chosen": -0.7191406488418579,
      "logits/rejected": -0.61181640625,
      "logps/chosen": -0.70263671875,
      "logps/rejected": -0.9306640625,
      "loss": 0.8095,
      "nll_loss": 0.750781238079071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07026366889476776,
      "rewards/margins": 0.02281494066119194,
      "rewards/rejected": -0.09309081733226776,
      "step": 20900
    },
    {
      "epoch": 0.783571602555695,
      "grad_norm": 2.179415352770508,
      "learning_rate": 5.532392330373758e-07,
      "log_odds_chosen": 0.35627442598342896,
      "log_odds_ratio": -0.6583007574081421,
      "logits/chosen": -0.7767578363418579,
      "logits/rejected": -0.686328113079071,
      "logps/chosen": -0.731249988079071,
      "logps/rejected": -0.9623047113418579,
      "loss": 0.8223,
      "nll_loss": 0.7938476800918579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.0731201171875,
      "rewards/margins": 0.023184966295957565,
      "rewards/rejected": -0.09632568061351776,
      "step": 20910
    },
    {
      "epoch": 0.7839463378988589,
      "grad_norm": 2.28587689735707,
      "learning_rate": 5.531069898818981e-07,
      "log_odds_chosen": 0.5890747308731079,
      "log_odds_ratio": -0.548046886920929,
      "logits/chosen": -0.80078125,
      "logits/rejected": -0.6732422113418579,
      "logps/chosen": -0.6773437261581421,
      "logps/rejected": -1.036718726158142,
      "loss": 0.8093,
      "nll_loss": 0.7447265386581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06773681938648224,
      "rewards/margins": 0.03592071682214737,
      "rewards/rejected": -0.10366211086511612,
      "step": 20920
    },
    {
      "epoch": 0.7843210732420228,
      "grad_norm": 2.3012384927972658,
      "learning_rate": 5.529748415130589e-07,
      "log_odds_chosen": 0.48127442598342896,
      "log_odds_ratio": -0.6209961175918579,
      "logits/chosen": -0.7440429925918579,
      "logits/rejected": -0.6302734613418579,
      "logps/chosen": -0.671191394329071,
      "logps/rejected": -0.9912109375,
      "loss": 0.8216,
      "nll_loss": 0.7665039300918579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06712646782398224,
      "rewards/margins": 0.03198852390050888,
      "rewards/rejected": -0.09904785454273224,
      "step": 20930
    },
    {
      "epoch": 0.7846958085851867,
      "grad_norm": 2.7771452649736474,
      "learning_rate": 5.528427878176804e-07,
      "log_odds_chosen": 0.47484129667282104,
      "log_odds_ratio": -0.619921863079071,
      "logits/chosen": -0.736328125,
      "logits/rejected": -0.6407226324081421,
      "logps/chosen": -0.720507800579071,
      "logps/rejected": -1.018164038658142,
      "loss": 0.8287,
      "nll_loss": 0.7724609375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07204589992761612,
      "rewards/margins": 0.02973632887005806,
      "rewards/rejected": -0.10179443657398224,
      "step": 20940
    },
    {
      "epoch": 0.7850705439283506,
      "grad_norm": 2.806267020265367,
      "learning_rate": 5.527108286827734e-07,
      "log_odds_chosen": 0.47270506620407104,
      "log_odds_ratio": -0.59716796875,
      "logits/chosen": -0.7230468988418579,
      "logits/rejected": -0.6239258050918579,
      "logps/chosen": -0.675000011920929,
      "logps/rejected": -0.954882800579071,
      "loss": 0.8087,
      "nll_loss": 0.736035168170929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06745605170726776,
      "rewards/margins": 0.02807006798684597,
      "rewards/rejected": -0.09553222358226776,
      "step": 20950
    },
    {
      "epoch": 0.7854452792715145,
      "grad_norm": 2.4502529681243996,
      "learning_rate": 5.525789639955376e-07,
      "log_odds_chosen": 0.3946899473667145,
      "log_odds_ratio": -0.606249988079071,
      "logits/chosen": -0.7281249761581421,
      "logits/rejected": -0.648730456829071,
      "logps/chosen": -0.6634765863418579,
      "logps/rejected": -0.912890613079071,
      "loss": 0.8167,
      "nll_loss": 0.732617199420929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06641845405101776,
      "rewards/margins": 0.02492065355181694,
      "rewards/rejected": -0.09133300930261612,
      "step": 20960
    },
    {
      "epoch": 0.7858200146146784,
      "grad_norm": 2.703242569395085,
      "learning_rate": 5.524471936433611e-07,
      "log_odds_chosen": 0.571826159954071,
      "log_odds_ratio": -0.604785144329071,
      "logits/chosen": -0.7515624761581421,
      "logits/rejected": -0.6634765863418579,
      "logps/chosen": -0.740234375,
      "logps/rejected": -1.119726538658142,
      "loss": 0.8304,
      "nll_loss": 0.8038085699081421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.073974609375,
      "rewards/margins": 0.03800659254193306,
      "rewards/rejected": -0.11201171576976776,
      "step": 20970
    },
    {
      "epoch": 0.7861947499578422,
      "grad_norm": 2.779810443610996,
      "learning_rate": 5.523155175138198e-07,
      "log_odds_chosen": 0.569628894329071,
      "log_odds_ratio": -0.548828125,
      "logits/chosen": -0.760937511920929,
      "logits/rejected": -0.62939453125,
      "logps/chosen": -0.6548827886581421,
      "logps/rejected": -0.996874988079071,
      "loss": 0.8205,
      "nll_loss": 0.7162109613418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.0654296875,
      "rewards/margins": 0.03424225002527237,
      "rewards/rejected": -0.09968261420726776,
      "step": 20980
    },
    {
      "epoch": 0.7865694853010061,
      "grad_norm": 2.4679031144997454,
      "learning_rate": 5.52183935494677e-07,
      "log_odds_chosen": 0.5234130620956421,
      "log_odds_ratio": -0.573535144329071,
      "logits/chosen": -0.729296863079071,
      "logits/rejected": -0.6416991949081421,
      "logps/chosen": -0.6546875238418579,
      "logps/rejected": -0.9847656488418579,
      "loss": 0.8224,
      "nll_loss": 0.7705078125,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.0654296875,
      "rewards/margins": 0.03302764892578125,
      "rewards/rejected": -0.09857177734375,
      "step": 20990
    },
    {
      "epoch": 0.78694422064417,
      "grad_norm": 2.6453334825033545,
      "learning_rate": 5.520524474738833e-07,
      "log_odds_chosen": 0.608642578125,
      "log_odds_ratio": -0.583300769329071,
      "logits/chosen": -0.7222656011581421,
      "logits/rejected": -0.650585949420929,
      "logps/chosen": -0.669921875,
      "logps/rejected": -1.0732421875,
      "loss": 0.8195,
      "nll_loss": 0.7261718511581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06697998195886612,
      "rewards/margins": 0.04035339504480362,
      "rewards/rejected": -0.10732422024011612,
      "step": 21000
    },
    {
      "epoch": 0.7873189559873339,
      "grad_norm": 2.4649346346302927,
      "learning_rate": 5.519210533395758e-07,
      "log_odds_chosen": 0.5882568359375,
      "log_odds_ratio": -0.5604492425918579,
      "logits/chosen": -0.7455078363418579,
      "logits/rejected": -0.5824218988418579,
      "logps/chosen": -0.6832031011581421,
      "logps/rejected": -1.058007836341858,
      "loss": 0.8017,
      "nll_loss": 0.7774413824081421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06834717094898224,
      "rewards/margins": 0.03740234300494194,
      "rewards/rejected": -0.10576172173023224,
      "step": 21010
    },
    {
      "epoch": 0.7876936913304978,
      "grad_norm": 2.7558884548557407,
      "learning_rate": 5.517897529800779e-07,
      "log_odds_chosen": 0.5342041254043579,
      "log_odds_ratio": -0.581250011920929,
      "logits/chosen": -0.784960925579071,
      "logits/rejected": -0.652636706829071,
      "logps/chosen": -0.65625,
      "logps/rejected": -1.0166015625,
      "loss": 0.8118,
      "nll_loss": 0.736328125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06563720852136612,
      "rewards/margins": 0.036077119410037994,
      "rewards/rejected": -0.10166015475988388,
      "step": 21020
    },
    {
      "epoch": 0.7880684266736617,
      "grad_norm": 2.4958085172195585,
      "learning_rate": 5.51658546283899e-07,
      "log_odds_chosen": 0.5819946527481079,
      "log_odds_ratio": -0.62109375,
      "logits/chosen": -0.744335949420929,
      "logits/rejected": -0.6358398199081421,
      "logps/chosen": -0.66357421875,
      "logps/rejected": -1.054101586341858,
      "loss": 0.8148,
      "nll_loss": 0.7466796636581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06636963039636612,
      "rewards/margins": 0.039081573486328125,
      "rewards/rejected": -0.10541991889476776,
      "step": 21030
    },
    {
      "epoch": 0.7884431620168256,
      "grad_norm": 2.892426094659579,
      "learning_rate": 5.515274331397337e-07,
      "log_odds_chosen": 0.4895263612270355,
      "log_odds_ratio": -0.5884765386581421,
      "logits/chosen": -0.770312488079071,
      "logits/rejected": -0.6446288824081421,
      "logps/chosen": -0.6830078363418579,
      "logps/rejected": -0.9970703125,
      "loss": 0.8079,
      "nll_loss": 0.7646484375,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06817626953125,
      "rewards/margins": 0.03145141527056694,
      "rewards/rejected": -0.09969482570886612,
      "step": 21040
    },
    {
      "epoch": 0.7888178973599895,
      "grad_norm": 2.394143625034607,
      "learning_rate": 5.51396413436462e-07,
      "log_odds_chosen": 0.3450683653354645,
      "log_odds_ratio": -0.652148425579071,
      "logits/chosen": -0.7171875238418579,
      "logits/rejected": -0.651660144329071,
      "logps/chosen": -0.7085937261581421,
      "logps/rejected": -0.9496093988418579,
      "loss": 0.8269,
      "nll_loss": 0.7562500238418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07088623195886612,
      "rewards/margins": 0.024066925048828125,
      "rewards/rejected": -0.09489746391773224,
      "step": 21050
    },
    {
      "epoch": 0.7891926327031534,
      "grad_norm": 2.504762426562734,
      "learning_rate": 5.512654870631487e-07,
      "log_odds_chosen": 0.6505492925643921,
      "log_odds_ratio": -0.559863269329071,
      "logits/chosen": -0.732617199420929,
      "logits/rejected": -0.656054675579071,
      "logps/chosen": -0.6971679925918579,
      "logps/rejected": -1.093359351158142,
      "loss": 0.8225,
      "nll_loss": 0.77392578125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06965331733226776,
      "rewards/margins": 0.03961143642663956,
      "rewards/rejected": -0.10936279594898224,
      "step": 21060
    },
    {
      "epoch": 0.7895673680463173,
      "grad_norm": 2.0297947127168228,
      "learning_rate": 5.511346539090424e-07,
      "log_odds_chosen": 0.397705078125,
      "log_odds_ratio": -0.6182616949081421,
      "logits/chosen": -0.7603515386581421,
      "logits/rejected": -0.676953136920929,
      "logps/chosen": -0.7037109136581421,
      "logps/rejected": -0.9554687738418579,
      "loss": 0.8221,
      "nll_loss": 0.8163086175918579,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07042236626148224,
      "rewards/margins": 0.0251007080078125,
      "rewards/rejected": -0.09552001953125,
      "step": 21070
    },
    {
      "epoch": 0.7899421033894812,
      "grad_norm": 2.254172968458483,
      "learning_rate": 5.51003913863576e-07,
      "log_odds_chosen": 0.3173828125,
      "log_odds_ratio": -0.654980480670929,
      "logits/chosen": -0.7503906488418579,
      "logits/rejected": -0.683886706829071,
      "logps/chosen": -0.6786133050918579,
      "logps/rejected": -0.8941406011581421,
      "loss": 0.8038,
      "nll_loss": 0.7542968988418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06788329780101776,
      "rewards/margins": 0.02153320237994194,
      "rewards/rejected": -0.08944091945886612,
      "step": 21080
    },
    {
      "epoch": 0.790316838732645,
      "grad_norm": 2.2027964520019943,
      "learning_rate": 5.508732668163659e-07,
      "log_odds_chosen": 0.5808471441268921,
      "log_odds_ratio": -0.583300769329071,
      "logits/chosen": -0.715039074420929,
      "logits/rejected": -0.6449218988418579,
      "logps/chosen": -0.6468750238418579,
      "logps/rejected": -0.9750000238418579,
      "loss": 0.8187,
      "nll_loss": 0.739550769329071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06464843451976776,
      "rewards/margins": 0.032765962183475494,
      "rewards/rejected": -0.09748534858226776,
      "step": 21090
    },
    {
      "epoch": 0.7906915740758089,
      "grad_norm": 2.6258450110470846,
      "learning_rate": 5.507427126572114e-07,
      "log_odds_chosen": 0.49798583984375,
      "log_odds_ratio": -0.5884765386581421,
      "logits/chosen": -0.747851550579071,
      "logits/rejected": -0.631640613079071,
      "logps/chosen": -0.640625,
      "logps/rejected": -0.9478515386581421,
      "loss": 0.811,
      "nll_loss": 0.772265613079071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06405029445886612,
      "rewards/margins": 0.030779266729950905,
      "rewards/rejected": -0.09483642876148224,
      "step": 21100
    },
    {
      "epoch": 0.7910663094189728,
      "grad_norm": 2.8173901718935017,
      "learning_rate": 5.506122512760947e-07,
      "log_odds_chosen": 0.6092529296875,
      "log_odds_ratio": -0.5774902105331421,
      "logits/chosen": -0.8011718988418579,
      "logits/rejected": -0.702929675579071,
      "logps/chosen": -0.6761718988418579,
      "logps/rejected": -1.091406226158142,
      "loss": 0.8241,
      "nll_loss": 0.785351574420929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06761474907398224,
      "rewards/margins": 0.04149627685546875,
      "rewards/rejected": -0.10905762016773224,
      "step": 21110
    },
    {
      "epoch": 0.7914410447621367,
      "grad_norm": 2.4503809082576664,
      "learning_rate": 5.504818825631803e-07,
      "log_odds_chosen": 0.534130871295929,
      "log_odds_ratio": -0.6064453125,
      "logits/chosen": -0.7587890625,
      "logits/rejected": -0.6661132574081421,
      "logps/chosen": -0.6998046636581421,
      "logps/rejected": -1.0380859375,
      "loss": 0.8235,
      "nll_loss": 0.8052734136581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06997070461511612,
      "rewards/margins": 0.03382454067468643,
      "rewards/rejected": -0.10374756157398224,
      "step": 21120
    },
    {
      "epoch": 0.7918157801053006,
      "grad_norm": 2.372403600405812,
      "learning_rate": 5.503516064088146e-07,
      "log_odds_chosen": 0.502429187297821,
      "log_odds_ratio": -0.615429699420929,
      "logits/chosen": -0.7237304449081421,
      "logits/rejected": -0.600292980670929,
      "logps/chosen": -0.6583007574081421,
      "logps/rejected": -0.978710949420929,
      "loss": 0.8287,
      "nll_loss": 0.729785144329071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06583251804113388,
      "rewards/margins": 0.031987763941287994,
      "rewards/rejected": -0.09785155951976776,
      "step": 21130
    },
    {
      "epoch": 0.7921905154484645,
      "grad_norm": 2.2491063801517477,
      "learning_rate": 5.502214227035259e-07,
      "log_odds_chosen": 0.539294421672821,
      "log_odds_ratio": -0.5863281488418579,
      "logits/chosen": -0.7525390386581421,
      "logits/rejected": -0.65478515625,
      "logps/chosen": -0.6700195074081421,
      "logps/rejected": -1.012304663658142,
      "loss": 0.8229,
      "nll_loss": 0.7763671875,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.0670166015625,
      "rewards/margins": 0.0341949462890625,
      "rewards/rejected": -0.10125732421875,
      "step": 21140
    },
    {
      "epoch": 0.7925652507916284,
      "grad_norm": 2.4436342304751033,
      "learning_rate": 5.500913313380231e-07,
      "log_odds_chosen": 0.530468761920929,
      "log_odds_ratio": -0.5824218988418579,
      "logits/chosen": -0.740039050579071,
      "logits/rejected": -0.668164074420929,
      "logps/chosen": -0.6797851324081421,
      "logps/rejected": -1.001367211341858,
      "loss": 0.8222,
      "nll_loss": 0.7476562261581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0679931640625,
      "rewards/margins": 0.03214569017291069,
      "rewards/rejected": -0.10013427585363388,
      "step": 21150
    },
    {
      "epoch": 0.7929399861347923,
      "grad_norm": 2.860943898590582,
      "learning_rate": 5.499613322031964e-07,
      "log_odds_chosen": 0.5586181879043579,
      "log_odds_ratio": -0.5718749761581421,
      "logits/chosen": -0.7749999761581421,
      "logits/rejected": -0.666308581829071,
      "logps/chosen": -0.6302734613418579,
      "logps/rejected": -0.974414050579071,
      "loss": 0.814,
      "nll_loss": 0.7289062738418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06306152045726776,
      "rewards/margins": 0.034366607666015625,
      "rewards/rejected": -0.09744872897863388,
      "step": 21160
    },
    {
      "epoch": 0.7933147214779562,
      "grad_norm": 2.2419753188968956,
      "learning_rate": 5.498314251901161e-07,
      "log_odds_chosen": 0.60272216796875,
      "log_odds_ratio": -0.57958984375,
      "logits/chosen": -0.7593749761581421,
      "logits/rejected": -0.6737304925918579,
      "logps/chosen": -0.696093738079071,
      "logps/rejected": -1.0939452648162842,
      "loss": 0.8039,
      "nll_loss": 0.7876952886581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06964111328125,
      "rewards/margins": 0.039789579808712006,
      "rewards/rejected": -0.10938720405101776,
      "step": 21170
    },
    {
      "epoch": 0.7936894568211201,
      "grad_norm": 2.473692803814651,
      "learning_rate": 5.497016101900326e-07,
      "log_odds_chosen": 0.3722167909145355,
      "log_odds_ratio": -0.616503894329071,
      "logits/chosen": -0.790234386920929,
      "logits/rejected": -0.680371105670929,
      "logps/chosen": -0.697070300579071,
      "logps/rejected": -0.9263671636581421,
      "loss": 0.8326,
      "nll_loss": 0.780468761920929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0697021484375,
      "rewards/margins": 0.02294921875,
      "rewards/rejected": -0.09267578274011612,
      "step": 21180
    },
    {
      "epoch": 0.7940641921642839,
      "grad_norm": 3.165242342619016,
      "learning_rate": 5.495718870943762e-07,
      "log_odds_chosen": 0.4620605409145355,
      "log_odds_ratio": -0.608105480670929,
      "logits/chosen": -0.72802734375,
      "logits/rejected": -0.637011706829071,
      "logps/chosen": -0.6499999761581421,
      "logps/rejected": -0.936718761920929,
      "loss": 0.8065,
      "nll_loss": 0.780078113079071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06497802585363388,
      "rewards/margins": 0.028660964220762253,
      "rewards/rejected": -0.09372558444738388,
      "step": 21190
    },
    {
      "epoch": 0.7944389275074478,
      "grad_norm": 2.8593299870976505,
      "learning_rate": 5.494422557947561e-07,
      "log_odds_chosen": 0.591113269329071,
      "log_odds_ratio": -0.570117175579071,
      "logits/chosen": -0.7431640625,
      "logits/rejected": -0.65478515625,
      "logps/chosen": -0.678417980670929,
      "logps/rejected": -1.062109351158142,
      "loss": 0.8193,
      "nll_loss": 0.72314453125,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06790771335363388,
      "rewards/margins": 0.03838653489947319,
      "rewards/rejected": -0.10627441108226776,
      "step": 21200
    },
    {
      "epoch": 0.7948136628506117,
      "grad_norm": 2.407267325112152,
      "learning_rate": 5.493127161829605e-07,
      "log_odds_chosen": 0.572155773639679,
      "log_odds_ratio": -0.546093761920929,
      "logits/chosen": -0.7212890386581421,
      "logits/rejected": -0.665722668170929,
      "logps/chosen": -0.6615234613418579,
      "logps/rejected": -1.024999976158142,
      "loss": 0.8176,
      "nll_loss": 0.7708984613418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.066162109375,
      "rewards/margins": 0.036301422864198685,
      "rewards/rejected": -0.10247802734375,
      "step": 21210
    },
    {
      "epoch": 0.7951883981937756,
      "grad_norm": 2.874141696419555,
      "learning_rate": 5.491832681509561e-07,
      "log_odds_chosen": 0.533032238483429,
      "log_odds_ratio": -0.5877929925918579,
      "logits/chosen": -0.781054675579071,
      "logits/rejected": -0.676562488079071,
      "logps/chosen": -0.6763671636581421,
      "logps/rejected": -1.006738305091858,
      "loss": 0.827,
      "nll_loss": 0.783398449420929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06755371391773224,
      "rewards/margins": 0.03303069993853569,
      "rewards/rejected": -0.10056152194738388,
      "step": 21220
    },
    {
      "epoch": 0.7955631335369395,
      "grad_norm": 2.49490998215956,
      "learning_rate": 5.490539115908881e-07,
      "log_odds_chosen": 0.5372314453125,
      "log_odds_ratio": -0.5663086175918579,
      "logits/chosen": -0.7476562261581421,
      "logits/rejected": -0.668749988079071,
      "logps/chosen": -0.625,
      "logps/rejected": -0.9365234375,
      "loss": 0.816,
      "nll_loss": 0.713183581829071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06251220405101776,
      "rewards/margins": 0.03119964525103569,
      "rewards/rejected": -0.09375,
      "step": 21230
    },
    {
      "epoch": 0.7959378688801034,
      "grad_norm": 2.5001552639523053,
      "learning_rate": 5.489246463950787e-07,
      "log_odds_chosen": 0.41802978515625,
      "log_odds_ratio": -0.6050781011581421,
      "logits/chosen": -0.75341796875,
      "logits/rejected": -0.6631835699081421,
      "logps/chosen": -0.6639648675918579,
      "logps/rejected": -0.9140625,
      "loss": 0.81,
      "nll_loss": 0.78125,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06644286960363388,
      "rewards/margins": 0.02498016320168972,
      "rewards/rejected": -0.09135742485523224,
      "step": 21240
    },
    {
      "epoch": 0.7963126042232673,
      "grad_norm": 2.3213401133518827,
      "learning_rate": 5.487954724560283e-07,
      "log_odds_chosen": 0.565258800983429,
      "log_odds_ratio": -0.5708984136581421,
      "logits/chosen": -0.844921886920929,
      "logits/rejected": -0.731738269329071,
      "logps/chosen": -0.658886730670929,
      "logps/rejected": -1.0398437976837158,
      "loss": 0.809,
      "nll_loss": 0.7201172113418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06588134914636612,
      "rewards/margins": 0.03815002366900444,
      "rewards/rejected": -0.10402832180261612,
      "step": 21250
    },
    {
      "epoch": 0.7966873395664312,
      "grad_norm": 2.55826337157751,
      "learning_rate": 5.486663896664134e-07,
      "log_odds_chosen": 0.535290539264679,
      "log_odds_ratio": -0.5859375,
      "logits/chosen": -0.754101574420929,
      "logits/rejected": -0.66259765625,
      "logps/chosen": -0.6373046636581421,
      "logps/rejected": -0.934374988079071,
      "loss": 0.8218,
      "nll_loss": 0.762890636920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.063720703125,
      "rewards/margins": 0.02972564660012722,
      "rewards/rejected": -0.0933837890625,
      "step": 21260
    },
    {
      "epoch": 0.7970620749095951,
      "grad_norm": 3.9512565020745574,
      "learning_rate": 5.485373979190881e-07,
      "log_odds_chosen": 0.509716808795929,
      "log_odds_ratio": -0.579296886920929,
      "logits/chosen": -0.7464843988418579,
      "logits/rejected": -0.614941418170929,
      "logps/chosen": -0.646191418170929,
      "logps/rejected": -0.966992199420929,
      "loss": 0.8163,
      "nll_loss": 0.75341796875,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06466064602136612,
      "rewards/margins": 0.03204955905675888,
      "rewards/rejected": -0.09674072265625,
      "step": 21270
    },
    {
      "epoch": 0.797436810252759,
      "grad_norm": 2.2960177903930648,
      "learning_rate": 5.484084971070817e-07,
      "log_odds_chosen": 0.608386218547821,
      "log_odds_ratio": -0.5640624761581421,
      "logits/chosen": -0.7548828125,
      "logits/rejected": -0.63232421875,
      "logps/chosen": -0.6768554449081421,
      "logps/rejected": -1.0558593273162842,
      "loss": 0.8249,
      "nll_loss": 0.7724609375,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06765136867761612,
      "rewards/margins": 0.03793029859662056,
      "rewards/rejected": -0.10557861626148224,
      "step": 21280
    },
    {
      "epoch": 0.7978115455959229,
      "grad_norm": 2.968795209384304,
      "learning_rate": 5.482796871236004e-07,
      "log_odds_chosen": 0.609179675579071,
      "log_odds_ratio": -0.5669921636581421,
      "logits/chosen": -0.758105456829071,
      "logits/rejected": -0.6363281011581421,
      "logps/chosen": -0.6919921636581421,
      "logps/rejected": -1.0880858898162842,
      "loss": 0.8214,
      "nll_loss": 0.7847656011581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06918945163488388,
      "rewards/margins": 0.03952179104089737,
      "rewards/rejected": -0.10869140923023224,
      "step": 21290
    },
    {
      "epoch": 0.7981862809390867,
      "grad_norm": 2.3697257612971474,
      "learning_rate": 5.481509678620253e-07,
      "log_odds_chosen": 0.5130370855331421,
      "log_odds_ratio": -0.586230456829071,
      "logits/chosen": -0.742871105670929,
      "logits/rejected": -0.660839855670929,
      "logps/chosen": -0.6539062261581421,
      "logps/rejected": -0.959179699420929,
      "loss": 0.8323,
      "nll_loss": 0.7425781488418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06538085639476776,
      "rewards/margins": 0.030484771355986595,
      "rewards/rejected": -0.09580077975988388,
      "step": 21300
    },
    {
      "epoch": 0.7985610162822506,
      "grad_norm": 2.239732673945638,
      "learning_rate": 5.480223392159126e-07,
      "log_odds_chosen": 0.5567260980606079,
      "log_odds_ratio": -0.568408191204071,
      "logits/chosen": -0.7705078125,
      "logits/rejected": -0.6822265386581421,
      "logps/chosen": -0.6153320074081421,
      "logps/rejected": -0.9281250238418579,
      "loss": 0.8318,
      "nll_loss": 0.720898449420929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06158447265625,
      "rewards/margins": 0.03127136081457138,
      "rewards/rejected": -0.09285888820886612,
      "step": 21310
    },
    {
      "epoch": 0.7989357516254145,
      "grad_norm": 2.579998430493437,
      "learning_rate": 5.478938010789937e-07,
      "log_odds_chosen": 0.591357409954071,
      "log_odds_ratio": -0.560546875,
      "logits/chosen": -0.812304675579071,
      "logits/rejected": -0.6636718511581421,
      "logps/chosen": -0.676562488079071,
      "logps/rejected": -1.0615234375,
      "loss": 0.8045,
      "nll_loss": 0.7412109375,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06771240383386612,
      "rewards/margins": 0.03838501125574112,
      "rewards/rejected": -0.10605468600988388,
      "step": 21320
    },
    {
      "epoch": 0.7993104869685784,
      "grad_norm": 2.4857605016444264,
      "learning_rate": 5.477653533451739e-07,
      "log_odds_chosen": 0.4811035096645355,
      "log_odds_ratio": -0.6356445550918579,
      "logits/chosen": -0.771484375,
      "logits/rejected": -0.675000011920929,
      "logps/chosen": -0.7490234375,
      "logps/rejected": -1.0847656726837158,
      "loss": 0.8242,
      "nll_loss": 0.804492175579071,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07491455227136612,
      "rewards/margins": 0.03359680250287056,
      "rewards/rejected": -0.10855712741613388,
      "step": 21330
    },
    {
      "epoch": 0.7996852223117423,
      "grad_norm": 2.326728220264609,
      "learning_rate": 5.476369959085329e-07,
      "log_odds_chosen": 0.60223388671875,
      "log_odds_ratio": -0.55615234375,
      "logits/chosen": -0.760937511920929,
      "logits/rejected": -0.6548827886581421,
      "logps/chosen": -0.6451171636581421,
      "logps/rejected": -1.0458984375,
      "loss": 0.8044,
      "nll_loss": 0.7845703363418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06452636420726776,
      "rewards/margins": 0.040135957300662994,
      "rewards/rejected": -0.10468749701976776,
      "step": 21340
    },
    {
      "epoch": 0.8000599576549062,
      "grad_norm": 2.5669471127933012,
      "learning_rate": 5.475087286633238e-07,
      "log_odds_chosen": 0.7051025629043579,
      "log_odds_ratio": -0.5248047113418579,
      "logits/chosen": -0.837695300579071,
      "logits/rejected": -0.69677734375,
      "logps/chosen": -0.6787109375,
      "logps/rejected": -1.1437499523162842,
      "loss": 0.8339,
      "nll_loss": 0.798632800579071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06779785454273224,
      "rewards/margins": 0.04658203199505806,
      "rewards/rejected": -0.11440429836511612,
      "step": 21350
    },
    {
      "epoch": 0.8004346929980701,
      "grad_norm": 2.2884460920794925,
      "learning_rate": 5.473805515039733e-07,
      "log_odds_chosen": 0.42437744140625,
      "log_odds_ratio": -0.6146484613418579,
      "logits/chosen": -0.791015625,
      "logits/rejected": -0.6742187738418579,
      "logps/chosen": -0.6265624761581421,
      "logps/rejected": -0.896484375,
      "loss": 0.8314,
      "nll_loss": 0.73046875,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0626220703125,
      "rewards/margins": 0.02701873704791069,
      "rewards/rejected": -0.08969726413488388,
      "step": 21360
    },
    {
      "epoch": 0.800809428341234,
      "grad_norm": 2.160975313902927,
      "learning_rate": 5.472524643250806e-07,
      "log_odds_chosen": 0.40467530488967896,
      "log_odds_ratio": -0.616015613079071,
      "logits/chosen": -0.798828125,
      "logits/rejected": -0.703320324420929,
      "logps/chosen": -0.7041015625,
      "logps/rejected": -0.9603515863418579,
      "loss": 0.8264,
      "nll_loss": 0.803906261920929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07038573920726776,
      "rewards/margins": 0.02565917931497097,
      "rewards/rejected": -0.09602050483226776,
      "step": 21370
    },
    {
      "epoch": 0.8011841636843979,
      "grad_norm": 2.6784551711735225,
      "learning_rate": 5.471244670214178e-07,
      "log_odds_chosen": 0.77911376953125,
      "log_odds_ratio": -0.49775391817092896,
      "logits/chosen": -0.775390625,
      "logits/rejected": -0.646484375,
      "logps/chosen": -0.624218761920929,
      "logps/rejected": -1.109960913658142,
      "loss": 0.8267,
      "nll_loss": 0.725390613079071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06242675706744194,
      "rewards/margins": 0.04858856275677681,
      "rewards/rejected": -0.11101074516773224,
      "step": 21380
    },
    {
      "epoch": 0.8015588990275618,
      "grad_norm": 2.417599770917121,
      "learning_rate": 5.469965594879291e-07,
      "log_odds_chosen": 0.45135498046875,
      "log_odds_ratio": -0.625683605670929,
      "logits/chosen": -0.729296863079071,
      "logits/rejected": -0.6435546875,
      "logps/chosen": -0.690234363079071,
      "logps/rejected": -0.9937499761581421,
      "loss": 0.8361,
      "nll_loss": 0.7875000238418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06895752251148224,
      "rewards/margins": 0.03037719801068306,
      "rewards/rejected": -0.09938964992761612,
      "step": 21390
    },
    {
      "epoch": 0.8019336343707256,
      "grad_norm": 2.2638832845655403,
      "learning_rate": 5.468687416197306e-07,
      "log_odds_chosen": 0.5338989496231079,
      "log_odds_ratio": -0.574999988079071,
      "logits/chosen": -0.780078113079071,
      "logits/rejected": -0.634960949420929,
      "logps/chosen": -0.680859386920929,
      "logps/rejected": -1.0419921875,
      "loss": 0.8303,
      "nll_loss": 0.7197265625,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06809081882238388,
      "rewards/margins": 0.036042023450136185,
      "rewards/rejected": -0.10405273735523224,
      "step": 21400
    },
    {
      "epoch": 0.8023083697138895,
      "grad_norm": 2.5278116626852287,
      "learning_rate": 5.467410133121096e-07,
      "log_odds_chosen": 0.42888182401657104,
      "log_odds_ratio": -0.6025390625,
      "logits/chosen": -0.731249988079071,
      "logits/rejected": -0.667285144329071,
      "logps/chosen": -0.697070300579071,
      "logps/rejected": -0.96484375,
      "loss": 0.8301,
      "nll_loss": 0.7894531488418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06975097954273224,
      "rewards/margins": 0.026688385754823685,
      "rewards/rejected": -0.096435546875,
      "step": 21410
    },
    {
      "epoch": 0.8026831050570534,
      "grad_norm": 2.5234969833169987,
      "learning_rate": 5.466133744605251e-07,
      "log_odds_chosen": 0.5143676996231079,
      "log_odds_ratio": -0.588574230670929,
      "logits/chosen": -0.800000011920929,
      "logits/rejected": -0.6880859136581421,
      "logps/chosen": -0.646484375,
      "logps/rejected": -0.98046875,
      "loss": 0.8139,
      "nll_loss": 0.767773449420929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06461181491613388,
      "rewards/margins": 0.033385466784238815,
      "rewards/rejected": -0.09805908054113388,
      "step": 21420
    },
    {
      "epoch": 0.8030578404002173,
      "grad_norm": 2.2498320312629807,
      "learning_rate": 5.464858249606063e-07,
      "log_odds_chosen": 0.4752441346645355,
      "log_odds_ratio": -0.59814453125,
      "logits/chosen": -0.806445300579071,
      "logits/rejected": -0.671093761920929,
      "logps/chosen": -0.694140613079071,
      "logps/rejected": -1.0134766101837158,
      "loss": 0.7985,
      "nll_loss": 0.7529296875,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06937255710363388,
      "rewards/margins": 0.03189849853515625,
      "rewards/rejected": -0.10124512016773224,
      "step": 21430
    },
    {
      "epoch": 0.8034325757433812,
      "grad_norm": 3.4388041771597755,
      "learning_rate": 5.46358364708153e-07,
      "log_odds_chosen": 0.556689441204071,
      "log_odds_ratio": -0.594531238079071,
      "logits/chosen": -0.7738281488418579,
      "logits/rejected": -0.684277355670929,
      "logps/chosen": -0.6830078363418579,
      "logps/rejected": -1.0353515148162842,
      "loss": 0.832,
      "nll_loss": 0.763671875,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.068359375,
      "rewards/margins": 0.035247802734375,
      "rewards/rejected": -0.10358886420726776,
      "step": 21440
    },
    {
      "epoch": 0.8038073110865451,
      "grad_norm": 2.279467426908722,
      "learning_rate": 5.46230993599135e-07,
      "log_odds_chosen": 0.5100952386856079,
      "log_odds_ratio": -0.576171875,
      "logits/chosen": -0.7978515625,
      "logits/rejected": -0.6712890863418579,
      "logps/chosen": -0.674121081829071,
      "logps/rejected": -1.004296898841858,
      "loss": 0.8189,
      "nll_loss": 0.7647460699081421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06737060844898224,
      "rewards/margins": 0.0330352783203125,
      "rewards/rejected": -0.10041503608226776,
      "step": 21450
    },
    {
      "epoch": 0.804182046429709,
      "grad_norm": 3.171344239876498,
      "learning_rate": 5.46103711529692e-07,
      "log_odds_chosen": 0.80767822265625,
      "log_odds_ratio": -0.501220703125,
      "logits/chosen": -0.788281261920929,
      "logits/rejected": -0.628710925579071,
      "logps/chosen": -0.6239258050918579,
      "logps/rejected": -1.1306641101837158,
      "loss": 0.8162,
      "nll_loss": 0.7857421636581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06243896484375,
      "rewards/margins": 0.05070037767291069,
      "rewards/rejected": -0.11308594048023224,
      "step": 21460
    },
    {
      "epoch": 0.8045567817728729,
      "grad_norm": 2.2048783202877904,
      "learning_rate": 5.459765183961328e-07,
      "log_odds_chosen": 0.4959960877895355,
      "log_odds_ratio": -0.5843750238418579,
      "logits/chosen": -0.808398425579071,
      "logits/rejected": -0.664843738079071,
      "logps/chosen": -0.6416991949081421,
      "logps/rejected": -0.9478515386581421,
      "loss": 0.8094,
      "nll_loss": 0.760937511920929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06417236477136612,
      "rewards/margins": 0.030655670911073685,
      "rewards/rejected": -0.09483642876148224,
      "step": 21470
    },
    {
      "epoch": 0.8049315171160368,
      "grad_norm": 2.5346911159133865,
      "learning_rate": 5.458494140949352e-07,
      "log_odds_chosen": 0.6200195550918579,
      "log_odds_ratio": -0.5342773199081421,
      "logits/chosen": -0.8226562738418579,
      "logits/rejected": -0.724609375,
      "logps/chosen": -0.650585949420929,
      "logps/rejected": -1.023828148841858,
      "loss": 0.829,
      "nll_loss": 0.7572265863418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06507568061351776,
      "rewards/margins": 0.03733215481042862,
      "rewards/rejected": -0.10231933742761612,
      "step": 21480
    },
    {
      "epoch": 0.8053062524592007,
      "grad_norm": 2.4965966161845805,
      "learning_rate": 5.457223985227455e-07,
      "log_odds_chosen": 0.46588134765625,
      "log_odds_ratio": -0.6167968511581421,
      "logits/chosen": -0.8003906011581421,
      "logits/rejected": -0.691210925579071,
      "logps/chosen": -0.691601574420929,
      "logps/rejected": -0.999804675579071,
      "loss": 0.8143,
      "nll_loss": 0.7525390386581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06911621242761612,
      "rewards/margins": 0.03088836744427681,
      "rewards/rejected": -0.10009765625,
      "step": 21490
    },
    {
      "epoch": 0.8056809878023646,
      "grad_norm": 2.2602570996316946,
      "learning_rate": 5.455954715763787e-07,
      "log_odds_chosen": 0.5014892816543579,
      "log_odds_ratio": -0.585742175579071,
      "logits/chosen": -0.7962890863418579,
      "logits/rejected": -0.666210949420929,
      "logps/chosen": -0.663281261920929,
      "logps/rejected": -0.9859374761581421,
      "loss": 0.7991,
      "nll_loss": 0.8037109375,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06632079929113388,
      "rewards/margins": 0.03228912502527237,
      "rewards/rejected": -0.09862060844898224,
      "step": 21500
    },
    {
      "epoch": 0.8060557231455284,
      "grad_norm": 2.771877708379672,
      "learning_rate": 5.454686331528174e-07,
      "log_odds_chosen": 0.5465087890625,
      "log_odds_ratio": -0.5801757574081421,
      "logits/chosen": -0.771777331829071,
      "logits/rejected": -0.6314452886581421,
      "logps/chosen": -0.7230468988418579,
      "logps/rejected": -1.0947265625,
      "loss": 0.8357,
      "nll_loss": 0.804882824420929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07233886420726776,
      "rewards/margins": 0.03713226318359375,
      "rewards/rejected": -0.1094970703125,
      "step": 21510
    },
    {
      "epoch": 0.8064304584886923,
      "grad_norm": 2.3840313685938326,
      "learning_rate": 5.453418831492118e-07,
      "log_odds_chosen": 0.4488525390625,
      "log_odds_ratio": -0.623242199420929,
      "logits/chosen": -0.7989257574081421,
      "logits/rejected": -0.689453125,
      "logps/chosen": -0.672656238079071,
      "logps/rejected": -0.9842773675918579,
      "loss": 0.8118,
      "nll_loss": 0.7455078363418579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06722412258386612,
      "rewards/margins": 0.03119201585650444,
      "rewards/rejected": -0.09847412258386612,
      "step": 21520
    },
    {
      "epoch": 0.8068051938318562,
      "grad_norm": 2.651221043571853,
      "learning_rate": 5.452152214628792e-07,
      "log_odds_chosen": 0.5667968988418579,
      "log_odds_ratio": -0.5733398199081421,
      "logits/chosen": -0.7593749761581421,
      "logits/rejected": -0.660449206829071,
      "logps/chosen": -0.652148425579071,
      "logps/rejected": -1.015625,
      "loss": 0.8183,
      "nll_loss": 0.7529296875,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06519775092601776,
      "rewards/margins": 0.03629608079791069,
      "rewards/rejected": -0.10158691555261612,
      "step": 21530
    },
    {
      "epoch": 0.8071799291750201,
      "grad_norm": 3.2115802318858617,
      "learning_rate": 5.450886479913041e-07,
      "log_odds_chosen": 0.451416015625,
      "log_odds_ratio": -0.645458996295929,
      "logits/chosen": -0.720898449420929,
      "logits/rejected": -0.593945324420929,
      "logps/chosen": -0.692187488079071,
      "logps/rejected": -1.019921898841858,
      "loss": 0.8095,
      "nll_loss": 0.747851550579071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.0692138671875,
      "rewards/margins": 0.032729338854551315,
      "rewards/rejected": -0.10198974609375,
      "step": 21540
    },
    {
      "epoch": 0.807554664518184,
      "grad_norm": 2.465963857246678,
      "learning_rate": 5.449621626321372e-07,
      "log_odds_chosen": 0.4427856504917145,
      "log_odds_ratio": -0.6200195550918579,
      "logits/chosen": -0.790234386920929,
      "logits/rejected": -0.6939452886581421,
      "logps/chosen": -0.67578125,
      "logps/rejected": -0.9664062261581421,
      "loss": 0.8069,
      "nll_loss": 0.7503906488418579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06759033352136612,
      "rewards/margins": 0.029085541144013405,
      "rewards/rejected": -0.09660644829273224,
      "step": 21550
    },
    {
      "epoch": 0.8079293998613479,
      "grad_norm": 2.395161732876328,
      "learning_rate": 5.448357652831955e-07,
      "log_odds_chosen": 0.363037109375,
      "log_odds_ratio": -0.6343749761581421,
      "logits/chosen": -0.7158203125,
      "logits/rejected": -0.6480468511581421,
      "logps/chosen": -0.6592773199081421,
      "logps/rejected": -0.8765624761581421,
      "loss": 0.811,
      "nll_loss": 0.741406261920929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06593017280101776,
      "rewards/margins": 0.02168731763958931,
      "rewards/rejected": -0.08767089992761612,
      "step": 21560
    },
    {
      "epoch": 0.8083041352045118,
      "grad_norm": 2.1962715584381796,
      "learning_rate": 5.44709455842462e-07,
      "log_odds_chosen": 0.54144287109375,
      "log_odds_ratio": -0.575976550579071,
      "logits/chosen": -0.7115234136581421,
      "logits/rejected": -0.619140625,
      "logps/chosen": -0.68017578125,
      "logps/rejected": -1.018164038658142,
      "loss": 0.8228,
      "nll_loss": 0.8046875,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06798096001148224,
      "rewards/margins": 0.033786773681640625,
      "rewards/rejected": -0.101806640625,
      "step": 21570
    },
    {
      "epoch": 0.8086788705476757,
      "grad_norm": 2.593458521434961,
      "learning_rate": 5.445832342080844e-07,
      "log_odds_chosen": 0.4848876893520355,
      "log_odds_ratio": -0.6064453125,
      "logits/chosen": -0.7388671636581421,
      "logits/rejected": -0.6170898675918579,
      "logps/chosen": -0.6513671875,
      "logps/rejected": -0.958984375,
      "loss": 0.8223,
      "nll_loss": 0.7437499761581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06508789211511612,
      "rewards/margins": 0.03082885779440403,
      "rewards/rejected": -0.09602050483226776,
      "step": 21580
    },
    {
      "epoch": 0.8090536058908396,
      "grad_norm": 2.353839856412983,
      "learning_rate": 5.44457100278377e-07,
      "log_odds_chosen": 0.48768311738967896,
      "log_odds_ratio": -0.619140625,
      "logits/chosen": -0.7505859136581421,
      "logits/rejected": -0.6767578125,
      "logps/chosen": -0.6454101800918579,
      "logps/rejected": -0.954882800579071,
      "loss": 0.8095,
      "nll_loss": 0.746874988079071,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0645751953125,
      "rewards/margins": 0.03091583214700222,
      "rewards/rejected": -0.095458984375,
      "step": 21590
    },
    {
      "epoch": 0.8094283412340035,
      "grad_norm": 2.278856181116072,
      "learning_rate": 5.443310539518173e-07,
      "log_odds_chosen": 0.4364013671875,
      "log_odds_ratio": -0.6329101324081421,
      "logits/chosen": -0.7396484613418579,
      "logits/rejected": -0.6348632574081421,
      "logps/chosen": -0.72119140625,
      "logps/rejected": -1.0158202648162842,
      "loss": 0.822,
      "nll_loss": 0.7886718511581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07205810397863388,
      "rewards/margins": 0.02947235107421875,
      "rewards/rejected": -0.10158691555261612,
      "step": 21600
    },
    {
      "epoch": 0.8098030765771673,
      "grad_norm": 2.4966634410231663,
      "learning_rate": 5.442050951270483e-07,
      "log_odds_chosen": 0.694811999797821,
      "log_odds_ratio": -0.5538085699081421,
      "logits/chosen": -0.753125011920929,
      "logits/rejected": -0.6747070550918579,
      "logps/chosen": -0.7269531488418579,
      "logps/rejected": -1.162109375,
      "loss": 0.8062,
      "nll_loss": 0.7962890863418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07268066704273224,
      "rewards/margins": 0.04357147216796875,
      "rewards/rejected": -0.1162109375,
      "step": 21610
    },
    {
      "epoch": 0.8101778119203312,
      "grad_norm": 2.4241976991608394,
      "learning_rate": 5.440792237028766e-07,
      "log_odds_chosen": 0.5194091796875,
      "log_odds_ratio": -0.578808605670929,
      "logits/chosen": -0.752734363079071,
      "logits/rejected": -0.636523425579071,
      "logps/chosen": -0.6126953363418579,
      "logps/rejected": -0.9183593988418579,
      "loss": 0.818,
      "nll_loss": 0.731640636920929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06129150465130806,
      "rewards/margins": 0.030565643683075905,
      "rewards/rejected": -0.0919189453125,
      "step": 21620
    },
    {
      "epoch": 0.8105525472634951,
      "grad_norm": 2.3016534245093263,
      "learning_rate": 5.439534395782728e-07,
      "log_odds_chosen": 0.52215576171875,
      "log_odds_ratio": -0.581738293170929,
      "logits/chosen": -0.729296863079071,
      "logits/rejected": -0.664746105670929,
      "logps/chosen": -0.6405273675918579,
      "logps/rejected": -0.958789050579071,
      "loss": 0.7993,
      "nll_loss": 0.7220703363418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06402587890625,
      "rewards/margins": 0.031821440905332565,
      "rewards/rejected": -0.09580077975988388,
      "step": 21630
    },
    {
      "epoch": 0.810927282606659,
      "grad_norm": 3.099793249748889,
      "learning_rate": 5.438277426523708e-07,
      "log_odds_chosen": 0.4701171815395355,
      "log_odds_ratio": -0.5831054449081421,
      "logits/chosen": -0.8099609613418579,
      "logits/rejected": -0.6919921636581421,
      "logps/chosen": -0.638867199420929,
      "logps/rejected": -0.9349609613418579,
      "loss": 0.8016,
      "nll_loss": 0.758984386920929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06386718899011612,
      "rewards/margins": 0.02959594689309597,
      "rewards/rejected": -0.093505859375,
      "step": 21640
    },
    {
      "epoch": 0.8113020179498229,
      "grad_norm": 2.387110133904593,
      "learning_rate": 5.437021328244679e-07,
      "log_odds_chosen": 0.4993896484375,
      "log_odds_ratio": -0.6009765863418579,
      "logits/chosen": -0.705859363079071,
      "logits/rejected": -0.628125011920929,
      "logps/chosen": -0.6781250238418579,
      "logps/rejected": -0.9886718988418579,
      "loss": 0.8134,
      "nll_loss": 0.75,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06778564304113388,
      "rewards/margins": 0.031118392944335938,
      "rewards/rejected": -0.09884033352136612,
      "step": 21650
    },
    {
      "epoch": 0.8116767532929868,
      "grad_norm": 3.624744012020794,
      "learning_rate": 5.435766099940235e-07,
      "log_odds_chosen": 0.49022215604782104,
      "log_odds_ratio": -0.59716796875,
      "logits/chosen": -0.813671886920929,
      "logits/rejected": -0.69091796875,
      "logps/chosen": -0.6859375238418579,
      "logps/rejected": -1.0068359375,
      "loss": 0.8266,
      "nll_loss": 0.807421863079071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.068603515625,
      "rewards/margins": 0.03212432935833931,
      "rewards/rejected": -0.10074462741613388,
      "step": 21660
    },
    {
      "epoch": 0.8120514886361507,
      "grad_norm": 2.424544757225175,
      "learning_rate": 5.434511740606597e-07,
      "log_odds_chosen": 0.56591796875,
      "log_odds_ratio": -0.578808605670929,
      "logits/chosen": -0.7484375238418579,
      "logits/rejected": -0.641406238079071,
      "logps/chosen": -0.697460949420929,
      "logps/rejected": -1.0681641101837158,
      "loss": 0.8054,
      "nll_loss": 0.766796886920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06976318359375,
      "rewards/margins": 0.037055205553770065,
      "rewards/rejected": -0.10673828423023224,
      "step": 21670
    },
    {
      "epoch": 0.8124262239793146,
      "grad_norm": 2.17965332762861,
      "learning_rate": 5.433258249241613e-07,
      "log_odds_chosen": 0.631701648235321,
      "log_odds_ratio": -0.56103515625,
      "logits/chosen": -0.7562500238418579,
      "logits/rejected": -0.642285168170929,
      "logps/chosen": -0.65283203125,
      "logps/rejected": -1.0578124523162842,
      "loss": 0.8046,
      "nll_loss": 0.6888672113418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06527099758386612,
      "rewards/margins": 0.04053611680865288,
      "rewards/rejected": -0.10578612983226776,
      "step": 21680
    },
    {
      "epoch": 0.8128009593224785,
      "grad_norm": 2.6706793096679884,
      "learning_rate": 5.432005624844737e-07,
      "log_odds_chosen": 0.5026611089706421,
      "log_odds_ratio": -0.60107421875,
      "logits/chosen": -0.714648425579071,
      "logits/rejected": -0.673535168170929,
      "logps/chosen": -0.664843738079071,
      "logps/rejected": -0.941699206829071,
      "loss": 0.8077,
      "nll_loss": 0.753125011920929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06650390475988388,
      "rewards/margins": 0.02762145921587944,
      "rewards/rejected": -0.09409179538488388,
      "step": 21690
    },
    {
      "epoch": 0.8131756946656424,
      "grad_norm": 2.238259414806943,
      "learning_rate": 5.430753866417044e-07,
      "log_odds_chosen": 0.5075927972793579,
      "log_odds_ratio": -0.596484363079071,
      "logits/chosen": -0.7544921636581421,
      "logits/rejected": -0.6412109136581421,
      "logps/chosen": -0.6460937261581421,
      "logps/rejected": -0.971484363079071,
      "loss": 0.8017,
      "nll_loss": 0.7408202886581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.0645751953125,
      "rewards/margins": 0.0325469970703125,
      "rewards/rejected": -0.09724120795726776,
      "step": 21700
    },
    {
      "epoch": 0.8135504300088063,
      "grad_norm": 2.1740468454460387,
      "learning_rate": 5.429502972961222e-07,
      "log_odds_chosen": 0.4168457090854645,
      "log_odds_ratio": -0.62841796875,
      "logits/chosen": -0.712695300579071,
      "logits/rejected": -0.648144543170929,
      "logps/chosen": -0.6341797113418579,
      "logps/rejected": -0.866406261920929,
      "loss": 0.8419,
      "nll_loss": 0.7271484136581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06343994289636612,
      "rewards/margins": 0.023151397705078125,
      "rewards/rejected": -0.08663330227136612,
      "step": 21710
    },
    {
      "epoch": 0.8139251653519701,
      "grad_norm": 2.5751791595553115,
      "learning_rate": 5.428252943481558e-07,
      "log_odds_chosen": 0.3997558653354645,
      "log_odds_ratio": -0.601367175579071,
      "logits/chosen": -0.73974609375,
      "logits/rejected": -0.663769543170929,
      "logps/chosen": -0.6610351800918579,
      "logps/rejected": -0.921093761920929,
      "loss": 0.7908,
      "nll_loss": 0.738476574420929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06613769382238388,
      "rewards/margins": 0.02593536302447319,
      "rewards/rejected": -0.09202881157398224,
      "step": 21720
    },
    {
      "epoch": 0.814299900695134,
      "grad_norm": 2.300892522011265,
      "learning_rate": 5.427003776983951e-07,
      "log_odds_chosen": 0.5169006586074829,
      "log_odds_ratio": -0.57568359375,
      "logits/chosen": -0.7216796875,
      "logits/rejected": -0.641406238079071,
      "logps/chosen": -0.640429675579071,
      "logps/rejected": -0.9613281488418579,
      "loss": 0.8098,
      "nll_loss": 0.755078136920929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06403808295726776,
      "rewards/margins": 0.03212928771972656,
      "rewards/rejected": -0.09616699069738388,
      "step": 21730
    },
    {
      "epoch": 0.8146746360382979,
      "grad_norm": 2.703159263989283,
      "learning_rate": 5.425755472475893e-07,
      "log_odds_chosen": 0.4532226622104645,
      "log_odds_ratio": -0.575976550579071,
      "logits/chosen": -0.760937511920929,
      "logits/rejected": -0.6932617425918579,
      "logps/chosen": -0.6307617425918579,
      "logps/rejected": -0.8921874761581421,
      "loss": 0.7991,
      "nll_loss": 0.722851574420929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06303711235523224,
      "rewards/margins": 0.02615508995950222,
      "rewards/rejected": -0.08920898288488388,
      "step": 21740
    },
    {
      "epoch": 0.8150493713814618,
      "grad_norm": 2.59036626582169,
      "learning_rate": 5.424508028966484e-07,
      "log_odds_chosen": 0.4999633729457855,
      "log_odds_ratio": -0.6119140386581421,
      "logits/chosen": -0.712695300579071,
      "logits/rejected": -0.595507800579071,
      "logps/chosen": -0.6888672113418579,
      "logps/rejected": -1.011816382408142,
      "loss": 0.8131,
      "nll_loss": 0.713574230670929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06885986030101776,
      "rewards/margins": 0.03225860744714737,
      "rewards/rejected": -0.10122070461511612,
      "step": 21750
    },
    {
      "epoch": 0.8154241067246257,
      "grad_norm": 2.71984759101798,
      "learning_rate": 5.423261445466404e-07,
      "log_odds_chosen": 0.492919921875,
      "log_odds_ratio": -0.596972644329071,
      "logits/chosen": -0.7494140863418579,
      "logits/rejected": -0.6426757574081421,
      "logps/chosen": -0.683886706829071,
      "logps/rejected": -0.982226550579071,
      "loss": 0.8044,
      "nll_loss": 0.7734375,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06840820610523224,
      "rewards/margins": 0.02988891676068306,
      "rewards/rejected": -0.09833984076976776,
      "step": 21760
    },
    {
      "epoch": 0.8157988420677896,
      "grad_norm": 2.1856372579371244,
      "learning_rate": 5.422015720987936e-07,
      "log_odds_chosen": 0.5093628168106079,
      "log_odds_ratio": -0.5927734375,
      "logits/chosen": -0.7925781011581421,
      "logits/rejected": -0.6773437261581421,
      "logps/chosen": -0.681640625,
      "logps/rejected": -1.019921898841858,
      "loss": 0.8048,
      "nll_loss": 0.7392578125,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06821288913488388,
      "rewards/margins": 0.03374481201171875,
      "rewards/rejected": -0.10190429538488388,
      "step": 21770
    },
    {
      "epoch": 0.8161735774109535,
      "grad_norm": 2.4056465400039118,
      "learning_rate": 5.420770854544944e-07,
      "log_odds_chosen": 0.39763182401657104,
      "log_odds_ratio": -0.625781238079071,
      "logits/chosen": -0.751757800579071,
      "logits/rejected": -0.651074230670929,
      "logps/chosen": -0.6539062261581421,
      "logps/rejected": -0.907031238079071,
      "loss": 0.8095,
      "nll_loss": 0.7217773199081421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06541748344898224,
      "rewards/margins": 0.025304412469267845,
      "rewards/rejected": -0.09072265774011612,
      "step": 21780
    },
    {
      "epoch": 0.8165483127541174,
      "grad_norm": 3.103468905876388,
      "learning_rate": 5.419526845152879e-07,
      "log_odds_chosen": 0.5424743890762329,
      "log_odds_ratio": -0.5874999761581421,
      "logits/chosen": -0.7642577886581421,
      "logits/rejected": -0.6329101324081421,
      "logps/chosen": -0.695507824420929,
      "logps/rejected": -1.0451171398162842,
      "loss": 0.8023,
      "nll_loss": 0.7808593511581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06950683891773224,
      "rewards/margins": 0.034949492663145065,
      "rewards/rejected": -0.10451660305261612,
      "step": 21790
    },
    {
      "epoch": 0.8169230480972813,
      "grad_norm": 2.4026631344338525,
      "learning_rate": 5.418283691828771e-07,
      "log_odds_chosen": 0.4933837950229645,
      "log_odds_ratio": -0.576367199420929,
      "logits/chosen": -0.78759765625,
      "logits/rejected": -0.669726550579071,
      "logps/chosen": -0.66796875,
      "logps/rejected": -0.9722656011581421,
      "loss": 0.8061,
      "nll_loss": 0.779296875,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06683349609375,
      "rewards/margins": 0.030428314581513405,
      "rewards/rejected": -0.09719238430261612,
      "step": 21800
    },
    {
      "epoch": 0.8172977834404452,
      "grad_norm": 2.3945982346506702,
      "learning_rate": 5.417041393591227e-07,
      "log_odds_chosen": 0.42437744140625,
      "log_odds_ratio": -0.6122070550918579,
      "logits/chosen": -0.769726574420929,
      "logits/rejected": -0.65673828125,
      "logps/chosen": -0.665234386920929,
      "logps/rejected": -0.9322265386581421,
      "loss": 0.8066,
      "nll_loss": 0.7298828363418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06647948920726776,
      "rewards/margins": 0.026735687628388405,
      "rewards/rejected": -0.09320068359375,
      "step": 21810
    },
    {
      "epoch": 0.817672518783609,
      "grad_norm": 2.9463515976234302,
      "learning_rate": 5.415799949460433e-07,
      "log_odds_chosen": 0.6039673089981079,
      "log_odds_ratio": -0.5552734136581421,
      "logits/chosen": -0.783203125,
      "logits/rejected": -0.666796863079071,
      "logps/chosen": -0.6783202886581421,
      "logps/rejected": -1.065820336341858,
      "loss": 0.812,
      "nll_loss": 0.7447265386581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06783447414636612,
      "rewards/margins": 0.0388031005859375,
      "rewards/rejected": -0.10661621391773224,
      "step": 21820
    },
    {
      "epoch": 0.8180472541267729,
      "grad_norm": 2.4473394620672235,
      "learning_rate": 5.414559358458137e-07,
      "log_odds_chosen": 0.6731811761856079,
      "log_odds_ratio": -0.5462890863418579,
      "logits/chosen": -0.80859375,
      "logits/rejected": -0.698046863079071,
      "logps/chosen": -0.6234375238418579,
      "logps/rejected": -1.0613281726837158,
      "loss": 0.8154,
      "nll_loss": 0.7466796636581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06232910230755806,
      "rewards/margins": 0.04375915601849556,
      "rewards/rejected": -0.10612793266773224,
      "step": 21830
    },
    {
      "epoch": 0.8184219894699368,
      "grad_norm": 2.5926948526960025,
      "learning_rate": 5.413319619607667e-07,
      "log_odds_chosen": 0.3970947265625,
      "log_odds_ratio": -0.6302734613418579,
      "logits/chosen": -0.706835925579071,
      "logits/rejected": -0.6107422113418579,
      "logps/chosen": -0.685742199420929,
      "logps/rejected": -0.923632800579071,
      "loss": 0.8134,
      "nll_loss": 0.7738281488418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06856689602136612,
      "rewards/margins": 0.023850250989198685,
      "rewards/rejected": -0.09245605766773224,
      "step": 21840
    },
    {
      "epoch": 0.8187967248131007,
      "grad_norm": 2.528459194894015,
      "learning_rate": 5.412080731933907e-07,
      "log_odds_chosen": 0.39362794160842896,
      "log_odds_ratio": -0.627148449420929,
      "logits/chosen": -0.689257800579071,
      "logits/rejected": -0.645312488079071,
      "logps/chosen": -0.692578136920929,
      "logps/rejected": -0.931835949420929,
      "loss": 0.8233,
      "nll_loss": 0.774218738079071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06929931789636612,
      "rewards/margins": 0.02393035963177681,
      "rewards/rejected": -0.0931396484375,
      "step": 21850
    },
    {
      "epoch": 0.8191714601562646,
      "grad_norm": 2.4971685752702713,
      "learning_rate": 5.410842694463302e-07,
      "log_odds_chosen": 0.4550537168979645,
      "log_odds_ratio": -0.626953125,
      "logits/chosen": -0.740039050579071,
      "logits/rejected": -0.64892578125,
      "logps/chosen": -0.7392578125,
      "logps/rejected": -1.052734375,
      "loss": 0.8418,
      "nll_loss": 0.8345702886581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07386474311351776,
      "rewards/margins": 0.03134918212890625,
      "rewards/rejected": -0.10527344048023224,
      "step": 21860
    },
    {
      "epoch": 0.8195461954994285,
      "grad_norm": 2.614988511805042,
      "learning_rate": 5.409605506223862e-07,
      "log_odds_chosen": 0.515551745891571,
      "log_odds_ratio": -0.591015636920929,
      "logits/chosen": -0.7119140625,
      "logits/rejected": -0.652539074420929,
      "logps/chosen": -0.658203125,
      "logps/rejected": -0.956250011920929,
      "loss": 0.7937,
      "nll_loss": 0.7158203125,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06584472954273224,
      "rewards/margins": 0.02982635423541069,
      "rewards/rejected": -0.09572754055261612,
      "step": 21870
    },
    {
      "epoch": 0.8199209308425924,
      "grad_norm": 2.884423891847974,
      "learning_rate": 5.408369166245146e-07,
      "log_odds_chosen": 0.47349852323532104,
      "log_odds_ratio": -0.60986328125,
      "logits/chosen": -0.7154296636581421,
      "logits/rejected": -0.6226562261581421,
      "logps/chosen": -0.6771484613418579,
      "logps/rejected": -0.9830077886581421,
      "loss": 0.8045,
      "nll_loss": 0.76171875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06772460788488388,
      "rewards/margins": 0.03054504469037056,
      "rewards/rejected": -0.09825439751148224,
      "step": 21880
    },
    {
      "epoch": 0.8202956661857563,
      "grad_norm": 2.2182261232890337,
      "learning_rate": 5.407133673558267e-07,
      "log_odds_chosen": 0.568737804889679,
      "log_odds_ratio": -0.5511718988418579,
      "logits/chosen": -0.7451171875,
      "logits/rejected": -0.6591796875,
      "logps/chosen": -0.6265624761581421,
      "logps/rejected": -0.95703125,
      "loss": 0.7977,
      "nll_loss": 0.7466796636581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06268310546875,
      "rewards/margins": 0.03304900974035263,
      "rewards/rejected": -0.09580077975988388,
      "step": 21890
    },
    {
      "epoch": 0.8206704015289202,
      "grad_norm": 3.484900908860969,
      "learning_rate": 5.405899027195888e-07,
      "log_odds_chosen": 0.5579833984375,
      "log_odds_ratio": -0.6001952886581421,
      "logits/chosen": -0.729296863079071,
      "logits/rejected": -0.6358398199081421,
      "logps/chosen": -0.650585949420929,
      "logps/rejected": -0.994921863079071,
      "loss": 0.796,
      "nll_loss": 0.712207019329071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06510009616613388,
      "rewards/margins": 0.0343170166015625,
      "rewards/rejected": -0.09941406548023224,
      "step": 21900
    },
    {
      "epoch": 0.8210451368720841,
      "grad_norm": 2.5590182742972005,
      "learning_rate": 5.404665226192212e-07,
      "log_odds_chosen": 0.5765136480331421,
      "log_odds_ratio": -0.5672851800918579,
      "logits/chosen": -0.745800793170929,
      "logits/rejected": -0.659960925579071,
      "logps/chosen": -0.6615234613418579,
      "logps/rejected": -1.003320336341858,
      "loss": 0.8176,
      "nll_loss": 0.699999988079071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06618652492761612,
      "rewards/margins": 0.03414001315832138,
      "rewards/rejected": -0.10031738132238388,
      "step": 21910
    },
    {
      "epoch": 0.8214198722152479,
      "grad_norm": 2.5897092078870965,
      "learning_rate": 5.403432269582991e-07,
      "log_odds_chosen": 0.587695300579071,
      "log_odds_ratio": -0.565625011920929,
      "logits/chosen": -0.776562511920929,
      "logits/rejected": -0.656542956829071,
      "logps/chosen": -0.6375976800918579,
      "logps/rejected": -0.9867187738418579,
      "loss": 0.8013,
      "nll_loss": 0.7271484136581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06380615383386612,
      "rewards/margins": 0.034847259521484375,
      "rewards/rejected": -0.09864501655101776,
      "step": 21920
    },
    {
      "epoch": 0.8217946075584118,
      "grad_norm": 2.9700156804331965,
      "learning_rate": 5.402200156405514e-07,
      "log_odds_chosen": 0.6402343511581421,
      "log_odds_ratio": -0.580859363079071,
      "logits/chosen": -0.7572265863418579,
      "logits/rejected": -0.6324218511581421,
      "logps/chosen": -0.667675793170929,
      "logps/rejected": -1.0890624523162842,
      "loss": 0.8047,
      "nll_loss": 0.7489258050918579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06679687649011612,
      "rewards/margins": 0.04208984225988388,
      "rewards/rejected": -0.10885009914636612,
      "step": 21930
    },
    {
      "epoch": 0.8221693429015757,
      "grad_norm": 2.394137205298014,
      "learning_rate": 5.400968885698603e-07,
      "log_odds_chosen": 0.42506104707717896,
      "log_odds_ratio": -0.616015613079071,
      "logits/chosen": -0.829882800579071,
      "logits/rejected": -0.702343761920929,
      "logps/chosen": -0.6851562261581421,
      "logps/rejected": -0.9579101800918579,
      "loss": 0.8067,
      "nll_loss": 0.7445312738418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06856689602136612,
      "rewards/margins": 0.02722930908203125,
      "rewards/rejected": -0.09576416015625,
      "step": 21940
    },
    {
      "epoch": 0.8225440782447396,
      "grad_norm": 2.3541728059412823,
      "learning_rate": 5.399738456502616e-07,
      "log_odds_chosen": 0.524243175983429,
      "log_odds_ratio": -0.599804699420929,
      "logits/chosen": -0.737500011920929,
      "logits/rejected": -0.65185546875,
      "logps/chosen": -0.6883789300918579,
      "logps/rejected": -1.027734398841858,
      "loss": 0.8265,
      "nll_loss": 0.7486327886581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06883545219898224,
      "rewards/margins": 0.03394775465130806,
      "rewards/rejected": -0.10273437201976776,
      "step": 21950
    },
    {
      "epoch": 0.8229188135879035,
      "grad_norm": 2.5053383960858975,
      "learning_rate": 5.398508867859439e-07,
      "log_odds_chosen": 0.509228527545929,
      "log_odds_ratio": -0.594433605670929,
      "logits/chosen": -0.7943359613418579,
      "logits/rejected": -0.6709960699081421,
      "logps/chosen": -0.625195324420929,
      "logps/rejected": -0.9535156488418579,
      "loss": 0.7888,
      "nll_loss": 0.7372070550918579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06248779222369194,
      "rewards/margins": 0.03286895900964737,
      "rewards/rejected": -0.09541015326976776,
      "step": 21960
    },
    {
      "epoch": 0.8232935489310674,
      "grad_norm": 2.580610061995996,
      "learning_rate": 5.397280118812487e-07,
      "log_odds_chosen": 0.36107176542282104,
      "log_odds_ratio": -0.617968738079071,
      "logits/chosen": -0.805468738079071,
      "logits/rejected": -0.675976574420929,
      "logps/chosen": -0.6605468988418579,
      "logps/rejected": -0.8658202886581421,
      "loss": 0.7957,
      "nll_loss": 0.7613281011581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06608887016773224,
      "rewards/margins": 0.02053375169634819,
      "rewards/rejected": -0.08664550632238388,
      "step": 21970
    },
    {
      "epoch": 0.8236682842742313,
      "grad_norm": 2.9420106229212513,
      "learning_rate": 5.396052208406695e-07,
      "log_odds_chosen": 0.578320324420929,
      "log_odds_ratio": -0.576855480670929,
      "logits/chosen": -0.7357422113418579,
      "logits/rejected": -0.64990234375,
      "logps/chosen": -0.646484375,
      "logps/rejected": -0.995312511920929,
      "loss": 0.8069,
      "nll_loss": 0.758984386920929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06461181491613388,
      "rewards/margins": 0.03486023098230362,
      "rewards/rejected": -0.09951172024011612,
      "step": 21980
    },
    {
      "epoch": 0.8240430196173952,
      "grad_norm": 2.4304768094676286,
      "learning_rate": 5.394825135688519e-07,
      "log_odds_chosen": 0.2856689393520355,
      "log_odds_ratio": -0.6737304925918579,
      "logits/chosen": -0.7767578363418579,
      "logits/rejected": -0.716015636920929,
      "logps/chosen": -0.752246081829071,
      "logps/rejected": -0.9458984136581421,
      "loss": 0.824,
      "nll_loss": 0.7889648675918579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07526855170726776,
      "rewards/margins": 0.01937408372759819,
      "rewards/rejected": -0.09465332329273224,
      "step": 21990
    },
    {
      "epoch": 0.8244177549605591,
      "grad_norm": 2.900639920391162,
      "learning_rate": 5.393598899705937e-07,
      "log_odds_chosen": 0.588732898235321,
      "log_odds_ratio": -0.5897461175918579,
      "logits/chosen": -0.7144531011581421,
      "logits/rejected": -0.643750011920929,
      "logps/chosen": -0.667773425579071,
      "logps/rejected": -1.054101586341858,
      "loss": 0.8292,
      "nll_loss": 0.7890625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0667724609375,
      "rewards/margins": 0.03861083835363388,
      "rewards/rejected": -0.10539551079273224,
      "step": 22000
    },
    {
      "epoch": 0.824792490303723,
      "grad_norm": 2.4258336058350367,
      "learning_rate": 5.392373499508432e-07,
      "log_odds_chosen": 0.533642590045929,
      "log_odds_ratio": -0.5625,
      "logits/chosen": -0.758007824420929,
      "logits/rejected": -0.6513671875,
      "logps/chosen": -0.673046886920929,
      "logps/rejected": -0.986328125,
      "loss": 0.8053,
      "nll_loss": 0.7486327886581421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06727294623851776,
      "rewards/margins": 0.03138580173254013,
      "rewards/rejected": -0.09868164360523224,
      "step": 22010
    },
    {
      "epoch": 0.8251672256468869,
      "grad_norm": 2.5799345358543313,
      "learning_rate": 5.391148934147006e-07,
      "log_odds_chosen": 0.58428955078125,
      "log_odds_ratio": -0.6011718511581421,
      "logits/chosen": -0.757617175579071,
      "logits/rejected": -0.65869140625,
      "logps/chosen": -0.676074206829071,
      "logps/rejected": -1.070898413658142,
      "loss": 0.8003,
      "nll_loss": 0.742382824420929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06756591796875,
      "rewards/margins": 0.039397429674863815,
      "rewards/rejected": -0.10703124850988388,
      "step": 22020
    },
    {
      "epoch": 0.8255419609900507,
      "grad_norm": 2.4285065841231592,
      "learning_rate": 5.389925202674166e-07,
      "log_odds_chosen": 0.6222168207168579,
      "log_odds_ratio": -0.575878918170929,
      "logits/chosen": -0.767773449420929,
      "logits/rejected": -0.688281238079071,
      "logps/chosen": -0.669140636920929,
      "logps/rejected": -1.0751953125,
      "loss": 0.8333,
      "nll_loss": 0.824414074420929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06694336235523224,
      "rewards/margins": 0.040557861328125,
      "rewards/rejected": -0.10751952975988388,
      "step": 22030
    },
    {
      "epoch": 0.8259166963332146,
      "grad_norm": 2.5819252329022366,
      "learning_rate": 5.388702304143923e-07,
      "log_odds_chosen": 0.5150146484375,
      "log_odds_ratio": -0.6025390625,
      "logits/chosen": -0.8021484613418579,
      "logits/rejected": -0.715624988079071,
      "logps/chosen": -0.6728515625,
      "logps/rejected": -1.0177733898162842,
      "loss": 0.7876,
      "nll_loss": 0.69384765625,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06729736179113388,
      "rewards/margins": 0.03454894945025444,
      "rewards/rejected": -0.101806640625,
      "step": 22040
    },
    {
      "epoch": 0.8262914316763785,
      "grad_norm": 2.1943611773077127,
      "learning_rate": 5.387480237611791e-07,
      "log_odds_chosen": 0.614550769329071,
      "log_odds_ratio": -0.541210949420929,
      "logits/chosen": -0.7083984613418579,
      "logits/rejected": -0.658886730670929,
      "logps/chosen": -0.65625,
      "logps/rejected": -1.031640648841858,
      "loss": 0.7939,
      "nll_loss": 0.7314453125,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06564941257238388,
      "rewards/margins": 0.0374908447265625,
      "rewards/rejected": -0.10312499850988388,
      "step": 22050
    },
    {
      "epoch": 0.8266661670195424,
      "grad_norm": 2.8857255315902677,
      "learning_rate": 5.386259002134781e-07,
      "log_odds_chosen": 0.4966796934604645,
      "log_odds_ratio": -0.5843750238418579,
      "logits/chosen": -0.794921875,
      "logits/rejected": -0.7015625238418579,
      "logps/chosen": -0.6436523199081421,
      "logps/rejected": -0.9419921636581421,
      "loss": 0.8273,
      "nll_loss": 0.7193359136581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0643310546875,
      "rewards/margins": 0.02985992468893528,
      "rewards/rejected": -0.09418945014476776,
      "step": 22060
    },
    {
      "epoch": 0.8270409023627063,
      "grad_norm": 2.396454276630909,
      "learning_rate": 5.385038596771402e-07,
      "log_odds_chosen": 0.6543823480606079,
      "log_odds_ratio": -0.532519519329071,
      "logits/chosen": -0.752734363079071,
      "logits/rejected": -0.63232421875,
      "logps/chosen": -0.6363281011581421,
      "logps/rejected": -1.031835913658142,
      "loss": 0.8009,
      "nll_loss": 0.6947265863418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06363525241613388,
      "rewards/margins": 0.039572905749082565,
      "rewards/rejected": -0.10322265326976776,
      "step": 22070
    },
    {
      "epoch": 0.8274156377058702,
      "grad_norm": 2.634522553917714,
      "learning_rate": 5.383819020581654e-07,
      "log_odds_chosen": 0.5653320550918579,
      "log_odds_ratio": -0.5708984136581421,
      "logits/chosen": -0.7642577886581421,
      "logits/rejected": -0.629101574420929,
      "logps/chosen": -0.6436523199081421,
      "logps/rejected": -1.0203125476837158,
      "loss": 0.8105,
      "nll_loss": 0.681640625,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06439208984375,
      "rewards/margins": 0.03764495998620987,
      "rewards/rejected": -0.10207519680261612,
      "step": 22080
    },
    {
      "epoch": 0.8277903730490341,
      "grad_norm": 2.365966180852383,
      "learning_rate": 5.382600272627028e-07,
      "log_odds_chosen": 0.4034667909145355,
      "log_odds_ratio": -0.651074230670929,
      "logits/chosen": -0.737500011920929,
      "logits/rejected": -0.6636718511581421,
      "logps/chosen": -0.704785168170929,
      "logps/rejected": -0.9638671875,
      "loss": 0.8262,
      "nll_loss": 0.774218738079071,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07049560546875,
      "rewards/margins": 0.02592010423541069,
      "rewards/rejected": -0.09638671576976776,
      "step": 22090
    },
    {
      "epoch": 0.828165108392198,
      "grad_norm": 2.6794223133468598,
      "learning_rate": 5.381382351970499e-07,
      "log_odds_chosen": 0.4501709043979645,
      "log_odds_ratio": -0.6058593988418579,
      "logits/chosen": -0.7574218511581421,
      "logits/rejected": -0.6988281011581421,
      "logps/chosen": -0.7310546636581421,
      "logps/rejected": -1.0310547351837158,
      "loss": 0.8146,
      "nll_loss": 0.790234386920929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07310791313648224,
      "rewards/margins": 0.0300445556640625,
      "rewards/rejected": -0.10310058295726776,
      "step": 22100
    },
    {
      "epoch": 0.8285398437353619,
      "grad_norm": 2.4818559177203157,
      "learning_rate": 5.380165257676528e-07,
      "log_odds_chosen": 0.50543212890625,
      "log_odds_ratio": -0.570117175579071,
      "logits/chosen": -0.786328136920929,
      "logits/rejected": -0.6698242425918579,
      "logps/chosen": -0.64794921875,
      "logps/rejected": -0.9535156488418579,
      "loss": 0.8211,
      "nll_loss": 0.7525390386581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06483153998851776,
      "rewards/margins": 0.03056487999856472,
      "rewards/rejected": -0.0953369140625,
      "step": 22110
    },
    {
      "epoch": 0.8289145790785258,
      "grad_norm": 2.535386153586092,
      "learning_rate": 5.378948988811054e-07,
      "log_odds_chosen": 0.608203113079071,
      "log_odds_ratio": -0.5469726324081421,
      "logits/chosen": -0.720898449420929,
      "logits/rejected": -0.621386706829071,
      "logps/chosen": -0.67626953125,
      "logps/rejected": -1.055078148841858,
      "loss": 0.8308,
      "nll_loss": 0.798632800579071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.067626953125,
      "rewards/margins": 0.03796996921300888,
      "rewards/rejected": -0.10561523586511612,
      "step": 22120
    },
    {
      "epoch": 0.8292893144216896,
      "grad_norm": 2.8153120217727032,
      "learning_rate": 5.377733544441496e-07,
      "log_odds_chosen": 0.4009033143520355,
      "log_odds_ratio": -0.6337890625,
      "logits/chosen": -0.7339843511581421,
      "logits/rejected": -0.6375976800918579,
      "logps/chosen": -0.663281261920929,
      "logps/rejected": -0.904296875,
      "loss": 0.7962,
      "nll_loss": 0.775585949420929,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06635741889476776,
      "rewards/margins": 0.02405548095703125,
      "rewards/rejected": -0.09040527045726776,
      "step": 22130
    },
    {
      "epoch": 0.8296640497648535,
      "grad_norm": 2.2218815747501917,
      "learning_rate": 5.376518923636746e-07,
      "log_odds_chosen": 0.517529308795929,
      "log_odds_ratio": -0.5777343511581421,
      "logits/chosen": -0.7886718511581421,
      "logits/rejected": -0.694531261920929,
      "logps/chosen": -0.684277355670929,
      "logps/rejected": -0.987500011920929,
      "loss": 0.8024,
      "nll_loss": 0.7171875238418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06839599460363388,
      "rewards/margins": 0.03036651574075222,
      "rewards/rejected": -0.09873046725988388,
      "step": 22140
    },
    {
      "epoch": 0.8300387851080174,
      "grad_norm": 2.292637036860787,
      "learning_rate": 5.375305125467166e-07,
      "log_odds_chosen": 0.60498046875,
      "log_odds_ratio": -0.5736328363418579,
      "logits/chosen": -0.7708984613418579,
      "logits/rejected": -0.7005859613418579,
      "logps/chosen": -0.595507800579071,
      "logps/rejected": -0.9642578363418579,
      "loss": 0.823,
      "nll_loss": 0.741992175579071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.05958252027630806,
      "rewards/margins": 0.0368804931640625,
      "rewards/rejected": -0.09642334282398224,
      "step": 22150
    },
    {
      "epoch": 0.8304135204511813,
      "grad_norm": 2.3875599444650457,
      "learning_rate": 5.37409214900459e-07,
      "log_odds_chosen": 0.5736938714981079,
      "log_odds_ratio": -0.554980456829071,
      "logits/chosen": -0.8056640625,
      "logits/rejected": -0.65087890625,
      "logps/chosen": -0.666015625,
      "logps/rejected": -1.0212891101837158,
      "loss": 0.8101,
      "nll_loss": 0.7373046875,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06661377102136612,
      "rewards/margins": 0.0356292724609375,
      "rewards/rejected": -0.10225830227136612,
      "step": 22160
    },
    {
      "epoch": 0.8307882557943452,
      "grad_norm": 2.768475604041983,
      "learning_rate": 5.372879993322315e-07,
      "log_odds_chosen": 0.583056628704071,
      "log_odds_ratio": -0.555957019329071,
      "logits/chosen": -0.7845703363418579,
      "logits/rejected": -0.6943359375,
      "logps/chosen": -0.662890613079071,
      "logps/rejected": -1.015039086341858,
      "loss": 0.8143,
      "nll_loss": 0.7503906488418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06617431342601776,
      "rewards/margins": 0.03525390475988388,
      "rewards/rejected": -0.10148926079273224,
      "step": 22170
    },
    {
      "epoch": 0.8311629911375091,
      "grad_norm": 2.3445430552154978,
      "learning_rate": 5.371668657495103e-07,
      "log_odds_chosen": 0.6095215082168579,
      "log_odds_ratio": -0.567089855670929,
      "logits/chosen": -0.7289062738418579,
      "logits/rejected": -0.629687488079071,
      "logps/chosen": -0.63232421875,
      "logps/rejected": -1.0244140625,
      "loss": 0.8015,
      "nll_loss": 0.7583984136581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.063232421875,
      "rewards/margins": 0.039275359362363815,
      "rewards/rejected": -0.10246582329273224,
      "step": 22180
    },
    {
      "epoch": 0.831537726480673,
      "grad_norm": 2.379968598892042,
      "learning_rate": 5.37045814059917e-07,
      "log_odds_chosen": 0.514233410358429,
      "log_odds_ratio": -0.601269543170929,
      "logits/chosen": -0.753613293170929,
      "logits/rejected": -0.675976574420929,
      "logps/chosen": -0.6578124761581421,
      "logps/rejected": -0.979296863079071,
      "loss": 0.8126,
      "nll_loss": 0.744140625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06577148288488388,
      "rewards/margins": 0.0320892333984375,
      "rewards/rejected": -0.09787597507238388,
      "step": 22190
    },
    {
      "epoch": 0.8319124618238369,
      "grad_norm": 2.6180698295937295,
      "learning_rate": 5.369248441712195e-07,
      "log_odds_chosen": 0.578198254108429,
      "log_odds_ratio": -0.5960937738418579,
      "logits/chosen": -0.7652343511581421,
      "logits/rejected": -0.6396484375,
      "logps/chosen": -0.682910144329071,
      "logps/rejected": -1.091796875,
      "loss": 0.8201,
      "nll_loss": 0.7572265863418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06829833984375,
      "rewards/margins": 0.040830232203006744,
      "rewards/rejected": -0.10919189453125,
      "step": 22200
    },
    {
      "epoch": 0.8322871971670008,
      "grad_norm": 2.498082655880627,
      "learning_rate": 5.368039559913306e-07,
      "log_odds_chosen": 0.7191406488418579,
      "log_odds_ratio": -0.532519519329071,
      "logits/chosen": -0.7813476324081421,
      "logits/rejected": -0.6659179925918579,
      "logps/chosen": -0.665722668170929,
      "logps/rejected": -1.1359374523162842,
      "loss": 0.8141,
      "nll_loss": 0.757031261920929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06656493991613388,
      "rewards/margins": 0.04700164869427681,
      "rewards/rejected": -0.11357422173023224,
      "step": 22210
    },
    {
      "epoch": 0.8326619325101647,
      "grad_norm": 2.237073216359698,
      "learning_rate": 5.366831494283084e-07,
      "log_odds_chosen": 0.571044921875,
      "log_odds_ratio": -0.5824218988418579,
      "logits/chosen": -0.728515625,
      "logits/rejected": -0.61376953125,
      "logps/chosen": -0.626757800579071,
      "logps/rejected": -1.003320336341858,
      "loss": 0.8053,
      "nll_loss": 0.797070324420929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06265868991613388,
      "rewards/margins": 0.03763275220990181,
      "rewards/rejected": -0.10029296576976776,
      "step": 22220
    },
    {
      "epoch": 0.8330366678533286,
      "grad_norm": 2.4613481393121117,
      "learning_rate": 5.365624243903558e-07,
      "log_odds_chosen": 0.4475341737270355,
      "log_odds_ratio": -0.60302734375,
      "logits/chosen": -0.762499988079071,
      "logits/rejected": -0.692187488079071,
      "logps/chosen": -0.664257824420929,
      "logps/rejected": -0.9351562261581421,
      "loss": 0.7963,
      "nll_loss": 0.7701171636581421,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06646728515625,
      "rewards/margins": 0.02705993689596653,
      "rewards/rejected": -0.09348144382238388,
      "step": 22230
    },
    {
      "epoch": 0.8334114031964924,
      "grad_norm": 2.258673252920256,
      "learning_rate": 5.3644178078582e-07,
      "log_odds_chosen": 0.5057617425918579,
      "log_odds_ratio": -0.6039062738418579,
      "logits/chosen": -0.765625,
      "logits/rejected": -0.6996093988418579,
      "logps/chosen": -0.670605480670929,
      "logps/rejected": -1.014062523841858,
      "loss": 0.8362,
      "nll_loss": 0.69287109375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06707763671875,
      "rewards/margins": 0.03428802639245987,
      "rewards/rejected": -0.10136719048023224,
      "step": 22240
    },
    {
      "epoch": 0.8337861385396563,
      "grad_norm": 2.86179406133567,
      "learning_rate": 5.363212185231927e-07,
      "log_odds_chosen": 0.39915770292282104,
      "log_odds_ratio": -0.6290038824081421,
      "logits/chosen": -0.7738281488418579,
      "logits/rejected": -0.7171875238418579,
      "logps/chosen": -0.698437511920929,
      "logps/rejected": -0.951171875,
      "loss": 0.8015,
      "nll_loss": 0.7896484136581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06982421875,
      "rewards/margins": 0.02530212327837944,
      "rewards/rejected": -0.09512939304113388,
      "step": 22250
    },
    {
      "epoch": 0.8341608738828202,
      "grad_norm": 2.6013200350109997,
      "learning_rate": 5.362007375111091e-07,
      "log_odds_chosen": 0.602294921875,
      "log_odds_ratio": -0.561816394329071,
      "logits/chosen": -0.791210949420929,
      "logits/rejected": -0.6851562261581421,
      "logps/chosen": -0.694140613079071,
      "logps/rejected": -1.0886719226837158,
      "loss": 0.8242,
      "nll_loss": 0.7777343988418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06939697265625,
      "rewards/margins": 0.03941955417394638,
      "rewards/rejected": -0.10878906399011612,
      "step": 22260
    },
    {
      "epoch": 0.8345356092259841,
      "grad_norm": 3.0049668414632515,
      "learning_rate": 5.360803376583483e-07,
      "log_odds_chosen": 0.5697387456893921,
      "log_odds_ratio": -0.5565429925918579,
      "logits/chosen": -0.7982422113418579,
      "logits/rejected": -0.6822265386581421,
      "logps/chosen": -0.646289050579071,
      "logps/rejected": -1.0089843273162842,
      "loss": 0.8093,
      "nll_loss": 0.733203113079071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06470946967601776,
      "rewards/margins": 0.036125946789979935,
      "rewards/rejected": -0.10080566257238388,
      "step": 22270
    },
    {
      "epoch": 0.834910344569148,
      "grad_norm": 2.26094737791139,
      "learning_rate": 5.359600188738324e-07,
      "log_odds_chosen": 0.558947741985321,
      "log_odds_ratio": -0.585644543170929,
      "logits/chosen": -0.74609375,
      "logits/rejected": -0.6209961175918579,
      "logps/chosen": -0.66552734375,
      "logps/rejected": -1.021875023841858,
      "loss": 0.8016,
      "nll_loss": 0.7144531011581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06657715141773224,
      "rewards/margins": 0.03557128831744194,
      "rewards/rejected": -0.1021728515625,
      "step": 22280
    },
    {
      "epoch": 0.8352850799123119,
      "grad_norm": 2.3380388874681843,
      "learning_rate": 5.35839781066627e-07,
      "log_odds_chosen": 0.526721179485321,
      "log_odds_ratio": -0.59521484375,
      "logits/chosen": -0.773242175579071,
      "logits/rejected": -0.645214855670929,
      "logps/chosen": -0.6708008050918579,
      "logps/rejected": -1.0031249523162842,
      "loss": 0.8268,
      "nll_loss": 0.773242175579071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06715087592601776,
      "rewards/margins": 0.03321533277630806,
      "rewards/rejected": -0.10029296576976776,
      "step": 22290
    },
    {
      "epoch": 0.8356598152554758,
      "grad_norm": 2.7381111022686606,
      "learning_rate": 5.357196241459401e-07,
      "log_odds_chosen": 0.44575196504592896,
      "log_odds_ratio": -0.602246105670929,
      "logits/chosen": -0.758007824420929,
      "logits/rejected": -0.6727539300918579,
      "logps/chosen": -0.6703125238418579,
      "logps/rejected": -0.941210925579071,
      "loss": 0.8158,
      "nll_loss": 0.7759765386581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06697998195886612,
      "rewards/margins": 0.0270843505859375,
      "rewards/rejected": -0.0941162109375,
      "step": 22300
    },
    {
      "epoch": 0.8360345505986397,
      "grad_norm": 2.3158292712246404,
      "learning_rate": 5.355995480211221e-07,
      "log_odds_chosen": 0.46369630098342896,
      "log_odds_ratio": -0.603320300579071,
      "logits/chosen": -0.746874988079071,
      "logits/rejected": -0.61279296875,
      "logps/chosen": -0.684863269329071,
      "logps/rejected": -0.982421875,
      "loss": 0.8054,
      "nll_loss": 0.771289050579071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06846924126148224,
      "rewards/margins": 0.02981872484087944,
      "rewards/rejected": -0.09827880561351776,
      "step": 22310
    },
    {
      "epoch": 0.8364092859418036,
      "grad_norm": 2.8841584214178635,
      "learning_rate": 5.354795526016659e-07,
      "log_odds_chosen": 0.528124988079071,
      "log_odds_ratio": -0.574999988079071,
      "logits/chosen": -0.7294921875,
      "logits/rejected": -0.652148425579071,
      "logps/chosen": -0.6626952886581421,
      "logps/rejected": -0.9755859375,
      "loss": 0.7884,
      "nll_loss": 0.7259765863418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06622314453125,
      "rewards/margins": 0.03134002536535263,
      "rewards/rejected": -0.09763183444738388,
      "step": 22320
    },
    {
      "epoch": 0.8367840212849675,
      "grad_norm": 2.4079921759634177,
      "learning_rate": 5.353596377972059e-07,
      "log_odds_chosen": 0.72021484375,
      "log_odds_ratio": -0.5162109136581421,
      "logits/chosen": -0.7279297113418579,
      "logits/rejected": -0.6146484613418579,
      "logps/chosen": -0.6109374761581421,
      "logps/rejected": -1.035546898841858,
      "loss": 0.7989,
      "nll_loss": 0.7431640625,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06114501878619194,
      "rewards/margins": 0.04242859035730362,
      "rewards/rejected": -0.10356445610523224,
      "step": 22330
    },
    {
      "epoch": 0.8371587566281313,
      "grad_norm": 2.900743896094826,
      "learning_rate": 5.352398035175184e-07,
      "log_odds_chosen": 0.37043458223342896,
      "log_odds_ratio": -0.6278320550918579,
      "logits/chosen": -0.7421875,
      "logits/rejected": -0.635546863079071,
      "logps/chosen": -0.6646484136581421,
      "logps/rejected": -0.89501953125,
      "loss": 0.8121,
      "nll_loss": 0.78125,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.0665283203125,
      "rewards/margins": 0.022997666150331497,
      "rewards/rejected": -0.0894775390625,
      "step": 22340
    },
    {
      "epoch": 0.8375334919712952,
      "grad_norm": 2.2952349223619124,
      "learning_rate": 5.351200496725209e-07,
      "log_odds_chosen": 0.4290527403354645,
      "log_odds_ratio": -0.6185547113418579,
      "logits/chosen": -0.7630859613418579,
      "logits/rejected": -0.681347668170929,
      "logps/chosen": -0.7232421636581421,
      "logps/rejected": -0.998242199420929,
      "loss": 0.8207,
      "nll_loss": 0.775390625,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07232666015625,
      "rewards/margins": 0.027483368292450905,
      "rewards/rejected": -0.09978027641773224,
      "step": 22350
    },
    {
      "epoch": 0.8379082273144591,
      "grad_norm": 2.3416075440466675,
      "learning_rate": 5.350003761722717e-07,
      "log_odds_chosen": 0.519116222858429,
      "log_odds_ratio": -0.57763671875,
      "logits/chosen": -0.734179675579071,
      "logits/rejected": -0.648632824420929,
      "logps/chosen": -0.6783202886581421,
      "logps/rejected": -1.0138671398162842,
      "loss": 0.8195,
      "nll_loss": 0.727734386920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06783447414636612,
      "rewards/margins": 0.03359375149011612,
      "rewards/rejected": -0.1014404296875,
      "step": 22360
    },
    {
      "epoch": 0.838282962657623,
      "grad_norm": 2.8499299628898105,
      "learning_rate": 5.348807829269702e-07,
      "log_odds_chosen": 0.4526000916957855,
      "log_odds_ratio": -0.6167968511581421,
      "logits/chosen": -0.763671875,
      "logits/rejected": -0.6871093511581421,
      "logps/chosen": -0.667187511920929,
      "logps/rejected": -0.926953136920929,
      "loss": 0.8268,
      "nll_loss": 0.7828124761581421,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06668701022863388,
      "rewards/margins": 0.02597656287252903,
      "rewards/rejected": -0.09273681789636612,
      "step": 22370
    },
    {
      "epoch": 0.8386576980007869,
      "grad_norm": 2.318539851486098,
      "learning_rate": 5.347612698469559e-07,
      "log_odds_chosen": 0.7465575933456421,
      "log_odds_ratio": -0.520068347454071,
      "logits/chosen": -0.7090820074081421,
      "logits/rejected": -0.591113269329071,
      "logps/chosen": -0.6668945550918579,
      "logps/rejected": -1.1501953601837158,
      "loss": 0.8185,
      "nll_loss": 0.7596679925918579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06669922173023224,
      "rewards/margins": 0.04830627515912056,
      "rewards/rejected": -0.11500243842601776,
      "step": 22380
    },
    {
      "epoch": 0.8390324333439508,
      "grad_norm": 2.634022356495211,
      "learning_rate": 5.346418368427089e-07,
      "log_odds_chosen": 0.570874035358429,
      "log_odds_ratio": -0.576367199420929,
      "logits/chosen": -0.7289062738418579,
      "logits/rejected": -0.6171875,
      "logps/chosen": -0.64111328125,
      "logps/rejected": -0.9798828363418579,
      "loss": 0.8103,
      "nll_loss": 0.728710949420929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06412353366613388,
      "rewards/margins": 0.033890534192323685,
      "rewards/rejected": -0.09792480617761612,
      "step": 22390
    },
    {
      "epoch": 0.8394071686871147,
      "grad_norm": 2.638772751936635,
      "learning_rate": 5.345224838248488e-07,
      "log_odds_chosen": 0.522167980670929,
      "log_odds_ratio": -0.582812488079071,
      "logits/chosen": -0.7730468511581421,
      "logits/rejected": -0.6341797113418579,
      "logps/chosen": -0.672070324420929,
      "logps/rejected": -1.0203125476837158,
      "loss": 0.8201,
      "nll_loss": 0.763867199420929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06717529147863388,
      "rewards/margins": 0.03478393703699112,
      "rewards/rejected": -0.10200195014476776,
      "step": 22400
    },
    {
      "epoch": 0.8397819040302786,
      "grad_norm": 2.606965245148365,
      "learning_rate": 5.344032107041349e-07,
      "log_odds_chosen": 0.5951172113418579,
      "log_odds_ratio": -0.574999988079071,
      "logits/chosen": -0.773632824420929,
      "logits/rejected": -0.6444336175918579,
      "logps/chosen": -0.6666015386581421,
      "logps/rejected": -1.037109375,
      "loss": 0.8074,
      "nll_loss": 0.727832019329071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.066650390625,
      "rewards/margins": 0.03699035570025444,
      "rewards/rejected": -0.10371093451976776,
      "step": 22410
    },
    {
      "epoch": 0.8401566393734425,
      "grad_norm": 2.2538498578731496,
      "learning_rate": 5.34284017391466e-07,
      "log_odds_chosen": 0.6292968988418579,
      "log_odds_ratio": -0.54443359375,
      "logits/chosen": -0.7197265625,
      "logits/rejected": -0.614453136920929,
      "logps/chosen": -0.631054699420929,
      "logps/rejected": -1.037109375,
      "loss": 0.8086,
      "nll_loss": 0.742968738079071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.0631103515625,
      "rewards/margins": 0.04051513597369194,
      "rewards/rejected": -0.10373535007238388,
      "step": 22420
    },
    {
      "epoch": 0.8405313747166064,
      "grad_norm": 2.219233211989636,
      "learning_rate": 5.341649037978797e-07,
      "log_odds_chosen": 0.5501464605331421,
      "log_odds_ratio": -0.5965820550918579,
      "logits/chosen": -0.772656261920929,
      "logits/rejected": -0.6410156488418579,
      "logps/chosen": -0.7085937261581421,
      "logps/rejected": -1.05078125,
      "loss": 0.8125,
      "nll_loss": 0.754101574420929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07080078125,
      "rewards/margins": 0.03420410305261612,
      "rewards/rejected": -0.10500488430261612,
      "step": 22430
    },
    {
      "epoch": 0.8409061100597703,
      "grad_norm": 2.6830272849207577,
      "learning_rate": 5.340458698345527e-07,
      "log_odds_chosen": 0.4120727479457855,
      "log_odds_ratio": -0.612500011920929,
      "logits/chosen": -0.7826172113418579,
      "logits/rejected": -0.715039074420929,
      "logps/chosen": -0.6841796636581421,
      "logps/rejected": -0.9296875,
      "loss": 0.8121,
      "nll_loss": 0.7818359136581421,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06844482570886612,
      "rewards/margins": 0.02451782301068306,
      "rewards/rejected": -0.09293212741613388,
      "step": 22440
    },
    {
      "epoch": 0.8412808454029341,
      "grad_norm": 3.1500246492260895,
      "learning_rate": 5.339269154127999e-07,
      "log_odds_chosen": 0.6156371831893921,
      "log_odds_ratio": -0.557421863079071,
      "logits/chosen": -0.6968749761581421,
      "logits/rejected": -0.6470702886581421,
      "logps/chosen": -0.679492175579071,
      "logps/rejected": -1.046875,
      "loss": 0.7954,
      "nll_loss": 0.78125,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06791992485523224,
      "rewards/margins": 0.036762237548828125,
      "rewards/rejected": -0.10463867336511612,
      "step": 22450
    },
    {
      "epoch": 0.841655580746098,
      "grad_norm": 2.4528626198210763,
      "learning_rate": 5.338080404440744e-07,
      "log_odds_chosen": 0.4711547791957855,
      "log_odds_ratio": -0.622363269329071,
      "logits/chosen": -0.748828113079071,
      "logits/rejected": -0.63525390625,
      "logps/chosen": -0.694628894329071,
      "logps/rejected": -0.997265636920929,
      "loss": 0.8003,
      "nll_loss": 0.7232421636581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06943359225988388,
      "rewards/margins": 0.03022003173828125,
      "rewards/rejected": -0.09970702975988388,
      "step": 22460
    },
    {
      "epoch": 0.8420303160892619,
      "grad_norm": 2.515781514188732,
      "learning_rate": 5.336892448399675e-07,
      "log_odds_chosen": 0.4124999940395355,
      "log_odds_ratio": -0.6253906488418579,
      "logits/chosen": -0.7386718988418579,
      "logits/rejected": -0.6390625238418579,
      "logps/chosen": -0.701171875,
      "logps/rejected": -0.9615234136581421,
      "loss": 0.8112,
      "nll_loss": 0.7767578363418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07016601413488388,
      "rewards/margins": 0.02588806115090847,
      "rewards/rejected": -0.09613037109375,
      "step": 22470
    },
    {
      "epoch": 0.8424050514324258,
      "grad_norm": 2.4577392702245673,
      "learning_rate": 5.335705285122082e-07,
      "log_odds_chosen": 0.6065429449081421,
      "log_odds_ratio": -0.5555664300918579,
      "logits/chosen": -0.8001953363418579,
      "logits/rejected": -0.65576171875,
      "logps/chosen": -0.664843738079071,
      "logps/rejected": -1.020898461341858,
      "loss": 0.7971,
      "nll_loss": 0.716015636920929,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06646728515625,
      "rewards/margins": 0.0356903076171875,
      "rewards/rejected": -0.1021728515625,
      "step": 22480
    },
    {
      "epoch": 0.8427797867755897,
      "grad_norm": 2.3647048212497435,
      "learning_rate": 5.334518913726623e-07,
      "log_odds_chosen": 0.5880126953125,
      "log_odds_ratio": -0.58837890625,
      "logits/chosen": -0.7210937738418579,
      "logits/rejected": -0.626171886920929,
      "logps/chosen": -0.709667980670929,
      "logps/rejected": -1.082421898841858,
      "loss": 0.7988,
      "nll_loss": 0.740039050579071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07095947116613388,
      "rewards/margins": 0.03738250583410263,
      "rewards/rejected": -0.10834960639476776,
      "step": 22490
    },
    {
      "epoch": 0.8431545221187536,
      "grad_norm": 2.3764911120484356,
      "learning_rate": 5.333333333333333e-07,
      "log_odds_chosen": 0.5751587152481079,
      "log_odds_ratio": -0.5782226324081421,
      "logits/chosen": -0.7496093511581421,
      "logits/rejected": -0.6328125,
      "logps/chosen": -0.6221679449081421,
      "logps/rejected": -0.970898449420929,
      "loss": 0.8075,
      "nll_loss": 0.7339843511581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06223144382238388,
      "rewards/margins": 0.03484802320599556,
      "rewards/rejected": -0.0970458984375,
      "step": 22500
    },
    {
      "epoch": 0.8435292574619175,
      "grad_norm": 2.761012674840254,
      "learning_rate": 5.332148543063615e-07,
      "log_odds_chosen": 0.6164795160293579,
      "log_odds_ratio": -0.56396484375,
      "logits/chosen": -0.711132824420929,
      "logits/rejected": -0.604785144329071,
      "logps/chosen": -0.6211913824081421,
      "logps/rejected": -0.992968738079071,
      "loss": 0.8254,
      "nll_loss": 0.682812511920929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06214599683880806,
      "rewards/margins": 0.03717651218175888,
      "rewards/rejected": -0.09931640326976776,
      "step": 22510
    },
    {
      "epoch": 0.8439039928050814,
      "grad_norm": 2.474373528877567,
      "learning_rate": 5.330964542040233e-07,
      "log_odds_chosen": 0.6239258050918579,
      "log_odds_ratio": -0.5582031011581421,
      "logits/chosen": -0.787890613079071,
      "logits/rejected": -0.6744140386581421,
      "logps/chosen": -0.6444336175918579,
      "logps/rejected": -1.0285155773162842,
      "loss": 0.8085,
      "nll_loss": 0.6957031488418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06444092094898224,
      "rewards/margins": 0.03847198560833931,
      "rewards/rejected": -0.10288085788488388,
      "step": 22520
    },
    {
      "epoch": 0.8442787281482453,
      "grad_norm": 2.5075441897300506,
      "learning_rate": 5.32978132938732e-07,
      "log_odds_chosen": 0.6935180425643921,
      "log_odds_ratio": -0.5570312738418579,
      "logits/chosen": -0.7857421636581421,
      "logits/rejected": -0.6380859613418579,
      "logps/chosen": -0.6859375238418579,
      "logps/rejected": -1.1416015625,
      "loss": 0.7909,
      "nll_loss": 0.735546886920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.068603515625,
      "rewards/margins": 0.04551086574792862,
      "rewards/rejected": -0.1141357421875,
      "step": 22530
    },
    {
      "epoch": 0.8446534634914092,
      "grad_norm": 2.6951205626713275,
      "learning_rate": 5.328598904230365e-07,
      "log_odds_chosen": 0.3251953125,
      "log_odds_ratio": -0.63916015625,
      "logits/chosen": -0.751953125,
      "logits/rejected": -0.67724609375,
      "logps/chosen": -0.66845703125,
      "logps/rejected": -0.8441406488418579,
      "loss": 0.7937,
      "nll_loss": 0.726757824420929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06684570014476776,
      "rewards/margins": 0.017556000500917435,
      "rewards/rejected": -0.08433838188648224,
      "step": 22540
    },
    {
      "epoch": 0.845028198834573,
      "grad_norm": 2.3336398142007475,
      "learning_rate": 5.327417265696215e-07,
      "log_odds_chosen": 0.42805176973342896,
      "log_odds_ratio": -0.603515625,
      "logits/chosen": -0.8003906011581421,
      "logits/rejected": -0.673828125,
      "logps/chosen": -0.721484363079071,
      "logps/rejected": -0.9769531488418579,
      "loss": 0.808,
      "nll_loss": 0.751269519329071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07213135063648224,
      "rewards/margins": 0.025585174560546875,
      "rewards/rejected": -0.09770508110523224,
      "step": 22550
    },
    {
      "epoch": 0.8454029341777369,
      "grad_norm": 2.1392287703257673,
      "learning_rate": 5.326236412913074e-07,
      "log_odds_chosen": 0.5119873285293579,
      "log_odds_ratio": -0.60595703125,
      "logits/chosen": -0.73388671875,
      "logits/rejected": -0.6126953363418579,
      "logps/chosen": -0.6845703125,
      "logps/rejected": -1.0085937976837158,
      "loss": 0.8121,
      "nll_loss": 0.742968738079071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06846924126148224,
      "rewards/margins": 0.03229675441980362,
      "rewards/rejected": -0.10085449367761612,
      "step": 22560
    },
    {
      "epoch": 0.8457776695209008,
      "grad_norm": 2.955492033059416,
      "learning_rate": 5.325056345010497e-07,
      "log_odds_chosen": 0.584912121295929,
      "log_odds_ratio": -0.5887695550918579,
      "logits/chosen": -0.7421875,
      "logits/rejected": -0.669238269329071,
      "logps/chosen": -0.6792968511581421,
      "logps/rejected": -1.0232422351837158,
      "loss": 0.8141,
      "nll_loss": 0.744140625,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06793212890625,
      "rewards/margins": 0.034398652613162994,
      "rewards/rejected": -0.10225830227136612,
      "step": 22570
    },
    {
      "epoch": 0.8461524048640647,
      "grad_norm": 3.4278092951579695,
      "learning_rate": 5.323877061119386e-07,
      "log_odds_chosen": 0.630615234375,
      "log_odds_ratio": -0.5443359613418579,
      "logits/chosen": -0.7623046636581421,
      "logits/rejected": -0.6631835699081421,
      "logps/chosen": -0.6924804449081421,
      "logps/rejected": -1.0984375476837158,
      "loss": 0.7968,
      "nll_loss": 0.751171886920929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06929931789636612,
      "rewards/margins": 0.04049835354089737,
      "rewards/rejected": -0.10983886569738388,
      "step": 22580
    },
    {
      "epoch": 0.8465271402072286,
      "grad_norm": 2.8216827842958208,
      "learning_rate": 5.322698560371995e-07,
      "log_odds_chosen": 0.6316894292831421,
      "log_odds_ratio": -0.550976574420929,
      "logits/chosen": -0.7943359613418579,
      "logits/rejected": -0.670117199420929,
      "logps/chosen": -0.644335925579071,
      "logps/rejected": -1.0535156726837158,
      "loss": 0.8169,
      "nll_loss": 0.748828113079071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06444092094898224,
      "rewards/margins": 0.04097862169146538,
      "rewards/rejected": -0.10541991889476776,
      "step": 22590
    },
    {
      "epoch": 0.8469018755503925,
      "grad_norm": 2.321787395328883,
      "learning_rate": 5.321520841901914e-07,
      "log_odds_chosen": 0.46259766817092896,
      "log_odds_ratio": -0.5814453363418579,
      "logits/chosen": -0.773632824420929,
      "logits/rejected": -0.699023425579071,
      "logps/chosen": -0.646289050579071,
      "logps/rejected": -0.9263671636581421,
      "loss": 0.7831,
      "nll_loss": 0.723828136920929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06456299126148224,
      "rewards/margins": 0.02805023267865181,
      "rewards/rejected": -0.09262695163488388,
      "step": 22600
    },
    {
      "epoch": 0.8472766108935564,
      "grad_norm": 2.7553295438278433,
      "learning_rate": 5.320343904844084e-07,
      "log_odds_chosen": 0.5588623285293579,
      "log_odds_ratio": -0.566210925579071,
      "logits/chosen": -0.78125,
      "logits/rejected": -0.665234386920929,
      "logps/chosen": -0.630664050579071,
      "logps/rejected": -0.992382824420929,
      "loss": 0.7863,
      "nll_loss": 0.7236328125,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06307373195886612,
      "rewards/margins": 0.036190032958984375,
      "rewards/rejected": -0.09931640326976776,
      "step": 22610
    },
    {
      "epoch": 0.8476513462367203,
      "grad_norm": 2.237098078806197,
      "learning_rate": 5.319167748334776e-07,
      "log_odds_chosen": 0.53643798828125,
      "log_odds_ratio": -0.5907226800918579,
      "logits/chosen": -0.739062488079071,
      "logits/rejected": -0.652050793170929,
      "logps/chosen": -0.686230480670929,
      "logps/rejected": -1.019921898841858,
      "loss": 0.7967,
      "nll_loss": 0.7298828363418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06864013522863388,
      "rewards/margins": 0.03331298753619194,
      "rewards/rejected": -0.1019287109375,
      "step": 22620
    },
    {
      "epoch": 0.8480260815798842,
      "grad_norm": 2.626410561466052,
      "learning_rate": 5.317992371511601e-07,
      "log_odds_chosen": 0.4851318299770355,
      "log_odds_ratio": -0.6104980707168579,
      "logits/chosen": -0.788281261920929,
      "logits/rejected": -0.713574230670929,
      "logps/chosen": -0.6700195074081421,
      "logps/rejected": -0.97265625,
      "loss": 0.8074,
      "nll_loss": 0.736328125,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06697998195886612,
      "rewards/margins": 0.03027801588177681,
      "rewards/rejected": -0.09726562350988388,
      "step": 22630
    },
    {
      "epoch": 0.8484008169230481,
      "grad_norm": 2.9615391477498076,
      "learning_rate": 5.316817773513505e-07,
      "log_odds_chosen": 0.5543578863143921,
      "log_odds_ratio": -0.5748046636581421,
      "logits/chosen": -0.7269531488418579,
      "logits/rejected": -0.65234375,
      "logps/chosen": -0.6498047113418579,
      "logps/rejected": -0.9937499761581421,
      "loss": 0.8057,
      "nll_loss": 0.7264648675918579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06499023735523224,
      "rewards/margins": 0.034389495849609375,
      "rewards/rejected": -0.099365234375,
      "step": 22640
    },
    {
      "epoch": 0.848775552266212,
      "grad_norm": 2.6733695205567862,
      "learning_rate": 5.315643953480763e-07,
      "log_odds_chosen": 0.45122069120407104,
      "log_odds_ratio": -0.5948241949081421,
      "logits/chosen": -0.753125011920929,
      "logits/rejected": -0.6341797113418579,
      "logps/chosen": -0.63818359375,
      "logps/rejected": -0.9078124761581421,
      "loss": 0.8194,
      "nll_loss": 0.7353515625,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06380615383386612,
      "rewards/margins": 0.026973724365234375,
      "rewards/rejected": -0.09079589694738388,
      "step": 22650
    },
    {
      "epoch": 0.8491502876093758,
      "grad_norm": 2.2259488937261067,
      "learning_rate": 5.314470910554976e-07,
      "log_odds_chosen": 0.5295044183731079,
      "log_odds_ratio": -0.595703125,
      "logits/chosen": -0.7769531011581421,
      "logits/rejected": -0.6767578125,
      "logps/chosen": -0.6953125,
      "logps/rejected": -1.047460913658142,
      "loss": 0.8164,
      "nll_loss": 0.7554687261581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06951904296875,
      "rewards/margins": 0.03530578687787056,
      "rewards/rejected": -0.10483398288488388,
      "step": 22660
    },
    {
      "epoch": 0.8495250229525397,
      "grad_norm": 3.236067351153158,
      "learning_rate": 5.313298643879074e-07,
      "log_odds_chosen": 0.535290539264679,
      "log_odds_ratio": -0.5782226324081421,
      "logits/chosen": -0.7734375,
      "logits/rejected": -0.6495116949081421,
      "logps/chosen": -0.690136730670929,
      "logps/rejected": -1.03125,
      "loss": 0.8099,
      "nll_loss": 0.7261718511581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06906738132238388,
      "rewards/margins": 0.03404846042394638,
      "rewards/rejected": -0.10310058295726776,
      "step": 22670
    },
    {
      "epoch": 0.8498997582957036,
      "grad_norm": 2.311305234869397,
      "learning_rate": 5.312127152597304e-07,
      "log_odds_chosen": 0.525646984577179,
      "log_odds_ratio": -0.5977538824081421,
      "logits/chosen": -0.781054675579071,
      "logits/rejected": -0.6812499761581421,
      "logps/chosen": -0.656445324420929,
      "logps/rejected": -0.962695300579071,
      "loss": 0.8004,
      "nll_loss": 0.7398437261581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06564941257238388,
      "rewards/margins": 0.03063659742474556,
      "rewards/rejected": -0.09625244140625,
      "step": 22680
    },
    {
      "epoch": 0.8502744936388675,
      "grad_norm": 3.028057729857873,
      "learning_rate": 5.310956435855243e-07,
      "log_odds_chosen": 0.47137451171875,
      "log_odds_ratio": -0.603515625,
      "logits/chosen": -0.7193359136581421,
      "logits/rejected": -0.6587890386581421,
      "logps/chosen": -0.7025390863418579,
      "logps/rejected": -0.9916015863418579,
      "loss": 0.8109,
      "nll_loss": 0.7455078363418579,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07025146484375,
      "rewards/margins": 0.028852080926299095,
      "rewards/rejected": -0.09916992485523224,
      "step": 22690
    },
    {
      "epoch": 0.8506492289820314,
      "grad_norm": 2.4211124656348635,
      "learning_rate": 5.309786492799776e-07,
      "log_odds_chosen": 0.39372557401657104,
      "log_odds_ratio": -0.62158203125,
      "logits/chosen": -0.7548828125,
      "logits/rejected": -0.61328125,
      "logps/chosen": -0.6815429925918579,
      "logps/rejected": -0.940625011920929,
      "loss": 0.824,
      "nll_loss": 0.749804675579071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.068115234375,
      "rewards/margins": 0.02599945105612278,
      "rewards/rejected": -0.09418945014476776,
      "step": 22700
    },
    {
      "epoch": 0.8510239643251953,
      "grad_norm": 2.3680047968099265,
      "learning_rate": 5.308617322579108e-07,
      "log_odds_chosen": 0.549426257610321,
      "log_odds_ratio": -0.572949230670929,
      "logits/chosen": -0.776171863079071,
      "logits/rejected": -0.657421886920929,
      "logps/chosen": -0.6592773199081421,
      "logps/rejected": -1.015625,
      "loss": 0.7963,
      "nll_loss": 0.732226550579071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06591796875,
      "rewards/margins": 0.03572235256433487,
      "rewards/rejected": -0.10163573920726776,
      "step": 22710
    },
    {
      "epoch": 0.8513986996683592,
      "grad_norm": 2.3627282096990148,
      "learning_rate": 5.307448924342752e-07,
      "log_odds_chosen": 0.6433960199356079,
      "log_odds_ratio": -0.543261706829071,
      "logits/chosen": -0.7777343988418579,
      "logits/rejected": -0.623828113079071,
      "logps/chosen": -0.632617175579071,
      "logps/rejected": -1.039453148841858,
      "loss": 0.7973,
      "nll_loss": 0.6866210699081421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06329345703125,
      "rewards/margins": 0.040625762194395065,
      "rewards/rejected": -0.10383300483226776,
      "step": 22720
    },
    {
      "epoch": 0.8517734350115231,
      "grad_norm": 2.885981949779278,
      "learning_rate": 5.306281297241538e-07,
      "log_odds_chosen": 0.535021960735321,
      "log_odds_ratio": -0.590039074420929,
      "logits/chosen": -0.8042968511581421,
      "logits/rejected": -0.6903320550918579,
      "logps/chosen": -0.69287109375,
      "logps/rejected": -1.029687523841858,
      "loss": 0.8204,
      "nll_loss": 0.7474609613418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06934814155101776,
      "rewards/margins": 0.03363342210650444,
      "rewards/rejected": -0.10301513969898224,
      "step": 22730
    },
    {
      "epoch": 0.852148170354687,
      "grad_norm": 2.3261237206445577,
      "learning_rate": 5.305114440427598e-07,
      "log_odds_chosen": 0.525891125202179,
      "log_odds_ratio": -0.58642578125,
      "logits/chosen": -0.818164050579071,
      "logits/rejected": -0.6957031488418579,
      "logps/chosen": -0.6556640863418579,
      "logps/rejected": -0.974609375,
      "loss": 0.8164,
      "nll_loss": 0.70654296875,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06552734225988388,
      "rewards/margins": 0.03191985934972763,
      "rewards/rejected": -0.09758301079273224,
      "step": 22740
    },
    {
      "epoch": 0.8525229056978509,
      "grad_norm": 2.838306656678465,
      "learning_rate": 5.303948353054367e-07,
      "log_odds_chosen": 0.5813537836074829,
      "log_odds_ratio": -0.5782226324081421,
      "logits/chosen": -0.7001953125,
      "logits/rejected": -0.6089843511581421,
      "logps/chosen": -0.6307617425918579,
      "logps/rejected": -0.975390613079071,
      "loss": 0.8059,
      "nll_loss": 0.7137695550918579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06309814751148224,
      "rewards/margins": 0.034471891820430756,
      "rewards/rejected": -0.0975341796875,
      "step": 22750
    },
    {
      "epoch": 0.8528976410410147,
      "grad_norm": 2.8053068122984675,
      "learning_rate": 5.302783034276587e-07,
      "log_odds_chosen": 0.49104005098342896,
      "log_odds_ratio": -0.6153320074081421,
      "logits/chosen": -0.7562500238418579,
      "logits/rejected": -0.6244140863418579,
      "logps/chosen": -0.700390636920929,
      "logps/rejected": -1.000390648841858,
      "loss": 0.8295,
      "nll_loss": 0.761035144329071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06999512016773224,
      "rewards/margins": 0.030042266473174095,
      "rewards/rejected": -0.0999755859375,
      "step": 22760
    },
    {
      "epoch": 0.8532723763841786,
      "grad_norm": 2.813697691281621,
      "learning_rate": 5.301618483250294e-07,
      "log_odds_chosen": 0.6813598871231079,
      "log_odds_ratio": -0.548046886920929,
      "logits/chosen": -0.783398449420929,
      "logits/rejected": -0.6324218511581421,
      "logps/chosen": -0.659863293170929,
      "logps/rejected": -1.0867187976837158,
      "loss": 0.8046,
      "nll_loss": 0.7855468988418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06600341945886612,
      "rewards/margins": 0.04266662523150444,
      "rewards/rejected": -0.108642578125,
      "step": 22770
    },
    {
      "epoch": 0.8536471117273425,
      "grad_norm": 2.7963433569792455,
      "learning_rate": 5.300454699132826e-07,
      "log_odds_chosen": 0.4897216856479645,
      "log_odds_ratio": -0.5953124761581421,
      "logits/chosen": -0.724609375,
      "logits/rejected": -0.6314452886581421,
      "logps/chosen": -0.6626952886581421,
      "logps/rejected": -0.950976550579071,
      "loss": 0.8025,
      "nll_loss": 0.7494140863418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06630859524011612,
      "rewards/margins": 0.02884826622903347,
      "rewards/rejected": -0.09506835788488388,
      "step": 22780
    },
    {
      "epoch": 0.8540218470705064,
      "grad_norm": 3.0215007441840074,
      "learning_rate": 5.299291681082812e-07,
      "log_odds_chosen": 0.43939208984375,
      "log_odds_ratio": -0.603515625,
      "logits/chosen": -0.748242199420929,
      "logits/rejected": -0.62939453125,
      "logps/chosen": -0.6656249761581421,
      "logps/rejected": -0.9623047113418579,
      "loss": 0.8111,
      "nll_loss": 0.7120116949081421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06656493991613388,
      "rewards/margins": 0.02970581129193306,
      "rewards/rejected": -0.09626464545726776,
      "step": 22790
    },
    {
      "epoch": 0.8543965824136703,
      "grad_norm": 2.8124048805115165,
      "learning_rate": 5.298129428260176e-07,
      "log_odds_chosen": 0.45665282011032104,
      "log_odds_ratio": -0.5926758050918579,
      "logits/chosen": -0.7554687261581421,
      "logits/rejected": -0.637988269329071,
      "logps/chosen": -0.689257800579071,
      "logps/rejected": -0.989453136920929,
      "loss": 0.8101,
      "nll_loss": 0.7669922113418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06892089545726776,
      "rewards/margins": 0.03002014197409153,
      "rewards/rejected": -0.09897460788488388,
      "step": 22800
    },
    {
      "epoch": 0.8547713177568342,
      "grad_norm": 3.0271222232581856,
      "learning_rate": 5.296967939826123e-07,
      "log_odds_chosen": 0.624310314655304,
      "log_odds_ratio": -0.528124988079071,
      "logits/chosen": -0.7171875238418579,
      "logits/rejected": -0.61376953125,
      "logps/chosen": -0.6673828363418579,
      "logps/rejected": -1.055078148841858,
      "loss": 0.7783,
      "nll_loss": 0.753125011920929,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06671142578125,
      "rewards/margins": 0.03877563402056694,
      "rewards/rejected": -0.10551758110523224,
      "step": 22810
    },
    {
      "epoch": 0.8551460530999981,
      "grad_norm": 2.9862391975346956,
      "learning_rate": 5.295807214943156e-07,
      "log_odds_chosen": 0.4765869081020355,
      "log_odds_ratio": -0.59375,
      "logits/chosen": -0.7035156488418579,
      "logits/rejected": -0.626660168170929,
      "logps/chosen": -0.5912109613418579,
      "logps/rejected": -0.8720703125,
      "loss": 0.8035,
      "nll_loss": 0.761914074420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.05913085862994194,
      "rewards/margins": 0.02807312086224556,
      "rewards/rejected": -0.08720703423023224,
      "step": 22820
    },
    {
      "epoch": 0.855520788443162,
      "grad_norm": 2.4604332720126267,
      "learning_rate": 5.294647252775055e-07,
      "log_odds_chosen": 0.599353015422821,
      "log_odds_ratio": -0.583691418170929,
      "logits/chosen": -0.78759765625,
      "logits/rejected": -0.68896484375,
      "logps/chosen": -0.6865234375,
      "logps/rejected": -1.088281273841858,
      "loss": 0.7963,
      "nll_loss": 0.7427734136581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06865234673023224,
      "rewards/margins": 0.04015808179974556,
      "rewards/rejected": -0.10883788764476776,
      "step": 22830
    },
    {
      "epoch": 0.8558955237863259,
      "grad_norm": 2.4883492513348355,
      "learning_rate": 5.293488052486882e-07,
      "log_odds_chosen": 0.6236327886581421,
      "log_odds_ratio": -0.559765636920929,
      "logits/chosen": -0.7662109136581421,
      "logits/rejected": -0.6563476324081421,
      "logps/chosen": -0.6654297113418579,
      "logps/rejected": -1.064062476158142,
      "loss": 0.8017,
      "nll_loss": 0.7310546636581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06654052436351776,
      "rewards/margins": 0.03985290601849556,
      "rewards/rejected": -0.10649414360523224,
      "step": 22840
    },
    {
      "epoch": 0.8562702591294898,
      "grad_norm": 2.5599118822625906,
      "learning_rate": 5.292329613244979e-07,
      "log_odds_chosen": 0.48054200410842896,
      "log_odds_ratio": -0.600781261920929,
      "logits/chosen": -0.7816406488418579,
      "logits/rejected": -0.6830078363418579,
      "logps/chosen": -0.68408203125,
      "logps/rejected": -0.98046875,
      "loss": 0.8171,
      "nll_loss": 0.7685546875,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06840820610523224,
      "rewards/margins": 0.029671479016542435,
      "rewards/rejected": -0.09807129204273224,
      "step": 22850
    },
    {
      "epoch": 0.8566449944726537,
      "grad_norm": 2.628712640830997,
      "learning_rate": 5.291171934216967e-07,
      "log_odds_chosen": 0.584716796875,
      "log_odds_ratio": -0.5679687261581421,
      "logits/chosen": -0.7061523199081421,
      "logits/rejected": -0.605761706829071,
      "logps/chosen": -0.684277355670929,
      "logps/rejected": -1.0363280773162842,
      "loss": 0.8096,
      "nll_loss": 0.7476562261581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06838379055261612,
      "rewards/margins": 0.03518066555261612,
      "rewards/rejected": -0.10354004055261612,
      "step": 22860
    },
    {
      "epoch": 0.8570197298158175,
      "grad_norm": 2.729186789453419,
      "learning_rate": 5.290015014571736e-07,
      "log_odds_chosen": 0.60400390625,
      "log_odds_ratio": -0.566210925579071,
      "logits/chosen": -0.7923828363418579,
      "logits/rejected": -0.6416991949081421,
      "logps/chosen": -0.642578125,
      "logps/rejected": -1.0203125476837158,
      "loss": 0.8192,
      "nll_loss": 0.722851574420929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06427001953125,
      "rewards/margins": 0.0378265380859375,
      "rewards/rejected": -0.10213623195886612,
      "step": 22870
    },
    {
      "epoch": 0.8573944651589814,
      "grad_norm": 2.4687278467221794,
      "learning_rate": 5.28885885347945e-07,
      "log_odds_chosen": 0.537670910358429,
      "log_odds_ratio": -0.5576171875,
      "logits/chosen": -0.771289050579071,
      "logits/rejected": -0.6507812738418579,
      "logps/chosen": -0.68798828125,
      "logps/rejected": -1.0070312023162842,
      "loss": 0.7959,
      "nll_loss": 0.68408203125,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0687255859375,
      "rewards/margins": 0.031998444348573685,
      "rewards/rejected": -0.10078124701976776,
      "step": 22880
    },
    {
      "epoch": 0.8577692005021453,
      "grad_norm": 2.4677311961289914,
      "learning_rate": 5.287703450111545e-07,
      "log_odds_chosen": 0.34440916776657104,
      "log_odds_ratio": -0.6552734375,
      "logits/chosen": -0.744140625,
      "logits/rejected": -0.633105456829071,
      "logps/chosen": -0.6714843511581421,
      "logps/rejected": -0.9140625,
      "loss": 0.8125,
      "nll_loss": 0.7162109613418579,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06716308742761612,
      "rewards/margins": 0.024163056164979935,
      "rewards/rejected": -0.09134521335363388,
      "step": 22890
    },
    {
      "epoch": 0.8581439358453092,
      "grad_norm": 2.5767226570716306,
      "learning_rate": 5.286548803640718e-07,
      "log_odds_chosen": 0.65234375,
      "log_odds_ratio": -0.5223633050918579,
      "logits/chosen": -0.7578125,
      "logits/rejected": -0.65087890625,
      "logps/chosen": -0.662109375,
      "logps/rejected": -1.079687476158142,
      "loss": 0.8034,
      "nll_loss": 0.779003918170929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06617431342601776,
      "rewards/margins": 0.04180298000574112,
      "rewards/rejected": -0.10795898735523224,
      "step": 22900
    },
    {
      "epoch": 0.8585186711884731,
      "grad_norm": 2.6592126232120195,
      "learning_rate": 5.285394913240933e-07,
      "log_odds_chosen": 0.46650391817092896,
      "log_odds_ratio": -0.6197265386581421,
      "logits/chosen": -0.713671863079071,
      "logits/rejected": -0.6136718988418579,
      "logps/chosen": -0.696582019329071,
      "logps/rejected": -1.0183594226837158,
      "loss": 0.8318,
      "nll_loss": 0.777148425579071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06962890923023224,
      "rewards/margins": 0.032245635986328125,
      "rewards/rejected": -0.101806640625,
      "step": 22910
    },
    {
      "epoch": 0.858893406531637,
      "grad_norm": 2.5759163762897668,
      "learning_rate": 5.284241778087417e-07,
      "log_odds_chosen": 0.4904541075229645,
      "log_odds_ratio": -0.609179675579071,
      "logits/chosen": -0.8033202886581421,
      "logits/rejected": -0.713574230670929,
      "logps/chosen": -0.640625,
      "logps/rejected": -0.938671886920929,
      "loss": 0.8012,
      "nll_loss": 0.7251952886581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06401367485523224,
      "rewards/margins": 0.029824066907167435,
      "rewards/rejected": -0.0938720703125,
      "step": 22920
    },
    {
      "epoch": 0.8592681418748009,
      "grad_norm": 2.6194760026733066,
      "learning_rate": 5.283089397356652e-07,
      "log_odds_chosen": 0.5016723871231079,
      "log_odds_ratio": -0.5819336175918579,
      "logits/chosen": -0.783984363079071,
      "logits/rejected": -0.6566406488418579,
      "logps/chosen": -0.671191394329071,
      "logps/rejected": -0.9791015386581421,
      "loss": 0.8101,
      "nll_loss": 0.7464843988418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06705322116613388,
      "rewards/margins": 0.03083343431353569,
      "rewards/rejected": -0.0980224609375,
      "step": 22930
    },
    {
      "epoch": 0.8596428772179648,
      "grad_norm": 2.524291121354254,
      "learning_rate": 5.28193777022638e-07,
      "log_odds_chosen": 0.4693359434604645,
      "log_odds_ratio": -0.6143554449081421,
      "logits/chosen": -0.6864258050918579,
      "logits/rejected": -0.6390625238418579,
      "logps/chosen": -0.652539074420929,
      "logps/rejected": -0.977734386920929,
      "loss": 0.8128,
      "nll_loss": 0.750781238079071,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06524658203125,
      "rewards/margins": 0.03257141262292862,
      "rewards/rejected": -0.09782715141773224,
      "step": 22940
    },
    {
      "epoch": 0.8600176125611287,
      "grad_norm": 2.145553179805462,
      "learning_rate": 5.280786895875596e-07,
      "log_odds_chosen": 0.507275402545929,
      "log_odds_ratio": -0.589062511920929,
      "logits/chosen": -0.772656261920929,
      "logits/rejected": -0.640820324420929,
      "logps/chosen": -0.66796875,
      "logps/rejected": -0.965039074420929,
      "loss": 0.801,
      "nll_loss": 0.70068359375,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06678466498851776,
      "rewards/margins": 0.02972259558737278,
      "rewards/rejected": -0.09650878608226776,
      "step": 22950
    },
    {
      "epoch": 0.8603923479042926,
      "grad_norm": 2.889798645992731,
      "learning_rate": 5.279636773484546e-07,
      "log_odds_chosen": 0.7303832769393921,
      "log_odds_ratio": -0.5245116949081421,
      "logits/chosen": -0.7763671875,
      "logits/rejected": -0.6949218511581421,
      "logps/chosen": -0.6509765386581421,
      "logps/rejected": -1.1240234375,
      "loss": 0.7831,
      "nll_loss": 0.721875011920929,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06517334282398224,
      "rewards/margins": 0.04735565185546875,
      "rewards/rejected": -0.1124267578125,
      "step": 22960
    },
    {
      "epoch": 0.8607670832474564,
      "grad_norm": 2.4470894063113944,
      "learning_rate": 5.278487402234727e-07,
      "log_odds_chosen": 0.5441039800643921,
      "log_odds_ratio": -0.5712890625,
      "logits/chosen": -0.7784179449081421,
      "logits/rejected": -0.689257800579071,
      "logps/chosen": -0.626757800579071,
      "logps/rejected": -0.978710949420929,
      "loss": 0.7947,
      "nll_loss": 0.7212890386581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06264648586511612,
      "rewards/margins": 0.03515319898724556,
      "rewards/rejected": -0.09780273586511612,
      "step": 22970
    },
    {
      "epoch": 0.8611418185906203,
      "grad_norm": 2.7621418434262215,
      "learning_rate": 5.27733878130888e-07,
      "log_odds_chosen": 0.580517590045929,
      "log_odds_ratio": -0.5928710699081421,
      "logits/chosen": -0.6923828125,
      "logits/rejected": -0.6309570074081421,
      "logps/chosen": -0.6460937261581421,
      "logps/rejected": -1.008886694908142,
      "loss": 0.7933,
      "nll_loss": 0.718554675579071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06463623046875,
      "rewards/margins": 0.03631896898150444,
      "rewards/rejected": -0.1009521484375,
      "step": 22980
    },
    {
      "epoch": 0.8615165539337842,
      "grad_norm": 2.969603076324686,
      "learning_rate": 5.276190909890993e-07,
      "log_odds_chosen": 0.619873046875,
      "log_odds_ratio": -0.5381835699081421,
      "logits/chosen": -0.776562511920929,
      "logits/rejected": -0.614453136920929,
      "logps/chosen": -0.6832031011581421,
      "logps/rejected": -1.070898413658142,
      "loss": 0.8141,
      "nll_loss": 0.746777355670929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06827392429113388,
      "rewards/margins": 0.03882446140050888,
      "rewards/rejected": -0.107177734375,
      "step": 22990
    },
    {
      "epoch": 0.8618912892769481,
      "grad_norm": 2.749586880071676,
      "learning_rate": 5.275043787166296e-07,
      "log_odds_chosen": 0.49620360136032104,
      "log_odds_ratio": -0.570019543170929,
      "logits/chosen": -0.7679687738418579,
      "logits/rejected": -0.6558593511581421,
      "logps/chosen": -0.637988269329071,
      "logps/rejected": -0.9505859613418579,
      "loss": 0.7857,
      "nll_loss": 0.719921886920929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06381835788488388,
      "rewards/margins": 0.03130187839269638,
      "rewards/rejected": -0.09504394233226776,
      "step": 23000
    },
    {
      "epoch": 0.862266024620112,
      "grad_norm": 2.4105398563888327,
      "learning_rate": 5.273897412321254e-07,
      "log_odds_chosen": 0.5959106683731079,
      "log_odds_ratio": -0.56689453125,
      "logits/chosen": -0.741894543170929,
      "logits/rejected": -0.630175769329071,
      "logps/chosen": -0.662402331829071,
      "logps/rejected": -1.0242187976837158,
      "loss": 0.8159,
      "nll_loss": 0.74609375,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06623534858226776,
      "rewards/margins": 0.03619994968175888,
      "rewards/rejected": -0.10239257663488388,
      "step": 23010
    },
    {
      "epoch": 0.8626407599632759,
      "grad_norm": 2.569029366072318,
      "learning_rate": 5.272751784543577e-07,
      "log_odds_chosen": 0.546154797077179,
      "log_odds_ratio": -0.580761730670929,
      "logits/chosen": -0.73046875,
      "logits/rejected": -0.6591796875,
      "logps/chosen": -0.668261706829071,
      "logps/rejected": -1.0246093273162842,
      "loss": 0.796,
      "nll_loss": 0.720410168170929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06674804538488388,
      "rewards/margins": 0.03568267822265625,
      "rewards/rejected": -0.10245361179113388,
      "step": 23020
    },
    {
      "epoch": 0.8630154953064398,
      "grad_norm": 2.846711328303072,
      "learning_rate": 5.2716069030222e-07,
      "log_odds_chosen": 0.41704100370407104,
      "log_odds_ratio": -0.6376953125,
      "logits/chosen": -0.717578113079071,
      "logits/rejected": -0.601757824420929,
      "logps/chosen": -0.7646484375,
      "logps/rejected": -1.0460937023162842,
      "loss": 0.797,
      "nll_loss": 0.7906249761581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07652588188648224,
      "rewards/margins": 0.02810363844037056,
      "rewards/rejected": -0.10458984225988388,
      "step": 23030
    },
    {
      "epoch": 0.8633902306496037,
      "grad_norm": 2.646847113326175,
      "learning_rate": 5.270462766947299e-07,
      "log_odds_chosen": 0.6308838129043579,
      "log_odds_ratio": -0.56201171875,
      "logits/chosen": -0.730273425579071,
      "logits/rejected": -0.6119140386581421,
      "logps/chosen": -0.6629883050918579,
      "logps/rejected": -1.05859375,
      "loss": 0.8048,
      "nll_loss": 0.7582031488418579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06629638373851776,
      "rewards/margins": 0.039550017565488815,
      "rewards/rejected": -0.10588379204273224,
      "step": 23040
    },
    {
      "epoch": 0.8637649659927676,
      "grad_norm": 2.6692858651198246,
      "learning_rate": 5.269319375510273e-07,
      "log_odds_chosen": 0.4422607421875,
      "log_odds_ratio": -0.6092773675918579,
      "logits/chosen": -0.762890636920929,
      "logits/rejected": -0.6529296636581421,
      "logps/chosen": -0.705078125,
      "logps/rejected": -0.9774414300918579,
      "loss": 0.7882,
      "nll_loss": 0.755664050579071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07049560546875,
      "rewards/margins": 0.02723388746380806,
      "rewards/rejected": -0.09782715141773224,
      "step": 23050
    },
    {
      "epoch": 0.8641397013359315,
      "grad_norm": 2.590605831911629,
      "learning_rate": 5.268176727903755e-07,
      "log_odds_chosen": 0.48896485567092896,
      "log_odds_ratio": -0.595703125,
      "logits/chosen": -0.706835925579071,
      "logits/rejected": -0.623339831829071,
      "logps/chosen": -0.641894519329071,
      "logps/rejected": -0.9566406011581421,
      "loss": 0.8195,
      "nll_loss": 0.7875000238418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06418456882238388,
      "rewards/margins": 0.031487274914979935,
      "rewards/rejected": -0.09569092094898224,
      "step": 23060
    },
    {
      "epoch": 0.8645144366790954,
      "grad_norm": 3.160212856242745,
      "learning_rate": 5.267034823321595e-07,
      "log_odds_chosen": 0.4576782286167145,
      "log_odds_ratio": -0.612500011920929,
      "logits/chosen": -0.724609375,
      "logits/rejected": -0.6714843511581421,
      "logps/chosen": -0.696582019329071,
      "logps/rejected": -0.9683593511581421,
      "loss": 0.811,
      "nll_loss": 0.81494140625,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.0697021484375,
      "rewards/margins": 0.02712097205221653,
      "rewards/rejected": -0.09682617336511612,
      "step": 23070
    },
    {
      "epoch": 0.8648891720222592,
      "grad_norm": 2.5673506300003823,
      "learning_rate": 5.265893660958874e-07,
      "log_odds_chosen": 0.67236328125,
      "log_odds_ratio": -0.5347656011581421,
      "logits/chosen": -0.747851550579071,
      "logits/rejected": -0.669921875,
      "logps/chosen": -0.6607421636581421,
      "logps/rejected": -1.0705077648162842,
      "loss": 0.8089,
      "nll_loss": 0.754687488079071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06602783501148224,
      "rewards/margins": 0.04095458984375,
      "rewards/rejected": -0.10708007961511612,
      "step": 23080
    },
    {
      "epoch": 0.8652639073654231,
      "grad_norm": 3.2940400052755723,
      "learning_rate": 5.264753240011888e-07,
      "log_odds_chosen": 0.6784301996231079,
      "log_odds_ratio": -0.5360351800918579,
      "logits/chosen": -0.774609386920929,
      "logits/rejected": -0.6629883050918579,
      "logps/chosen": -0.655566394329071,
      "logps/rejected": -1.0998046398162842,
      "loss": 0.803,
      "nll_loss": 0.7236328125,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06557617336511612,
      "rewards/margins": 0.044525910168886185,
      "rewards/rejected": -0.11008300632238388,
      "step": 23090
    },
    {
      "epoch": 0.865638642708587,
      "grad_norm": 3.023261319480094,
      "learning_rate": 5.263613559678151e-07,
      "log_odds_chosen": 0.43305665254592896,
      "log_odds_ratio": -0.647265613079071,
      "logits/chosen": -0.7353515625,
      "logits/rejected": -0.6368163824081421,
      "logps/chosen": -0.6587890386581421,
      "logps/rejected": -0.9302734136581421,
      "loss": 0.8068,
      "nll_loss": 0.7098633050918579,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06582031399011612,
      "rewards/margins": 0.02712383307516575,
      "rewards/rejected": -0.09300537407398224,
      "step": 23100
    },
    {
      "epoch": 0.8660133780517509,
      "grad_norm": 2.4513301609254325,
      "learning_rate": 5.262474619156395e-07,
      "log_odds_chosen": 0.3974853456020355,
      "log_odds_ratio": -0.6275390386581421,
      "logits/chosen": -0.7623046636581421,
      "logits/rejected": -0.68701171875,
      "logps/chosen": -0.6763671636581421,
      "logps/rejected": -0.933398425579071,
      "loss": 0.8012,
      "nll_loss": 0.795703113079071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06766357272863388,
      "rewards/margins": 0.02563629113137722,
      "rewards/rejected": -0.09328613430261612,
      "step": 23110
    },
    {
      "epoch": 0.8663881133949148,
      "grad_norm": 3.228344372484821,
      "learning_rate": 5.261336417646563e-07,
      "log_odds_chosen": 0.592114269733429,
      "log_odds_ratio": -0.562695324420929,
      "logits/chosen": -0.778124988079071,
      "logits/rejected": -0.6533203125,
      "logps/chosen": -0.6241210699081421,
      "logps/rejected": -0.9892578125,
      "loss": 0.7964,
      "nll_loss": 0.747265636920929,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.0623779296875,
      "rewards/margins": 0.03644409030675888,
      "rewards/rejected": -0.09886474907398224,
      "step": 23120
    },
    {
      "epoch": 0.8667628487380787,
      "grad_norm": 2.359937546425096,
      "learning_rate": 5.26019895434981e-07,
      "log_odds_chosen": 0.740551769733429,
      "log_odds_ratio": -0.5116211175918579,
      "logits/chosen": -0.756640613079071,
      "logits/rejected": -0.637011706829071,
      "logps/chosen": -0.669140636920929,
      "logps/rejected": -1.141015648841858,
      "loss": 0.7997,
      "nll_loss": 0.728515625,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06690673530101776,
      "rewards/margins": 0.04726715013384819,
      "rewards/rejected": -0.11423339694738388,
      "step": 23130
    },
    {
      "epoch": 0.8671375840812426,
      "grad_norm": 2.44382987595849,
      "learning_rate": 5.259062228468499e-07,
      "log_odds_chosen": 0.49896240234375,
      "log_odds_ratio": -0.612011730670929,
      "logits/chosen": -0.7474609613418579,
      "logits/rejected": -0.6532226800918579,
      "logps/chosen": -0.737500011920929,
      "logps/rejected": -1.0773437023162842,
      "loss": 0.8078,
      "nll_loss": 0.8041015863418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07370605319738388,
      "rewards/margins": 0.034050751477479935,
      "rewards/rejected": -0.10773925483226776,
      "step": 23140
    },
    {
      "epoch": 0.8675123194244065,
      "grad_norm": 2.6265083357704384,
      "learning_rate": 5.257926239206199e-07,
      "log_odds_chosen": 0.537548840045929,
      "log_odds_ratio": -0.567675769329071,
      "logits/chosen": -0.7123047113418579,
      "logits/rejected": -0.6328125,
      "logps/chosen": -0.6288086175918579,
      "logps/rejected": -0.9624999761581421,
      "loss": 0.7982,
      "nll_loss": 0.7420898675918579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06287841498851776,
      "rewards/margins": 0.033264923840761185,
      "rewards/rejected": -0.09611816704273224,
      "step": 23150
    },
    {
      "epoch": 0.8678870547675704,
      "grad_norm": 2.485394059054734,
      "learning_rate": 5.256790985767682e-07,
      "log_odds_chosen": 0.636608898639679,
      "log_odds_ratio": -0.57080078125,
      "logits/chosen": -0.7798827886581421,
      "logits/rejected": -0.6363281011581421,
      "logps/chosen": -0.667187511920929,
      "logps/rejected": -1.0900390148162842,
      "loss": 0.7929,
      "nll_loss": 0.7515624761581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06662597507238388,
      "rewards/margins": 0.04236755520105362,
      "rewards/rejected": -0.10903320461511612,
      "step": 23160
    },
    {
      "epoch": 0.8682617901107343,
      "grad_norm": 2.6783088964407398,
      "learning_rate": 5.255656467358922e-07,
      "log_odds_chosen": 0.46467286348342896,
      "log_odds_ratio": -0.6156250238418579,
      "logits/chosen": -0.777539074420929,
      "logits/rejected": -0.7095702886581421,
      "logps/chosen": -0.7049804925918579,
      "logps/rejected": -0.9945312738418579,
      "loss": 0.8006,
      "nll_loss": 0.764355480670929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.070556640625,
      "rewards/margins": 0.02896270714700222,
      "rewards/rejected": -0.0994873046875,
      "step": 23170
    },
    {
      "epoch": 0.8686365254538981,
      "grad_norm": 2.349827886745532,
      "learning_rate": 5.254522683187093e-07,
      "log_odds_chosen": 0.46281737089157104,
      "log_odds_ratio": -0.616503894329071,
      "logits/chosen": -0.7710937261581421,
      "logits/rejected": -0.6869140863418579,
      "logps/chosen": -0.7173827886581421,
      "logps/rejected": -0.983203113079071,
      "loss": 0.7989,
      "nll_loss": 0.7505859136581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07171630859375,
      "rewards/margins": 0.02657775953412056,
      "rewards/rejected": -0.09827880561351776,
      "step": 23180
    },
    {
      "epoch": 0.869011260797062,
      "grad_norm": 2.719377949162481,
      "learning_rate": 5.253389632460566e-07,
      "log_odds_chosen": 0.4887939393520355,
      "log_odds_ratio": -0.594531238079071,
      "logits/chosen": -0.7406250238418579,
      "logits/rejected": -0.672070324420929,
      "logps/chosen": -0.681347668170929,
      "logps/rejected": -0.9716796875,
      "loss": 0.7998,
      "nll_loss": 0.776562511920929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06815185397863388,
      "rewards/margins": 0.02904052659869194,
      "rewards/rejected": -0.09724120795726776,
      "step": 23190
    },
    {
      "epoch": 0.8693859961402259,
      "grad_norm": 2.7367541160505193,
      "learning_rate": 5.252257314388901e-07,
      "log_odds_chosen": 0.5058227777481079,
      "log_odds_ratio": -0.578417956829071,
      "logits/chosen": -0.7679687738418579,
      "logits/rejected": -0.7064453363418579,
      "logps/chosen": -0.73828125,
      "logps/rejected": -1.0490233898162842,
      "loss": 0.7992,
      "nll_loss": 0.7679687738418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07380370795726776,
      "rewards/margins": 0.03110199049115181,
      "rewards/rejected": -0.10493163764476776,
      "step": 23200
    },
    {
      "epoch": 0.8697607314833898,
      "grad_norm": 3.9808758372505157,
      "learning_rate": 5.251125728182861e-07,
      "log_odds_chosen": 0.5855346918106079,
      "log_odds_ratio": -0.584765613079071,
      "logits/chosen": -0.764453113079071,
      "logits/rejected": -0.6678711175918579,
      "logps/chosen": -0.6727539300918579,
      "logps/rejected": -1.0595703125,
      "loss": 0.8162,
      "nll_loss": 0.796679675579071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06728515774011612,
      "rewards/margins": 0.03867340087890625,
      "rewards/rejected": -0.10593261569738388,
      "step": 23210
    },
    {
      "epoch": 0.8701354668265537,
      "grad_norm": 3.431347988714219,
      "learning_rate": 5.249994873054385e-07,
      "log_odds_chosen": 0.501782238483429,
      "log_odds_ratio": -0.5962890386581421,
      "logits/chosen": -0.7452148199081421,
      "logits/rejected": -0.6346679925918579,
      "logps/chosen": -0.6792968511581421,
      "logps/rejected": -1.0144531726837158,
      "loss": 0.8062,
      "nll_loss": 0.732226550579071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06789550930261612,
      "rewards/margins": 0.03358001634478569,
      "rewards/rejected": -0.10142822563648224,
      "step": 23220
    },
    {
      "epoch": 0.8705102021697176,
      "grad_norm": 2.6037725079498637,
      "learning_rate": 5.24886474821661e-07,
      "log_odds_chosen": 0.632824718952179,
      "log_odds_ratio": -0.5728515386581421,
      "logits/chosen": -0.7611328363418579,
      "logits/rejected": -0.672558605670929,
      "logps/chosen": -0.638867199420929,
      "logps/rejected": -1.035546898841858,
      "loss": 0.798,
      "nll_loss": 0.701953113079071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06386718899011612,
      "rewards/margins": 0.039659880101680756,
      "rewards/rejected": -0.10354004055261612,
      "step": 23230
    },
    {
      "epoch": 0.8708849375128815,
      "grad_norm": 2.489166383644843,
      "learning_rate": 5.247735352883857e-07,
      "log_odds_chosen": 0.3530212342739105,
      "log_odds_ratio": -0.667285144329071,
      "logits/chosen": -0.7841796875,
      "logits/rejected": -0.712207019329071,
      "logps/chosen": -0.769726574420929,
      "logps/rejected": -1.010351538658142,
      "loss": 0.8196,
      "nll_loss": 0.7613281011581421,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0770263671875,
      "rewards/margins": 0.0239715576171875,
      "rewards/rejected": -0.10098876804113388,
      "step": 23240
    },
    {
      "epoch": 0.8712596728560454,
      "grad_norm": 2.3334773670075193,
      "learning_rate": 5.246606686271623e-07,
      "log_odds_chosen": 0.604418933391571,
      "log_odds_ratio": -0.556835949420929,
      "logits/chosen": -0.8080078363418579,
      "logits/rejected": -0.661816418170929,
      "logps/chosen": -0.6373046636581421,
      "logps/rejected": -0.9789062738418579,
      "loss": 0.8084,
      "nll_loss": 0.75048828125,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06367187201976776,
      "rewards/margins": 0.03421173244714737,
      "rewards/rejected": -0.09797362983226776,
      "step": 23250
    },
    {
      "epoch": 0.8716344081992093,
      "grad_norm": 2.2564014718789767,
      "learning_rate": 5.245478747596593e-07,
      "log_odds_chosen": 0.5962280035018921,
      "log_odds_ratio": -0.56201171875,
      "logits/chosen": -0.771484375,
      "logits/rejected": -0.645214855670929,
      "logps/chosen": -0.69091796875,
      "logps/rejected": -1.0695312023162842,
      "loss": 0.7936,
      "nll_loss": 0.7621093988418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06914062798023224,
      "rewards/margins": 0.03783111646771431,
      "rewards/rejected": -0.10700683295726776,
      "step": 23260
    },
    {
      "epoch": 0.8720091435423732,
      "grad_norm": 2.5372173329249117,
      "learning_rate": 5.244351536076629e-07,
      "log_odds_chosen": 0.51080322265625,
      "log_odds_ratio": -0.599609375,
      "logits/chosen": -0.7367187738418579,
      "logits/rejected": -0.6318359375,
      "logps/chosen": -0.680859386920929,
      "logps/rejected": -1.0320312976837158,
      "loss": 0.8136,
      "nll_loss": 0.7286132574081421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06805419921875,
      "rewards/margins": 0.0351409912109375,
      "rewards/rejected": -0.10312499850988388,
      "step": 23270
    },
    {
      "epoch": 0.872383878885537,
      "grad_norm": 2.511060306566341,
      "learning_rate": 5.243225050930763e-07,
      "log_odds_chosen": 0.694287121295929,
      "log_odds_ratio": -0.5450195074081421,
      "logits/chosen": -0.7835937738418579,
      "logits/rejected": -0.63037109375,
      "logps/chosen": -0.68359375,
      "logps/rejected": -1.1427733898162842,
      "loss": 0.7964,
      "nll_loss": 0.7230468988418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.068359375,
      "rewards/margins": 0.045928955078125,
      "rewards/rejected": -0.1143798828125,
      "step": 23280
    },
    {
      "epoch": 0.8727586142287009,
      "grad_norm": 3.0403106950322547,
      "learning_rate": 5.242099291379208e-07,
      "log_odds_chosen": 0.619555652141571,
      "log_odds_ratio": -0.5450195074081421,
      "logits/chosen": -0.7769531011581421,
      "logits/rejected": -0.673828125,
      "logps/chosen": -0.6299804449081421,
      "logps/rejected": -1.0271484851837158,
      "loss": 0.7881,
      "nll_loss": 0.7591797113418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06295166164636612,
      "rewards/margins": 0.03974456712603569,
      "rewards/rejected": -0.10272216796875,
      "step": 23290
    },
    {
      "epoch": 0.8731333495718648,
      "grad_norm": 2.5117285647520444,
      "learning_rate": 5.240974256643347e-07,
      "log_odds_chosen": 0.4560180604457855,
      "log_odds_ratio": -0.6021484136581421,
      "logits/chosen": -0.7964843511581421,
      "logits/rejected": -0.675097644329071,
      "logps/chosen": -0.6553710699081421,
      "logps/rejected": -0.955859363079071,
      "loss": 0.7957,
      "nll_loss": 0.73828125,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06553955376148224,
      "rewards/margins": 0.02999420091509819,
      "rewards/rejected": -0.09552001953125,
      "step": 23300
    },
    {
      "epoch": 0.8735080849150287,
      "grad_norm": 2.445648111710749,
      "learning_rate": 5.239849945945729e-07,
      "log_odds_chosen": 0.48359376192092896,
      "log_odds_ratio": -0.612011730670929,
      "logits/chosen": -0.72900390625,
      "logits/rejected": -0.6410156488418579,
      "logps/chosen": -0.650097668170929,
      "logps/rejected": -0.9496093988418579,
      "loss": 0.8032,
      "nll_loss": 0.78466796875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06500244140625,
      "rewards/margins": 0.02995605394244194,
      "rewards/rejected": -0.094970703125,
      "step": 23310
    },
    {
      "epoch": 0.8738828202581926,
      "grad_norm": 2.266684250309454,
      "learning_rate": 5.238726358510071e-07,
      "log_odds_chosen": 0.55059814453125,
      "log_odds_ratio": -0.5658203363418579,
      "logits/chosen": -0.746874988079071,
      "logits/rejected": -0.625195324420929,
      "logps/chosen": -0.63916015625,
      "logps/rejected": -0.9830077886581421,
      "loss": 0.8063,
      "nll_loss": 0.7347656488418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06383056938648224,
      "rewards/margins": 0.03443298488855362,
      "rewards/rejected": -0.09822998195886612,
      "step": 23320
    },
    {
      "epoch": 0.8742575556013565,
      "grad_norm": 2.6335552703513296,
      "learning_rate": 5.237603493561259e-07,
      "log_odds_chosen": 0.47270506620407104,
      "log_odds_ratio": -0.590039074420929,
      "logits/chosen": -0.823437511920929,
      "logits/rejected": -0.7099609375,
      "logps/chosen": -0.674023449420929,
      "logps/rejected": -0.968554675579071,
      "loss": 0.7983,
      "nll_loss": 0.785351574420929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06745605170726776,
      "rewards/margins": 0.02936401404440403,
      "rewards/rejected": -0.0968017578125,
      "step": 23330
    },
    {
      "epoch": 0.8746322909445204,
      "grad_norm": 2.485891258817316,
      "learning_rate": 5.236481350325335e-07,
      "log_odds_chosen": 0.5960448980331421,
      "log_odds_ratio": -0.565625011920929,
      "logits/chosen": -0.747851550579071,
      "logits/rejected": -0.691601574420929,
      "logps/chosen": -0.654492199420929,
      "logps/rejected": -1.0109374523162842,
      "loss": 0.8203,
      "nll_loss": 0.7984374761581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06541748344898224,
      "rewards/margins": 0.03567962720990181,
      "rewards/rejected": -0.10109863430261612,
      "step": 23340
    },
    {
      "epoch": 0.8750070262876843,
      "grad_norm": 3.1338604462063806,
      "learning_rate": 5.235359928029507e-07,
      "log_odds_chosen": 0.537890613079071,
      "log_odds_ratio": -0.584667980670929,
      "logits/chosen": -0.7396484613418579,
      "logits/rejected": -0.6333984136581421,
      "logps/chosen": -0.657421886920929,
      "logps/rejected": -0.9966796636581421,
      "loss": 0.8003,
      "nll_loss": 0.7582031488418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06572265923023224,
      "rewards/margins": 0.03395385667681694,
      "rewards/rejected": -0.099609375,
      "step": 23350
    },
    {
      "epoch": 0.8753817616308482,
      "grad_norm": 2.404153953593196,
      "learning_rate": 5.234239225902136e-07,
      "log_odds_chosen": 0.555310070514679,
      "log_odds_ratio": -0.5770508050918579,
      "logits/chosen": -0.765820324420929,
      "logits/rejected": -0.6626952886581421,
      "logps/chosen": -0.671093761920929,
      "logps/rejected": -1.01953125,
      "loss": 0.7976,
      "nll_loss": 0.7386718988418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06715087592601776,
      "rewards/margins": 0.03486023098230362,
      "rewards/rejected": -0.10195312649011612,
      "step": 23360
    },
    {
      "epoch": 0.8757564969740121,
      "grad_norm": 2.296844010747421,
      "learning_rate": 5.233119243172744e-07,
      "log_odds_chosen": 0.3099365234375,
      "log_odds_ratio": -0.65771484375,
      "logits/chosen": -0.7564452886581421,
      "logits/rejected": -0.6666015386581421,
      "logps/chosen": -0.7203124761581421,
      "logps/rejected": -0.9091796875,
      "loss": 0.796,
      "nll_loss": 0.755859375,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07208251953125,
      "rewards/margins": 0.018798828125,
      "rewards/rejected": -0.09079589694738388,
      "step": 23370
    },
    {
      "epoch": 0.876131232317176,
      "grad_norm": 2.7751176315387323,
      "learning_rate": 5.231999979071999e-07,
      "log_odds_chosen": 0.49798583984375,
      "log_odds_ratio": -0.5912109613418579,
      "logits/chosen": -0.7822265625,
      "logits/rejected": -0.6620117425918579,
      "logps/chosen": -0.695605456829071,
      "logps/rejected": -1.0226562023162842,
      "loss": 0.7969,
      "nll_loss": 0.775195300579071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06955566257238388,
      "rewards/margins": 0.03275909274816513,
      "rewards/rejected": -0.102294921875,
      "step": 23380
    },
    {
      "epoch": 0.8765059676603398,
      "grad_norm": 2.9024297414090365,
      "learning_rate": 5.23088143283173e-07,
      "log_odds_chosen": 0.4908447265625,
      "log_odds_ratio": -0.588085949420929,
      "logits/chosen": -0.774609386920929,
      "logits/rejected": -0.64794921875,
      "logps/chosen": -0.6558593511581421,
      "logps/rejected": -0.9551757574081421,
      "loss": 0.8008,
      "nll_loss": 0.7392578125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06557617336511612,
      "rewards/margins": 0.02995910681784153,
      "rewards/rejected": -0.09548339992761612,
      "step": 23390
    },
    {
      "epoch": 0.8768807030035037,
      "grad_norm": 2.7660521282951347,
      "learning_rate": 5.229763603684907e-07,
      "log_odds_chosen": 0.5635010004043579,
      "log_odds_ratio": -0.547558605670929,
      "logits/chosen": -0.749804675579071,
      "logits/rejected": -0.640820324420929,
      "logps/chosen": -0.661914050579071,
      "logps/rejected": -1.020898461341858,
      "loss": 0.8087,
      "nll_loss": 0.7992187738418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06622314453125,
      "rewards/margins": 0.03595580905675888,
      "rewards/rejected": -0.10207519680261612,
      "step": 23400
    },
    {
      "epoch": 0.8772554383466676,
      "grad_norm": 2.387001055819077,
      "learning_rate": 5.228646490865652e-07,
      "log_odds_chosen": 0.557751476764679,
      "log_odds_ratio": -0.576171875,
      "logits/chosen": -0.71728515625,
      "logits/rejected": -0.6318359375,
      "logps/chosen": -0.6654297113418579,
      "logps/rejected": -0.9830077886581421,
      "loss": 0.8058,
      "nll_loss": 0.756542980670929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06649170070886612,
      "rewards/margins": 0.03176574781537056,
      "rewards/rejected": -0.09822998195886612,
      "step": 23410
    },
    {
      "epoch": 0.8776301736898315,
      "grad_norm": 2.424364995473455,
      "learning_rate": 5.227530093609228e-07,
      "log_odds_chosen": 0.5972900390625,
      "log_odds_ratio": -0.5584961175918579,
      "logits/chosen": -0.7933593988418579,
      "logits/rejected": -0.681640625,
      "logps/chosen": -0.646679699420929,
      "logps/rejected": -1.0281250476837158,
      "loss": 0.7945,
      "nll_loss": 0.70458984375,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06470946967601776,
      "rewards/margins": 0.03814697265625,
      "rewards/rejected": -0.10288085788488388,
      "step": 23420
    },
    {
      "epoch": 0.8780049090329954,
      "grad_norm": 2.8554859738027387,
      "learning_rate": 5.226414411152042e-07,
      "log_odds_chosen": 0.518798828125,
      "log_odds_ratio": -0.5762695074081421,
      "logits/chosen": -0.7925781011581421,
      "logits/rejected": -0.704296886920929,
      "logps/chosen": -0.638671875,
      "logps/rejected": -0.9652343988418579,
      "loss": 0.7798,
      "nll_loss": 0.70263671875,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0638427734375,
      "rewards/margins": 0.03276367112994194,
      "rewards/rejected": -0.09656982123851776,
      "step": 23430
    },
    {
      "epoch": 0.8783796443761593,
      "grad_norm": 3.5033751247264693,
      "learning_rate": 5.225299442731641e-07,
      "log_odds_chosen": 0.30853271484375,
      "log_odds_ratio": -0.6649414300918579,
      "logits/chosen": -0.7157226800918579,
      "logits/rejected": -0.6202148199081421,
      "logps/chosen": -0.67578125,
      "logps/rejected": -0.870898425579071,
      "loss": 0.7929,
      "nll_loss": 0.73828125,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06755371391773224,
      "rewards/margins": 0.01958160474896431,
      "rewards/rejected": -0.08713378757238388,
      "step": 23440
    },
    {
      "epoch": 0.8787543797193232,
      "grad_norm": 2.6311793827916787,
      "learning_rate": 5.224185187586712e-07,
      "log_odds_chosen": 0.686755359172821,
      "log_odds_ratio": -0.575488269329071,
      "logits/chosen": -0.8228515386581421,
      "logits/rejected": -0.6502929925918579,
      "logps/chosen": -0.6400390863418579,
      "logps/rejected": -1.0935547351837158,
      "loss": 0.7729,
      "nll_loss": 0.690136730670929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06402587890625,
      "rewards/margins": 0.045409392565488815,
      "rewards/rejected": -0.10939941555261612,
      "step": 23450
    },
    {
      "epoch": 0.8791291150624871,
      "grad_norm": 2.152702375209156,
      "learning_rate": 5.223071644957079e-07,
      "log_odds_chosen": 0.48566895723342896,
      "log_odds_ratio": -0.592968761920929,
      "logits/chosen": -0.7591797113418579,
      "logits/rejected": -0.6973632574081421,
      "logps/chosen": -0.7054687738418579,
      "logps/rejected": -1.015039086341858,
      "loss": 0.8184,
      "nll_loss": 0.793164074420929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07054443657398224,
      "rewards/margins": 0.03099517896771431,
      "rewards/rejected": -0.10153808444738388,
      "step": 23460
    },
    {
      "epoch": 0.879503850405651,
      "grad_norm": 2.4928457180289443,
      "learning_rate": 5.221958814083692e-07,
      "log_odds_chosen": 0.502612292766571,
      "log_odds_ratio": -0.595410168170929,
      "logits/chosen": -0.80078125,
      "logits/rejected": -0.6767578125,
      "logps/chosen": -0.67626953125,
      "logps/rejected": -0.982421875,
      "loss": 0.8316,
      "nll_loss": 0.784375011920929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06766357272863388,
      "rewards/margins": 0.03057708777487278,
      "rewards/rejected": -0.09812011569738388,
      "step": 23470
    },
    {
      "epoch": 0.879878585748815,
      "grad_norm": 2.585616668708669,
      "learning_rate": 5.220846694208641e-07,
      "log_odds_chosen": 0.40080565214157104,
      "log_odds_ratio": -0.61083984375,
      "logits/chosen": -0.8013671636581421,
      "logits/rejected": -0.6561523675918579,
      "logps/chosen": -0.68359375,
      "logps/rejected": -0.946093738079071,
      "loss": 0.8046,
      "nll_loss": 0.7197265625,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06832275539636612,
      "rewards/margins": 0.02628173865377903,
      "rewards/rejected": -0.09467773139476776,
      "step": 23480
    },
    {
      "epoch": 0.8802533210919787,
      "grad_norm": 2.4760181268154797,
      "learning_rate": 5.219735284575142e-07,
      "log_odds_chosen": 0.46868896484375,
      "log_odds_ratio": -0.5912109613418579,
      "logits/chosen": -0.77734375,
      "logits/rejected": -0.6654297113418579,
      "logps/chosen": -0.670117199420929,
      "logps/rejected": -0.9624999761581421,
      "loss": 0.8147,
      "nll_loss": 0.7779296636581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06707763671875,
      "rewards/margins": 0.029163360595703125,
      "rewards/rejected": -0.09617920219898224,
      "step": 23490
    },
    {
      "epoch": 0.8806280564351426,
      "grad_norm": 2.5687718962770987,
      "learning_rate": 5.218624584427538e-07,
      "log_odds_chosen": 0.596606433391571,
      "log_odds_ratio": -0.5499023199081421,
      "logits/chosen": -0.732226550579071,
      "logits/rejected": -0.6434570550918579,
      "logps/chosen": -0.6869140863418579,
      "logps/rejected": -1.0392577648162842,
      "loss": 0.8099,
      "nll_loss": 0.792285144329071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06871338188648224,
      "rewards/margins": 0.035163115710020065,
      "rewards/rejected": -0.10383300483226776,
      "step": 23500
    },
    {
      "epoch": 0.8810027917783065,
      "grad_norm": 2.586019725805773,
      "learning_rate": 5.217514593011297e-07,
      "log_odds_chosen": 0.4046386778354645,
      "log_odds_ratio": -0.6185547113418579,
      "logits/chosen": -0.780468761920929,
      "logits/rejected": -0.707226574420929,
      "logps/chosen": -0.6976562738418579,
      "logps/rejected": -0.9437500238418579,
      "loss": 0.7934,
      "nll_loss": 0.7420898675918579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06979980319738388,
      "rewards/margins": 0.02459411695599556,
      "rewards/rejected": -0.09442138671875,
      "step": 23510
    },
    {
      "epoch": 0.8813775271214704,
      "grad_norm": 2.5147292667022363,
      "learning_rate": 5.216405309573011e-07,
      "log_odds_chosen": 0.511474609375,
      "log_odds_ratio": -0.571093738079071,
      "logits/chosen": -0.8216797113418579,
      "logits/rejected": -0.6903320550918579,
      "logps/chosen": -0.6996093988418579,
      "logps/rejected": -1.0158202648162842,
      "loss": 0.8126,
      "nll_loss": 0.755664050579071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07000732421875,
      "rewards/margins": 0.03159179538488388,
      "rewards/rejected": -0.10163573920726776,
      "step": 23520
    },
    {
      "epoch": 0.8817522624646343,
      "grad_norm": 3.0459373111140566,
      "learning_rate": 5.215296733360391e-07,
      "log_odds_chosen": 0.4854492247104645,
      "log_odds_ratio": -0.582324206829071,
      "logits/chosen": -0.734082043170929,
      "logits/rejected": -0.685351550579071,
      "logps/chosen": -0.6737304925918579,
      "logps/rejected": -0.9898437261581421,
      "loss": 0.7936,
      "nll_loss": 0.7208007574081421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.0673828125,
      "rewards/margins": 0.03160705417394638,
      "rewards/rejected": -0.09904785454273224,
      "step": 23530
    },
    {
      "epoch": 0.8821269978077982,
      "grad_norm": 2.6979959117671113,
      "learning_rate": 5.214188863622271e-07,
      "log_odds_chosen": 0.6179443597793579,
      "log_odds_ratio": -0.546191394329071,
      "logits/chosen": -0.759570300579071,
      "logits/rejected": -0.6744140386581421,
      "logps/chosen": -0.5986328125,
      "logps/rejected": -0.98046875,
      "loss": 0.8017,
      "nll_loss": 0.6957031488418579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.05987548828125,
      "rewards/margins": 0.03816833347082138,
      "rewards/rejected": -0.09801025688648224,
      "step": 23540
    },
    {
      "epoch": 0.8825017331509621,
      "grad_norm": 2.693043043860959,
      "learning_rate": 5.213081699608596e-07,
      "log_odds_chosen": 0.561108410358429,
      "log_odds_ratio": -0.57958984375,
      "logits/chosen": -0.7964843511581421,
      "logits/rejected": -0.692578136920929,
      "logps/chosen": -0.64208984375,
      "logps/rejected": -0.994335949420929,
      "loss": 0.7891,
      "nll_loss": 0.7271484136581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06409911811351776,
      "rewards/margins": 0.035218048840761185,
      "rewards/rejected": -0.09943847358226776,
      "step": 23550
    },
    {
      "epoch": 0.882876468494126,
      "grad_norm": 2.739571455187103,
      "learning_rate": 5.21197524057043e-07,
      "log_odds_chosen": 0.749218761920929,
      "log_odds_ratio": -0.50390625,
      "logits/chosen": -0.7748047113418579,
      "logits/rejected": -0.647656261920929,
      "logps/chosen": -0.628222644329071,
      "logps/rejected": -1.0734374523162842,
      "loss": 0.7957,
      "nll_loss": 0.763671875,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06278076022863388,
      "rewards/margins": 0.04453277587890625,
      "rewards/rejected": -0.10728760063648224,
      "step": 23560
    },
    {
      "epoch": 0.8832512038372899,
      "grad_norm": 3.1736629306685242,
      "learning_rate": 5.210869485759943e-07,
      "log_odds_chosen": 0.5403503179550171,
      "log_odds_ratio": -0.58349609375,
      "logits/chosen": -0.7613281011581421,
      "logits/rejected": -0.6441406011581421,
      "logps/chosen": -0.678906261920929,
      "logps/rejected": -1.026953101158142,
      "loss": 0.7843,
      "nll_loss": 0.750195324420929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06784667819738388,
      "rewards/margins": 0.034813690930604935,
      "rewards/rejected": -0.10272216796875,
      "step": 23570
    },
    {
      "epoch": 0.8836259391804538,
      "grad_norm": 2.5832459262949414,
      "learning_rate": 5.209764434430422e-07,
      "log_odds_chosen": 0.4836181700229645,
      "log_odds_ratio": -0.6064453125,
      "logits/chosen": -0.778515636920929,
      "logits/rejected": -0.678417980670929,
      "logps/chosen": -0.66650390625,
      "logps/rejected": -0.9681640863418579,
      "loss": 0.7962,
      "nll_loss": 0.69921875,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.066650390625,
      "rewards/margins": 0.03018493577837944,
      "rewards/rejected": -0.09682617336511612,
      "step": 23580
    },
    {
      "epoch": 0.8840006745236177,
      "grad_norm": 2.5313371996053777,
      "learning_rate": 5.20866008583626e-07,
      "log_odds_chosen": 0.3165039122104645,
      "log_odds_ratio": -0.640625,
      "logits/chosen": -0.7730468511581421,
      "logits/rejected": -0.6898437738418579,
      "logps/chosen": -0.70556640625,
      "logps/rejected": -0.91015625,
      "loss": 0.7961,
      "nll_loss": 0.7474609613418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07049560546875,
      "rewards/margins": 0.020369719713926315,
      "rewards/rejected": -0.09099121391773224,
      "step": 23590
    },
    {
      "epoch": 0.8843754098667815,
      "grad_norm": 3.06701836124124,
      "learning_rate": 5.207556439232955e-07,
      "log_odds_chosen": 0.39655762910842896,
      "log_odds_ratio": -0.62744140625,
      "logits/chosen": -0.817578136920929,
      "logits/rejected": -0.732226550579071,
      "logps/chosen": -0.6973632574081421,
      "logps/rejected": -0.934765636920929,
      "loss": 0.7991,
      "nll_loss": 0.757031261920929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06971435248851776,
      "rewards/margins": 0.02368469163775444,
      "rewards/rejected": -0.09335937350988388,
      "step": 23600
    },
    {
      "epoch": 0.8847501452099454,
      "grad_norm": 2.370870833335759,
      "learning_rate": 5.206453493877105e-07,
      "log_odds_chosen": 0.579150378704071,
      "log_odds_ratio": -0.5654296875,
      "logits/chosen": -0.823437511920929,
      "logits/rejected": -0.665234386920929,
      "logps/chosen": -0.65478515625,
      "logps/rejected": -1.0255858898162842,
      "loss": 0.7901,
      "nll_loss": 0.706835925579071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06544189155101776,
      "rewards/margins": 0.03700103610754013,
      "rewards/rejected": -0.10247802734375,
      "step": 23610
    },
    {
      "epoch": 0.8851248805531093,
      "grad_norm": 3.01626744980227,
      "learning_rate": 5.205351249026418e-07,
      "log_odds_chosen": 0.576708972454071,
      "log_odds_ratio": -0.5689452886581421,
      "logits/chosen": -0.769726574420929,
      "logits/rejected": -0.670605480670929,
      "logps/chosen": -0.665234386920929,
      "logps/rejected": -1.013671875,
      "loss": 0.8165,
      "nll_loss": 0.765429675579071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06654052436351776,
      "rewards/margins": 0.034835051745176315,
      "rewards/rejected": -0.10141601413488388,
      "step": 23620
    },
    {
      "epoch": 0.8854996158962732,
      "grad_norm": 2.931141544962545,
      "learning_rate": 5.204249703939695e-07,
      "log_odds_chosen": 0.4609619081020355,
      "log_odds_ratio": -0.603808581829071,
      "logits/chosen": -0.772753894329071,
      "logits/rejected": -0.667285144329071,
      "logps/chosen": -0.6866210699081421,
      "logps/rejected": -0.986132800579071,
      "loss": 0.819,
      "nll_loss": 0.805468738079071,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06868896633386612,
      "rewards/margins": 0.02997589111328125,
      "rewards/rejected": -0.0986328125,
      "step": 23630
    },
    {
      "epoch": 0.8858743512394371,
      "grad_norm": 2.660721845725094,
      "learning_rate": 5.203148857876837e-07,
      "log_odds_chosen": 0.6556640863418579,
      "log_odds_ratio": -0.5367187261581421,
      "logits/chosen": -0.7284179925918579,
      "logits/rejected": -0.6484375,
      "logps/chosen": -0.642773449420929,
      "logps/rejected": -1.049218773841858,
      "loss": 0.8001,
      "nll_loss": 0.7421875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06425781548023224,
      "rewards/margins": 0.04063262790441513,
      "rewards/rejected": -0.10488281399011612,
      "step": 23640
    },
    {
      "epoch": 0.886249086582601,
      "grad_norm": 2.8813526521074153,
      "learning_rate": 5.202048710098841e-07,
      "log_odds_chosen": 0.57659912109375,
      "log_odds_ratio": -0.580078125,
      "logits/chosen": -0.8121093511581421,
      "logits/rejected": -0.6854492425918579,
      "logps/chosen": -0.6416991949081421,
      "logps/rejected": -1.021093726158142,
      "loss": 0.7942,
      "nll_loss": 0.71435546875,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06423339992761612,
      "rewards/margins": 0.03779144212603569,
      "rewards/rejected": -0.10200195014476776,
      "step": 23650
    },
    {
      "epoch": 0.8866238219257649,
      "grad_norm": 2.6273942179543686,
      "learning_rate": 5.200949259867794e-07,
      "log_odds_chosen": 0.5951782464981079,
      "log_odds_ratio": -0.5460449457168579,
      "logits/chosen": -0.7835937738418579,
      "logits/rejected": -0.677929699420929,
      "logps/chosen": -0.666308581829071,
      "logps/rejected": -1.0353515148162842,
      "loss": 0.7801,
      "nll_loss": 0.71484375,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06663818657398224,
      "rewards/margins": 0.036887358874082565,
      "rewards/rejected": -0.10354004055261612,
      "step": 23660
    },
    {
      "epoch": 0.8869985572689288,
      "grad_norm": 2.9541615534642016,
      "learning_rate": 5.199850506446878e-07,
      "log_odds_chosen": 0.5234009027481079,
      "log_odds_ratio": -0.568164050579071,
      "logits/chosen": -0.7783203125,
      "logits/rejected": -0.6878906488418579,
      "logps/chosen": -0.6415039300918579,
      "logps/rejected": -0.9750000238418579,
      "loss": 0.7993,
      "nll_loss": 0.730273425579071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06411132961511612,
      "rewards/margins": 0.0334320068359375,
      "rewards/rejected": -0.09757079929113388,
      "step": 23670
    },
    {
      "epoch": 0.8873732926120927,
      "grad_norm": 3.0706742801022395,
      "learning_rate": 5.198752449100363e-07,
      "log_odds_chosen": 0.5751953125,
      "log_odds_ratio": -0.5614258050918579,
      "logits/chosen": -0.809374988079071,
      "logits/rejected": -0.6841796636581421,
      "logps/chosen": -0.636035144329071,
      "logps/rejected": -0.9937499761581421,
      "loss": 0.8034,
      "nll_loss": 0.7564452886581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.0635986328125,
      "rewards/margins": 0.03580322116613388,
      "rewards/rejected": -0.09945068508386612,
      "step": 23680
    },
    {
      "epoch": 0.8877480279552566,
      "grad_norm": 2.881062621752266,
      "learning_rate": 5.197655087093606e-07,
      "log_odds_chosen": 0.4727539122104645,
      "log_odds_ratio": -0.606152355670929,
      "logits/chosen": -0.7339843511581421,
      "logits/rejected": -0.637011706829071,
      "logps/chosen": -0.676953136920929,
      "logps/rejected": -0.971875011920929,
      "loss": 0.8018,
      "nll_loss": 0.755664050579071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06772460788488388,
      "rewards/margins": 0.02947692945599556,
      "rewards/rejected": -0.09714355319738388,
      "step": 23690
    },
    {
      "epoch": 0.8881227632984204,
      "grad_norm": 2.556929231625225,
      "learning_rate": 5.196558419693047e-07,
      "log_odds_chosen": 0.661242663860321,
      "log_odds_ratio": -0.5458008050918579,
      "logits/chosen": -0.7671874761581421,
      "logits/rejected": -0.6722656488418579,
      "logps/chosen": -0.624316394329071,
      "logps/rejected": -1.0556640625,
      "loss": 0.7895,
      "nll_loss": 0.734179675579071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06240234524011612,
      "rewards/margins": 0.043121337890625,
      "rewards/rejected": -0.10551758110523224,
      "step": 23700
    },
    {
      "epoch": 0.8884974986415843,
      "grad_norm": 2.63108144658107,
      "learning_rate": 5.195462446166212e-07,
      "log_odds_chosen": 0.5662597417831421,
      "log_odds_ratio": -0.557421863079071,
      "logits/chosen": -0.7865234613418579,
      "logits/rejected": -0.664746105670929,
      "logps/chosen": -0.644726574420929,
      "logps/rejected": -0.9769531488418579,
      "loss": 0.7822,
      "nll_loss": 0.717578113079071,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06444092094898224,
      "rewards/margins": 0.03316955640912056,
      "rewards/rejected": -0.09769286960363388,
      "step": 23710
    },
    {
      "epoch": 0.8888722339847482,
      "grad_norm": 2.548664841929456,
      "learning_rate": 5.194367165781708e-07,
      "log_odds_chosen": 0.45728760957717896,
      "log_odds_ratio": -0.5947265625,
      "logits/chosen": -0.828320324420929,
      "logits/rejected": -0.7040039300918579,
      "logps/chosen": -0.6258789300918579,
      "logps/rejected": -0.90234375,
      "loss": 0.8112,
      "nll_loss": 0.7181640863418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06259765475988388,
      "rewards/margins": 0.027600860223174095,
      "rewards/rejected": -0.09018554538488388,
      "step": 23720
    },
    {
      "epoch": 0.8892469693279121,
      "grad_norm": 2.4445627139045483,
      "learning_rate": 5.193272577809217e-07,
      "log_odds_chosen": 0.46486204862594604,
      "log_odds_ratio": -0.5966796875,
      "logits/chosen": -0.8001953363418579,
      "logits/rejected": -0.6708984375,
      "logps/chosen": -0.6689453125,
      "logps/rejected": -0.9603515863418579,
      "loss": 0.8075,
      "nll_loss": 0.761914074420929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06685791164636612,
      "rewards/margins": 0.029161835089325905,
      "rewards/rejected": -0.09602050483226776,
      "step": 23730
    },
    {
      "epoch": 0.889621704671076,
      "grad_norm": 3.645875745411632,
      "learning_rate": 5.192178681519504e-07,
      "log_odds_chosen": 0.44633787870407104,
      "log_odds_ratio": -0.61279296875,
      "logits/chosen": -0.7743164300918579,
      "logits/rejected": -0.692187488079071,
      "logps/chosen": -0.6991211175918579,
      "logps/rejected": -0.953125,
      "loss": 0.8024,
      "nll_loss": 0.7557617425918579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06982421875,
      "rewards/margins": 0.02546081505715847,
      "rewards/rejected": -0.09523925930261612,
      "step": 23740
    },
    {
      "epoch": 0.8899964400142399,
      "grad_norm": 3.0229245106453435,
      "learning_rate": 5.191085476184402e-07,
      "log_odds_chosen": 0.49342042207717896,
      "log_odds_ratio": -0.6185547113418579,
      "logits/chosen": -0.7730468511581421,
      "logits/rejected": -0.6507812738418579,
      "logps/chosen": -0.7212890386581421,
      "logps/rejected": -1.0470702648162842,
      "loss": 0.8008,
      "nll_loss": 0.78125,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07222900539636612,
      "rewards/margins": 0.03243865817785263,
      "rewards/rejected": -0.1046142578125,
      "step": 23750
    },
    {
      "epoch": 0.8903711753574038,
      "grad_norm": 2.6360497582676543,
      "learning_rate": 5.18999296107682e-07,
      "log_odds_chosen": 0.551684558391571,
      "log_odds_ratio": -0.590624988079071,
      "logits/chosen": -0.764843761920929,
      "logits/rejected": -0.6937500238418579,
      "logps/chosen": -0.7222656011581421,
      "logps/rejected": -1.023828148841858,
      "loss": 0.7943,
      "nll_loss": 0.7328125238418579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07221679389476776,
      "rewards/margins": 0.0302276611328125,
      "rewards/rejected": -0.10249023139476776,
      "step": 23760
    },
    {
      "epoch": 0.8907459107005677,
      "grad_norm": 2.3992577022779966,
      "learning_rate": 5.188901135470739e-07,
      "log_odds_chosen": 0.654736340045929,
      "log_odds_ratio": -0.5331054925918579,
      "logits/chosen": -0.788281261920929,
      "logits/rejected": -0.6845703125,
      "logps/chosen": -0.6343749761581421,
      "logps/rejected": -1.0412108898162842,
      "loss": 0.7987,
      "nll_loss": 0.7250000238418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0634765625,
      "rewards/margins": 0.04058227688074112,
      "rewards/rejected": -0.10406494140625,
      "step": 23770
    },
    {
      "epoch": 0.8911206460437316,
      "grad_norm": 3.052212306489975,
      "learning_rate": 5.187809998641207e-07,
      "log_odds_chosen": 0.5137573480606079,
      "log_odds_ratio": -0.60107421875,
      "logits/chosen": -0.7783203125,
      "logits/rejected": -0.6806640625,
      "logps/chosen": -0.7119140625,
      "logps/rejected": -1.036718726158142,
      "loss": 0.8037,
      "nll_loss": 0.7398437261581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07114257663488388,
      "rewards/margins": 0.032501984387636185,
      "rewards/rejected": -0.10366211086511612,
      "step": 23780
    },
    {
      "epoch": 0.8914953813868955,
      "grad_norm": 3.5151185203354696,
      "learning_rate": 5.186719549864339e-07,
      "log_odds_chosen": 0.4224609434604645,
      "log_odds_ratio": -0.621289074420929,
      "logits/chosen": -0.738085925579071,
      "logits/rejected": -0.6712890863418579,
      "logps/chosen": -0.6722656488418579,
      "logps/rejected": -0.9156249761581421,
      "loss": 0.8021,
      "nll_loss": 0.7259765863418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06721191108226776,
      "rewards/margins": 0.02439269982278347,
      "rewards/rejected": -0.09165038913488388,
      "step": 23790
    },
    {
      "epoch": 0.8918701167300594,
      "grad_norm": 3.2993359062148837,
      "learning_rate": 5.185629788417315e-07,
      "log_odds_chosen": 0.57373046875,
      "log_odds_ratio": -0.5770508050918579,
      "logits/chosen": -0.7701171636581421,
      "logits/rejected": -0.68115234375,
      "logps/chosen": -0.6587890386581421,
      "logps/rejected": -1.0515625476837158,
      "loss": 0.7865,
      "nll_loss": 0.7505859136581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06580810248851776,
      "rewards/margins": 0.03934936597943306,
      "rewards/rejected": -0.105224609375,
      "step": 23800
    },
    {
      "epoch": 0.8922448520732232,
      "grad_norm": 2.9490481023684234,
      "learning_rate": 5.184540713578377e-07,
      "log_odds_chosen": 0.540209949016571,
      "log_odds_ratio": -0.587109386920929,
      "logits/chosen": -0.7964843511581421,
      "logits/rejected": -0.6904296875,
      "logps/chosen": -0.61474609375,
      "logps/rejected": -0.9593750238418579,
      "loss": 0.8185,
      "nll_loss": 0.7339843511581421,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06145019456744194,
      "rewards/margins": 0.034537505358457565,
      "rewards/rejected": -0.09593506157398224,
      "step": 23810
    },
    {
      "epoch": 0.8926195874163871,
      "grad_norm": 2.685772354704749,
      "learning_rate": 5.183452324626827e-07,
      "log_odds_chosen": 0.5049072504043579,
      "log_odds_ratio": -0.5868164300918579,
      "logits/chosen": -0.780078113079071,
      "logits/rejected": -0.695117175579071,
      "logps/chosen": -0.690136730670929,
      "logps/rejected": -1.0154297351837158,
      "loss": 0.7991,
      "nll_loss": 0.732226550579071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06895752251148224,
      "rewards/margins": 0.032656095921993256,
      "rewards/rejected": -0.10166015475988388,
      "step": 23820
    },
    {
      "epoch": 0.892994322759551,
      "grad_norm": 2.834162132644446,
      "learning_rate": 5.182364620843029e-07,
      "log_odds_chosen": 0.574755847454071,
      "log_odds_ratio": -0.5702148675918579,
      "logits/chosen": -0.758496105670929,
      "logits/rejected": -0.662792980670929,
      "logps/chosen": -0.67724609375,
      "logps/rejected": -1.019921898841858,
      "loss": 0.7876,
      "nll_loss": 0.679882824420929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06777343899011612,
      "rewards/margins": 0.03430785983800888,
      "rewards/rejected": -0.10197754204273224,
      "step": 23830
    },
    {
      "epoch": 0.8933690581027149,
      "grad_norm": 3.3966528069235253,
      "learning_rate": 5.181277601508397e-07,
      "log_odds_chosen": 0.534130871295929,
      "log_odds_ratio": -0.5738281011581421,
      "logits/chosen": -0.755566418170929,
      "logits/rejected": -0.636425793170929,
      "logps/chosen": -0.680468738079071,
      "logps/rejected": -1.0046875476837158,
      "loss": 0.7937,
      "nll_loss": 0.690136730670929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06802978366613388,
      "rewards/margins": 0.032421112060546875,
      "rewards/rejected": -0.10041503608226776,
      "step": 23840
    },
    {
      "epoch": 0.8937437934458788,
      "grad_norm": 2.6371485671732713,
      "learning_rate": 5.180191265905408e-07,
      "log_odds_chosen": 0.55999755859375,
      "log_odds_ratio": -0.57958984375,
      "logits/chosen": -0.7577148675918579,
      "logits/rejected": -0.644824206829071,
      "logps/chosen": -0.616015613079071,
      "logps/rejected": -0.9697265625,
      "loss": 0.7695,
      "nll_loss": 0.732617199420929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06162109225988388,
      "rewards/margins": 0.03540496900677681,
      "rewards/rejected": -0.09696044772863388,
      "step": 23850
    },
    {
      "epoch": 0.8941185287890427,
      "grad_norm": 3.270860345716662,
      "learning_rate": 5.179105613317587e-07,
      "log_odds_chosen": 0.3984130918979645,
      "log_odds_ratio": -0.6263672113418579,
      "logits/chosen": -0.6943359375,
      "logits/rejected": -0.620898425579071,
      "logps/chosen": -0.6929687261581421,
      "logps/rejected": -0.951953113079071,
      "loss": 0.7997,
      "nll_loss": 0.751171886920929,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06922607123851776,
      "rewards/margins": 0.02583160437643528,
      "rewards/rejected": -0.09511718899011612,
      "step": 23860
    },
    {
      "epoch": 0.8944932641322066,
      "grad_norm": 2.8746595552739183,
      "learning_rate": 5.178020643029507e-07,
      "log_odds_chosen": 0.7240356206893921,
      "log_odds_ratio": -0.527050793170929,
      "logits/chosen": -0.7349609136581421,
      "logits/rejected": -0.645703136920929,
      "logps/chosen": -0.6534179449081421,
      "logps/rejected": -1.1101562976837158,
      "loss": 0.7908,
      "nll_loss": 0.7357422113418579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06534423679113388,
      "rewards/margins": 0.04561958461999893,
      "rewards/rejected": -0.11098632961511612,
      "step": 23870
    },
    {
      "epoch": 0.8948679994753705,
      "grad_norm": 2.972929592673397,
      "learning_rate": 5.176936354326795e-07,
      "log_odds_chosen": 0.49964600801467896,
      "log_odds_ratio": -0.575976550579071,
      "logits/chosen": -0.726367175579071,
      "logits/rejected": -0.6246093511581421,
      "logps/chosen": -0.619433581829071,
      "logps/rejected": -0.9488281011581421,
      "loss": 0.8095,
      "nll_loss": 0.758984386920929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06197509914636612,
      "rewards/margins": 0.032863616943359375,
      "rewards/rejected": -0.0948486328125,
      "step": 23880
    },
    {
      "epoch": 0.8952427348185344,
      "grad_norm": 2.5848176995729477,
      "learning_rate": 5.175852746496124e-07,
      "log_odds_chosen": 0.5293945074081421,
      "log_odds_ratio": -0.577441394329071,
      "logits/chosen": -0.806835949420929,
      "logits/rejected": -0.655468761920929,
      "logps/chosen": -0.7177734375,
      "logps/rejected": -1.0732421875,
      "loss": 0.8177,
      "nll_loss": 0.788867175579071,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07176513969898224,
      "rewards/margins": 0.03556213527917862,
      "rewards/rejected": -0.1072998046875,
      "step": 23890
    },
    {
      "epoch": 0.8956174701616983,
      "grad_norm": 2.711552354039133,
      "learning_rate": 5.174769818825206e-07,
      "log_odds_chosen": 0.5757812261581421,
      "log_odds_ratio": -0.560546875,
      "logits/chosen": -0.7740234136581421,
      "logits/rejected": -0.671093761920929,
      "logps/chosen": -0.665820300579071,
      "logps/rejected": -1.008398413658142,
      "loss": 0.8033,
      "nll_loss": 0.733203113079071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06657715141773224,
      "rewards/margins": 0.03427734225988388,
      "rewards/rejected": -0.10090331733226776,
      "step": 23900
    },
    {
      "epoch": 0.8959922055048621,
      "grad_norm": 3.0709859447189585,
      "learning_rate": 5.173687570602804e-07,
      "log_odds_chosen": 0.5671142339706421,
      "log_odds_ratio": -0.55712890625,
      "logits/chosen": -0.7740234136581421,
      "logits/rejected": -0.684374988079071,
      "logps/chosen": -0.665332019329071,
      "logps/rejected": -1.018164038658142,
      "loss": 0.7852,
      "nll_loss": 0.708789050579071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0665283203125,
      "rewards/margins": 0.03528137132525444,
      "rewards/rejected": -0.10179443657398224,
      "step": 23910
    },
    {
      "epoch": 0.896366940848026,
      "grad_norm": 2.636234903750073,
      "learning_rate": 5.172606001118717e-07,
      "log_odds_chosen": 0.49095457792282104,
      "log_odds_ratio": -0.600781261920929,
      "logits/chosen": -0.7822265625,
      "logits/rejected": -0.674023449420929,
      "logps/chosen": -0.6719726324081421,
      "logps/rejected": -0.985546886920929,
      "loss": 0.7914,
      "nll_loss": 0.6871093511581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06721191108226776,
      "rewards/margins": 0.03141174465417862,
      "rewards/rejected": -0.09868164360523224,
      "step": 23920
    },
    {
      "epoch": 0.8967416761911899,
      "grad_norm": 2.8879698202140363,
      "learning_rate": 5.171525109663781e-07,
      "log_odds_chosen": 0.52935791015625,
      "log_odds_ratio": -0.570507824420929,
      "logits/chosen": -0.756542980670929,
      "logits/rejected": -0.682812511920929,
      "logps/chosen": -0.656933605670929,
      "logps/rejected": -0.954296886920929,
      "loss": 0.7862,
      "nll_loss": 0.7662109136581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06569824367761612,
      "rewards/margins": 0.029705047607421875,
      "rewards/rejected": -0.09543456882238388,
      "step": 23930
    },
    {
      "epoch": 0.8971164115343538,
      "grad_norm": 2.3471723798044204,
      "learning_rate": 5.170444895529875e-07,
      "log_odds_chosen": 0.4716552793979645,
      "log_odds_ratio": -0.604785144329071,
      "logits/chosen": -0.783007800579071,
      "logits/rejected": -0.6200195550918579,
      "logps/chosen": -0.6683593988418579,
      "logps/rejected": -0.9867187738418579,
      "loss": 0.7997,
      "nll_loss": 0.7587890625,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06687011569738388,
      "rewards/margins": 0.03188171237707138,
      "rewards/rejected": -0.09874267876148224,
      "step": 23940
    },
    {
      "epoch": 0.8974911468775177,
      "grad_norm": 2.6361101313072104,
      "learning_rate": 5.169365358009907e-07,
      "log_odds_chosen": 0.61614990234375,
      "log_odds_ratio": -0.563183605670929,
      "logits/chosen": -0.7242187261581421,
      "logits/rejected": -0.643750011920929,
      "logps/chosen": -0.6429687738418579,
      "logps/rejected": -1.036718726158142,
      "loss": 0.8022,
      "nll_loss": 0.6947265863418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06429443508386612,
      "rewards/margins": 0.039373017847537994,
      "rewards/rejected": -0.10371093451976776,
      "step": 23950
    },
    {
      "epoch": 0.8978658822206816,
      "grad_norm": 3.013601163344419,
      "learning_rate": 5.168286496397822e-07,
      "log_odds_chosen": 0.690258800983429,
      "log_odds_ratio": -0.538281261920929,
      "logits/chosen": -0.755664050579071,
      "logits/rejected": -0.647167980670929,
      "logps/chosen": -0.634570300579071,
      "logps/rejected": -1.058007836341858,
      "loss": 0.7968,
      "nll_loss": 0.6976562738418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06350097805261612,
      "rewards/margins": 0.042316436767578125,
      "rewards/rejected": -0.10579834133386612,
      "step": 23960
    },
    {
      "epoch": 0.8982406175638455,
      "grad_norm": 2.7147998543968415,
      "learning_rate": 5.167208309988594e-07,
      "log_odds_chosen": 0.7586914300918579,
      "log_odds_ratio": -0.521679699420929,
      "logits/chosen": -0.76953125,
      "logits/rejected": -0.6499999761581421,
      "logps/chosen": -0.6728515625,
      "logps/rejected": -1.1857421398162842,
      "loss": 0.8199,
      "nll_loss": 0.746777355670929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06728515774011612,
      "rewards/margins": 0.05139465257525444,
      "rewards/rejected": -0.11859130859375,
      "step": 23970
    },
    {
      "epoch": 0.8986153529070094,
      "grad_norm": 2.3775869164680747,
      "learning_rate": 5.166130798078227e-07,
      "log_odds_chosen": 0.5708373785018921,
      "log_odds_ratio": -0.5438476800918579,
      "logits/chosen": -0.8187500238418579,
      "logits/rejected": -0.671191394329071,
      "logps/chosen": -0.644238293170929,
      "logps/rejected": -0.9957031011581421,
      "loss": 0.7817,
      "nll_loss": 0.6962890625,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06450195610523224,
      "rewards/margins": 0.03516387939453125,
      "rewards/rejected": -0.09963379055261612,
      "step": 23980
    },
    {
      "epoch": 0.8989900882501733,
      "grad_norm": 2.732620533579093,
      "learning_rate": 5.16505395996375e-07,
      "log_odds_chosen": 0.665942370891571,
      "log_odds_ratio": -0.542773425579071,
      "logits/chosen": -0.7767578363418579,
      "logits/rejected": -0.675976574420929,
      "logps/chosen": -0.672070324420929,
      "logps/rejected": -1.0890624523162842,
      "loss": 0.7865,
      "nll_loss": 0.758984386920929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06722412258386612,
      "rewards/margins": 0.04155273362994194,
      "rewards/rejected": -0.10878906399011612,
      "step": 23990
    },
    {
      "epoch": 0.8993648235933372,
      "grad_norm": 2.417345918665572,
      "learning_rate": 5.163977794943222e-07,
      "log_odds_chosen": 0.599011242389679,
      "log_odds_ratio": -0.5708984136581421,
      "logits/chosen": -0.770703136920929,
      "logits/rejected": -0.6558593511581421,
      "logps/chosen": -0.634960949420929,
      "logps/rejected": -0.9966796636581421,
      "loss": 0.8121,
      "nll_loss": 0.752734363079071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06348876655101776,
      "rewards/margins": 0.03609924390912056,
      "rewards/rejected": -0.09963379055261612,
      "step": 24000
    },
    {
      "epoch": 0.8997395589365011,
      "grad_norm": 3.8415891663650825,
      "learning_rate": 5.162902302315721e-07,
      "log_odds_chosen": 0.66650390625,
      "log_odds_ratio": -0.53857421875,
      "logits/chosen": -0.776562511920929,
      "logits/rejected": -0.62744140625,
      "logps/chosen": -0.679003894329071,
      "logps/rejected": -1.10546875,
      "loss": 0.7838,
      "nll_loss": 0.751171886920929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06793212890625,
      "rewards/margins": 0.04258880764245987,
      "rewards/rejected": -0.11054687201976776,
      "step": 24010
    },
    {
      "epoch": 0.9001142942796649,
      "grad_norm": 2.675689931816951,
      "learning_rate": 5.161827481381348e-07,
      "log_odds_chosen": 0.724536120891571,
      "log_odds_ratio": -0.5259765386581421,
      "logits/chosen": -0.808789074420929,
      "logits/rejected": -0.65673828125,
      "logps/chosen": -0.658203125,
      "logps/rejected": -1.1474609375,
      "loss": 0.7851,
      "nll_loss": 0.716503918170929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06585693359375,
      "rewards/margins": 0.048873137682676315,
      "rewards/rejected": -0.11470947414636612,
      "step": 24020
    },
    {
      "epoch": 0.9004890296228288,
      "grad_norm": 3.444888502867325,
      "learning_rate": 5.160753331441223e-07,
      "log_odds_chosen": 0.4674438536167145,
      "log_odds_ratio": -0.60986328125,
      "logits/chosen": -0.788867175579071,
      "logits/rejected": -0.682812511920929,
      "logps/chosen": -0.661328136920929,
      "logps/rejected": -0.9794921875,
      "loss": 0.7898,
      "nll_loss": 0.7265625,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.066162109375,
      "rewards/margins": 0.031777191907167435,
      "rewards/rejected": -0.09793701022863388,
      "step": 24030
    },
    {
      "epoch": 0.9008637649659927,
      "grad_norm": 2.774336499637925,
      "learning_rate": 5.159679851797486e-07,
      "log_odds_chosen": 0.645642101764679,
      "log_odds_ratio": -0.557812511920929,
      "logits/chosen": -0.758105456829071,
      "logits/rejected": -0.671875,
      "logps/chosen": -0.651171863079071,
      "logps/rejected": -1.055078148841858,
      "loss": 0.7944,
      "nll_loss": 0.634570300579071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06511230766773224,
      "rewards/margins": 0.04043731838464737,
      "rewards/rejected": -0.10552978515625,
      "step": 24040
    },
    {
      "epoch": 0.9012385003091566,
      "grad_norm": 2.8573194037151826,
      "learning_rate": 5.158607041753288e-07,
      "log_odds_chosen": 0.5318237543106079,
      "log_odds_ratio": -0.605761706829071,
      "logits/chosen": -0.703906238079071,
      "logits/rejected": -0.629589855670929,
      "logps/chosen": -0.6602538824081421,
      "logps/rejected": -1.005468726158142,
      "loss": 0.8128,
      "nll_loss": 0.7236328125,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06601562350988388,
      "rewards/margins": 0.03450164943933487,
      "rewards/rejected": -0.10064697265625,
      "step": 24050
    },
    {
      "epoch": 0.9016132356523205,
      "grad_norm": 2.6877386182102696,
      "learning_rate": 5.157534900612799e-07,
      "log_odds_chosen": 0.562817394733429,
      "log_odds_ratio": -0.577343761920929,
      "logits/chosen": -0.752148449420929,
      "logits/rejected": -0.635449230670929,
      "logps/chosen": -0.6485351324081421,
      "logps/rejected": -1.0285155773162842,
      "loss": 0.7915,
      "nll_loss": 0.730664074420929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06486816704273224,
      "rewards/margins": 0.03786773607134819,
      "rewards/rejected": -0.10270996391773224,
      "step": 24060
    },
    {
      "epoch": 0.9019879709954844,
      "grad_norm": 2.1697912202087624,
      "learning_rate": 5.156463427681196e-07,
      "log_odds_chosen": 0.38786619901657104,
      "log_odds_ratio": -0.626660168170929,
      "logits/chosen": -0.786914050579071,
      "logits/rejected": -0.7216796875,
      "logps/chosen": -0.6529296636581421,
      "logps/rejected": -0.8656250238418579,
      "loss": 0.7958,
      "nll_loss": 0.748828113079071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06533203274011612,
      "rewards/margins": 0.02126922644674778,
      "rewards/rejected": -0.08656005561351776,
      "step": 24070
    },
    {
      "epoch": 0.9023627063386483,
      "grad_norm": 2.773341763175397,
      "learning_rate": 5.155392622264671e-07,
      "log_odds_chosen": 0.6016601324081421,
      "log_odds_ratio": -0.536816418170929,
      "logits/chosen": -0.790820300579071,
      "logits/rejected": -0.6299804449081421,
      "logps/chosen": -0.636523425579071,
      "logps/rejected": -1.033203125,
      "loss": 0.8157,
      "nll_loss": 0.7607421875,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06362304836511612,
      "rewards/margins": 0.03970642015337944,
      "rewards/rejected": -0.10334472358226776,
      "step": 24080
    },
    {
      "epoch": 0.9027374416818122,
      "grad_norm": 2.5162359191048953,
      "learning_rate": 5.154322483670419e-07,
      "log_odds_chosen": 0.6153808832168579,
      "log_odds_ratio": -0.5821288824081421,
      "logits/chosen": -0.789843738079071,
      "logits/rejected": -0.671191394329071,
      "logps/chosen": -0.6611328125,
      "logps/rejected": -1.063867211341858,
      "loss": 0.7883,
      "nll_loss": 0.715039074420929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06612548977136612,
      "rewards/margins": 0.04018402099609375,
      "rewards/rejected": -0.1063232421875,
      "step": 24090
    },
    {
      "epoch": 0.9031121770249761,
      "grad_norm": 2.943296184612799,
      "learning_rate": 5.153253011206647e-07,
      "log_odds_chosen": 0.5019165277481079,
      "log_odds_ratio": -0.574023425579071,
      "logits/chosen": -0.8115234375,
      "logits/rejected": -0.687304675579071,
      "logps/chosen": -0.664355456829071,
      "logps/rejected": -0.9603515863418579,
      "loss": 0.7817,
      "nll_loss": 0.7369140386581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06646728515625,
      "rewards/margins": 0.029604339972138405,
      "rewards/rejected": -0.09598388522863388,
      "step": 24100
    },
    {
      "epoch": 0.90348691236814,
      "grad_norm": 3.440247803061129,
      "learning_rate": 5.15218420418256e-07,
      "log_odds_chosen": 0.6385742425918579,
      "log_odds_ratio": -0.5399414300918579,
      "logits/chosen": -0.7923828363418579,
      "logits/rejected": -0.6878906488418579,
      "logps/chosen": -0.637499988079071,
      "logps/rejected": -1.047265648841858,
      "loss": 0.8092,
      "nll_loss": 0.7396484613418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06373290717601776,
      "rewards/margins": 0.0409698486328125,
      "rewards/rejected": -0.10472412407398224,
      "step": 24110
    },
    {
      "epoch": 0.9038616477113038,
      "grad_norm": 2.3840757589941406,
      "learning_rate": 5.151116061908373e-07,
      "log_odds_chosen": 0.2786498963832855,
      "log_odds_ratio": -0.71044921875,
      "logits/chosen": -0.7601562738418579,
      "logits/rejected": -0.6962890625,
      "logps/chosen": -0.7349609136581421,
      "logps/rejected": -0.9189453125,
      "loss": 0.7859,
      "nll_loss": 0.77734375,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.07351074367761612,
      "rewards/margins": 0.01841583289206028,
      "rewards/rejected": -0.09193114936351776,
      "step": 24120
    },
    {
      "epoch": 0.9042363830544677,
      "grad_norm": 2.96678232186127,
      "learning_rate": 5.150048583695291e-07,
      "log_odds_chosen": 0.47978514432907104,
      "log_odds_ratio": -0.609667956829071,
      "logits/chosen": -0.80078125,
      "logits/rejected": -0.672656238079071,
      "logps/chosen": -0.6996093988418579,
      "logps/rejected": -1.0138671398162842,
      "loss": 0.7972,
      "nll_loss": 0.718457043170929,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06997070461511612,
      "rewards/margins": 0.03146057203412056,
      "rewards/rejected": -0.10141601413488388,
      "step": 24130
    },
    {
      "epoch": 0.9046111183976316,
      "grad_norm": 2.852522426911981,
      "learning_rate": 5.14898176885553e-07,
      "log_odds_chosen": 0.41734617948532104,
      "log_odds_ratio": -0.6065429449081421,
      "logits/chosen": -0.766796886920929,
      "logits/rejected": -0.7007812261581421,
      "logps/chosen": -0.681445300579071,
      "logps/rejected": -0.9169921875,
      "loss": 0.7778,
      "nll_loss": 0.727734386920929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06818847358226776,
      "rewards/margins": 0.02350463904440403,
      "rewards/rejected": -0.0916748046875,
      "step": 24140
    },
    {
      "epoch": 0.9049858537407955,
      "grad_norm": 2.640376381019405,
      "learning_rate": 5.147915616702294e-07,
      "log_odds_chosen": 0.41893309354782104,
      "log_odds_ratio": -0.626953125,
      "logits/chosen": -0.780468761920929,
      "logits/rejected": -0.7051757574081421,
      "logps/chosen": -0.701953113079071,
      "logps/rejected": -0.9560546875,
      "loss": 0.7986,
      "nll_loss": 0.7582031488418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07020263373851776,
      "rewards/margins": 0.02543792687356472,
      "rewards/rejected": -0.0955810546875,
      "step": 24150
    },
    {
      "epoch": 0.9053605890839594,
      "grad_norm": 3.2415097133702897,
      "learning_rate": 5.146850126549788e-07,
      "log_odds_chosen": 0.43842774629592896,
      "log_odds_ratio": -0.609375,
      "logits/chosen": -0.7596679925918579,
      "logits/rejected": -0.6629883050918579,
      "logps/chosen": -0.666699230670929,
      "logps/rejected": -0.9212890863418579,
      "loss": 0.7913,
      "nll_loss": 0.7177734375,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06662597507238388,
      "rewards/margins": 0.0254974365234375,
      "rewards/rejected": -0.09211425483226776,
      "step": 24160
    },
    {
      "epoch": 0.9057353244271233,
      "grad_norm": 3.6447671888192117,
      "learning_rate": 5.145785297713203e-07,
      "log_odds_chosen": 0.65673828125,
      "log_odds_ratio": -0.5337890386581421,
      "logits/chosen": -0.7904297113418579,
      "logits/rejected": -0.6727539300918579,
      "logps/chosen": -0.655957043170929,
      "logps/rejected": -1.0789062976837158,
      "loss": 0.7585,
      "nll_loss": 0.6800781488418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06563720852136612,
      "rewards/margins": 0.04223327711224556,
      "rewards/rejected": -0.10788574069738388,
      "step": 24170
    },
    {
      "epoch": 0.9061100597702872,
      "grad_norm": 3.1434166616686814,
      "learning_rate": 5.144721129508728e-07,
      "log_odds_chosen": 0.38328856229782104,
      "log_odds_ratio": -0.626660168170929,
      "logits/chosen": -0.784863293170929,
      "logits/rejected": -0.702343761920929,
      "logps/chosen": -0.687304675579071,
      "logps/rejected": -0.908398449420929,
      "loss": 0.784,
      "nll_loss": 0.701855480670929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06867675483226776,
      "rewards/margins": 0.022048186510801315,
      "rewards/rejected": -0.09078369289636612,
      "step": 24180
    },
    {
      "epoch": 0.9064847951134511,
      "grad_norm": 2.716027212960791,
      "learning_rate": 5.143657621253539e-07,
      "log_odds_chosen": 0.4759277403354645,
      "log_odds_ratio": -0.598925769329071,
      "logits/chosen": -0.725781261920929,
      "logits/rejected": -0.6357421875,
      "logps/chosen": -0.6680663824081421,
      "logps/rejected": -0.978710949420929,
      "loss": 0.7934,
      "nll_loss": 0.7442382574081421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.0667724609375,
      "rewards/margins": 0.031049346551299095,
      "rewards/rejected": -0.09785155951976776,
      "step": 24190
    },
    {
      "epoch": 0.906859530456615,
      "grad_norm": 2.4331199214454156,
      "learning_rate": 5.1425947722658e-07,
      "log_odds_chosen": 0.5214599370956421,
      "log_odds_ratio": -0.577587902545929,
      "logits/chosen": -0.76171875,
      "logits/rejected": -0.666308581829071,
      "logps/chosen": -0.6197265386581421,
      "logps/rejected": -0.939648449420929,
      "loss": 0.7758,
      "nll_loss": 0.7323242425918579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06195068359375,
      "rewards/margins": 0.031990814954042435,
      "rewards/rejected": -0.09393310546875,
      "step": 24200
    },
    {
      "epoch": 0.907234265799779,
      "grad_norm": 2.4539199513545413,
      "learning_rate": 5.141532581864661e-07,
      "log_odds_chosen": 0.5414794683456421,
      "log_odds_ratio": -0.5826171636581421,
      "logits/chosen": -0.7227538824081421,
      "logits/rejected": -0.634082019329071,
      "logps/chosen": -0.710253894329071,
      "logps/rejected": -1.0390625,
      "loss": 0.7858,
      "nll_loss": 0.723828136920929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07100830227136612,
      "rewards/margins": 0.032920073717832565,
      "rewards/rejected": -0.10393066704273224,
      "step": 24210
    },
    {
      "epoch": 0.9076090011429429,
      "grad_norm": 2.5012317844740077,
      "learning_rate": 5.140471049370253e-07,
      "log_odds_chosen": 0.5433593988418579,
      "log_odds_ratio": -0.5809570550918579,
      "logits/chosen": -0.7601562738418579,
      "logits/rejected": -0.668749988079071,
      "logps/chosen": -0.646289050579071,
      "logps/rejected": -0.973828136920929,
      "loss": 0.787,
      "nll_loss": 0.740234375,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06463623046875,
      "rewards/margins": 0.03266601637005806,
      "rewards/rejected": -0.09743652492761612,
      "step": 24220
    },
    {
      "epoch": 0.9079837364861066,
      "grad_norm": 2.1587416611774595,
      "learning_rate": 5.139410174103693e-07,
      "log_odds_chosen": 0.40618896484375,
      "log_odds_ratio": -0.6151367425918579,
      "logits/chosen": -0.71728515625,
      "logits/rejected": -0.628125011920929,
      "logps/chosen": -0.765429675579071,
      "logps/rejected": -1.020898461341858,
      "loss": 0.79,
      "nll_loss": 0.773242175579071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07655028998851776,
      "rewards/margins": 0.02562103234231472,
      "rewards/rejected": -0.10214843600988388,
      "step": 24230
    },
    {
      "epoch": 0.9083584718292705,
      "grad_norm": 2.428552584516777,
      "learning_rate": 5.138349955387079e-07,
      "log_odds_chosen": 0.522003173828125,
      "log_odds_ratio": -0.59228515625,
      "logits/chosen": -0.717578113079071,
      "logits/rejected": -0.6177734136581421,
      "logps/chosen": -0.721386730670929,
      "logps/rejected": -1.0681641101837158,
      "loss": 0.7875,
      "nll_loss": 0.792675793170929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07209472358226776,
      "rewards/margins": 0.03474273532629013,
      "rewards/rejected": -0.10683593899011612,
      "step": 24240
    },
    {
      "epoch": 0.9087332071724344,
      "grad_norm": 2.237336805015689,
      "learning_rate": 5.137290392543484e-07,
      "log_odds_chosen": 0.40492552518844604,
      "log_odds_ratio": -0.6190429925918579,
      "logits/chosen": -0.8187500238418579,
      "logits/rejected": -0.69091796875,
      "logps/chosen": -0.625683605670929,
      "logps/rejected": -0.869335949420929,
      "loss": 0.7826,
      "nll_loss": 0.721484363079071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06253661960363388,
      "rewards/margins": 0.0243377685546875,
      "rewards/rejected": -0.0869140625,
      "step": 24250
    },
    {
      "epoch": 0.9091079425155983,
      "grad_norm": 3.401146033580746,
      "learning_rate": 5.13623148489696e-07,
      "log_odds_chosen": 0.5600219964981079,
      "log_odds_ratio": -0.568164050579071,
      "logits/chosen": -0.7544921636581421,
      "logits/rejected": -0.6714843511581421,
      "logps/chosen": -0.6190429925918579,
      "logps/rejected": -0.9521484375,
      "loss": 0.8123,
      "nll_loss": 0.7259765863418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06192626804113388,
      "rewards/margins": 0.03335266187787056,
      "rewards/rejected": -0.09522704780101776,
      "step": 24260
    },
    {
      "epoch": 0.9094826778587622,
      "grad_norm": 2.635070106452537,
      "learning_rate": 5.135173231772532e-07,
      "log_odds_chosen": 0.47376710176467896,
      "log_odds_ratio": -0.5894531011581421,
      "logits/chosen": -0.7974609136581421,
      "logits/rejected": -0.6753906011581421,
      "logps/chosen": -0.62939453125,
      "logps/rejected": -0.906054675579071,
      "loss": 0.7941,
      "nll_loss": 0.7216796875,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06295166164636612,
      "rewards/margins": 0.02764129638671875,
      "rewards/rejected": -0.09066162258386612,
      "step": 24270
    },
    {
      "epoch": 0.9098574132019261,
      "grad_norm": 2.4174091929490746,
      "learning_rate": 5.134115632496199e-07,
      "log_odds_chosen": 0.3941894471645355,
      "log_odds_ratio": -0.6181640625,
      "logits/chosen": -0.7564452886581421,
      "logits/rejected": -0.656542956829071,
      "logps/chosen": -0.6953125,
      "logps/rejected": -0.9371093511581421,
      "loss": 0.785,
      "nll_loss": 0.783398449420929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06949462741613388,
      "rewards/margins": 0.02427215501666069,
      "rewards/rejected": -0.09371338039636612,
      "step": 24280
    },
    {
      "epoch": 0.91023214854509,
      "grad_norm": 2.7347969255514593,
      "learning_rate": 5.133058686394933e-07,
      "log_odds_chosen": 0.43046873807907104,
      "log_odds_ratio": -0.60400390625,
      "logits/chosen": -0.8184570074081421,
      "logits/rejected": -0.672656238079071,
      "logps/chosen": -0.6724609136581421,
      "logps/rejected": -0.951171875,
      "loss": 0.8059,
      "nll_loss": 0.753125011920929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06718750298023224,
      "rewards/margins": 0.02792816236615181,
      "rewards/rejected": -0.09516601264476776,
      "step": 24290
    },
    {
      "epoch": 0.910606883888254,
      "grad_norm": 2.363423298878791,
      "learning_rate": 5.132002392796672e-07,
      "log_odds_chosen": 0.46339112520217896,
      "log_odds_ratio": -0.618408203125,
      "logits/chosen": -0.760937511920929,
      "logits/rejected": -0.6434570550918579,
      "logps/chosen": -0.7099609375,
      "logps/rejected": -1.0109374523162842,
      "loss": 0.7994,
      "nll_loss": 0.7705078125,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07108154147863388,
      "rewards/margins": 0.03009643591940403,
      "rewards/rejected": -0.10107421875,
      "step": 24300
    },
    {
      "epoch": 0.9109816192314178,
      "grad_norm": 2.9312834387508806,
      "learning_rate": 5.130946751030328e-07,
      "log_odds_chosen": 0.541088879108429,
      "log_odds_ratio": -0.5777343511581421,
      "logits/chosen": -0.7749999761581421,
      "logits/rejected": -0.68115234375,
      "logps/chosen": -0.679492175579071,
      "logps/rejected": -1.036718726158142,
      "loss": 0.8046,
      "nll_loss": 0.7451171875,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06791992485523224,
      "rewards/margins": 0.0357818603515625,
      "rewards/rejected": -0.10366211086511612,
      "step": 24310
    },
    {
      "epoch": 0.9113563545745818,
      "grad_norm": 3.1427888203314716,
      "learning_rate": 5.129891760425771e-07,
      "log_odds_chosen": 0.501708984375,
      "log_odds_ratio": -0.5868164300918579,
      "logits/chosen": -0.738476574420929,
      "logits/rejected": -0.66845703125,
      "logps/chosen": -0.6390625238418579,
      "logps/rejected": -0.9339843988418579,
      "loss": 0.7834,
      "nll_loss": 0.7250000238418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06383056938648224,
      "rewards/margins": 0.02952270582318306,
      "rewards/rejected": -0.09340820461511612,
      "step": 24320
    },
    {
      "epoch": 0.9117310899177455,
      "grad_norm": 2.5409266679156555,
      "learning_rate": 5.128837420313838e-07,
      "log_odds_chosen": 0.5244140625,
      "log_odds_ratio": -0.567578136920929,
      "logits/chosen": -0.8306640386581421,
      "logits/rejected": -0.7149413824081421,
      "logps/chosen": -0.6197265386581421,
      "logps/rejected": -0.907031238079071,
      "loss": 0.804,
      "nll_loss": 0.6957031488418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06196289137005806,
      "rewards/margins": 0.028755951672792435,
      "rewards/rejected": -0.0906982421875,
      "step": 24330
    },
    {
      "epoch": 0.9121058252609094,
      "grad_norm": 2.6495518514152443,
      "learning_rate": 5.127783730026332e-07,
      "log_odds_chosen": 0.516247570514679,
      "log_odds_ratio": -0.588085949420929,
      "logits/chosen": -0.7875000238418579,
      "logits/rejected": -0.708789050579071,
      "logps/chosen": -0.68310546875,
      "logps/rejected": -0.9878906011581421,
      "loss": 0.8034,
      "nll_loss": 0.756542980670929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0682373046875,
      "rewards/margins": 0.030515288934111595,
      "rewards/rejected": -0.0987548828125,
      "step": 24340
    },
    {
      "epoch": 0.9124805606040733,
      "grad_norm": 3.1547968734290497,
      "learning_rate": 5.126730688896011e-07,
      "log_odds_chosen": 0.509875476360321,
      "log_odds_ratio": -0.584667980670929,
      "logits/chosen": -0.751171886920929,
      "logits/rejected": -0.6458984613418579,
      "logps/chosen": -0.6763671636581421,
      "logps/rejected": -0.9751952886581421,
      "loss": 0.7967,
      "nll_loss": 0.6910156011581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.067626953125,
      "rewards/margins": 0.02984314039349556,
      "rewards/rejected": -0.09746094048023224,
      "step": 24350
    },
    {
      "epoch": 0.9128552959472372,
      "grad_norm": 2.8350223744026013,
      "learning_rate": 5.125678296256597e-07,
      "log_odds_chosen": 0.33466798067092896,
      "log_odds_ratio": -0.6524413824081421,
      "logits/chosen": -0.7489258050918579,
      "logits/rejected": -0.701367199420929,
      "logps/chosen": -0.66015625,
      "logps/rejected": -0.8779296875,
      "loss": 0.8136,
      "nll_loss": 0.722851574420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06601562350988388,
      "rewards/margins": 0.02175598219037056,
      "rewards/rejected": -0.08774413913488388,
      "step": 24360
    },
    {
      "epoch": 0.9132300312904011,
      "grad_norm": 2.85831505354107,
      "learning_rate": 5.124626551442763e-07,
      "log_odds_chosen": 0.609667956829071,
      "log_odds_ratio": -0.5345703363418579,
      "logits/chosen": -0.744140625,
      "logits/rejected": -0.6861327886581421,
      "logps/chosen": -0.66015625,
      "logps/rejected": -1.0148437023162842,
      "loss": 0.7804,
      "nll_loss": 0.699511706829071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06605224311351776,
      "rewards/margins": 0.035411834716796875,
      "rewards/rejected": -0.1014404296875,
      "step": 24370
    },
    {
      "epoch": 0.913604766633565,
      "grad_norm": 2.850848511214052,
      "learning_rate": 5.123575453790144e-07,
      "log_odds_chosen": 0.505688488483429,
      "log_odds_ratio": -0.5702148675918579,
      "logits/chosen": -0.7847656011581421,
      "logits/rejected": -0.662304699420929,
      "logps/chosen": -0.654003918170929,
      "logps/rejected": -0.967578113079071,
      "loss": 0.8169,
      "nll_loss": 0.736523449420929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06540527194738388,
      "rewards/margins": 0.03140868991613388,
      "rewards/rejected": -0.0968017578125,
      "step": 24380
    },
    {
      "epoch": 0.913979501976729,
      "grad_norm": 2.6443460726242947,
      "learning_rate": 5.122525002635324e-07,
      "log_odds_chosen": 0.4921630918979645,
      "log_odds_ratio": -0.5972656011581421,
      "logits/chosen": -0.7618163824081421,
      "logits/rejected": -0.6499999761581421,
      "logps/chosen": -0.6419922113418579,
      "logps/rejected": -0.9482421875,
      "loss": 0.7839,
      "nll_loss": 0.697070300579071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06424560397863388,
      "rewards/margins": 0.03058776818215847,
      "rewards/rejected": -0.09479980170726776,
      "step": 24390
    },
    {
      "epoch": 0.9143542373198928,
      "grad_norm": 2.584302882063947,
      "learning_rate": 5.121475197315839e-07,
      "log_odds_chosen": 0.6128906011581421,
      "log_odds_ratio": -0.561328113079071,
      "logits/chosen": -0.709179699420929,
      "logits/rejected": -0.6533203125,
      "logps/chosen": -0.6390625238418579,
      "logps/rejected": -1.016210913658142,
      "loss": 0.7937,
      "nll_loss": 0.7542968988418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06389160454273224,
      "rewards/margins": 0.03770751878619194,
      "rewards/rejected": -0.1015625,
      "step": 24400
    },
    {
      "epoch": 0.9147289726630567,
      "grad_norm": 2.242387849505238,
      "learning_rate": 5.120426037170176e-07,
      "log_odds_chosen": 0.59429931640625,
      "log_odds_ratio": -0.5782226324081421,
      "logits/chosen": -0.798828125,
      "logits/rejected": -0.6937500238418579,
      "logps/chosen": -0.6861327886581421,
      "logps/rejected": -1.0803711414337158,
      "loss": 0.789,
      "nll_loss": 0.72509765625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06864013522863388,
      "rewards/margins": 0.03946838527917862,
      "rewards/rejected": -0.10821533203125,
      "step": 24410
    },
    {
      "epoch": 0.9151037080062207,
      "grad_norm": 3.1065164054205567,
      "learning_rate": 5.119377521537771e-07,
      "log_odds_chosen": 0.5813964605331421,
      "log_odds_ratio": -0.52685546875,
      "logits/chosen": -0.785937488079071,
      "logits/rejected": -0.641406238079071,
      "logps/chosen": -0.6382812261581421,
      "logps/rejected": -1.0046875476837158,
      "loss": 0.7798,
      "nll_loss": 0.695507824420929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06381835788488388,
      "rewards/margins": 0.0367279052734375,
      "rewards/rejected": -0.10053710639476776,
      "step": 24420
    },
    {
      "epoch": 0.9154784433493846,
      "grad_norm": 3.7776640883641996,
      "learning_rate": 5.118329649759004e-07,
      "log_odds_chosen": 0.725341796875,
      "log_odds_ratio": -0.52001953125,
      "logits/chosen": -0.7574218511581421,
      "logits/rejected": -0.6410156488418579,
      "logps/chosen": -0.6703125238418579,
      "logps/rejected": -1.142578125,
      "loss": 0.7981,
      "nll_loss": 0.674023449420929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0670166015625,
      "rewards/margins": 0.04725952073931694,
      "rewards/rejected": -0.11430664360523224,
      "step": 24430
    },
    {
      "epoch": 0.9158531786925483,
      "grad_norm": 2.799884020784389,
      "learning_rate": 5.117282421175202e-07,
      "log_odds_chosen": 0.46326905488967896,
      "log_odds_ratio": -0.6064453125,
      "logits/chosen": -0.744921863079071,
      "logits/rejected": -0.646289050579071,
      "logps/chosen": -0.644335925579071,
      "logps/rejected": -0.948046863079071,
      "loss": 0.7818,
      "nll_loss": 0.666699230670929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06441650539636612,
      "rewards/margins": 0.03033752366900444,
      "rewards/rejected": -0.09489746391773224,
      "step": 24440
    },
    {
      "epoch": 0.9162279140357122,
      "grad_norm": 2.5665758745479272,
      "learning_rate": 5.116235835128635e-07,
      "log_odds_chosen": 0.6070801019668579,
      "log_odds_ratio": -0.5484374761581421,
      "logits/chosen": -0.821093738079071,
      "logits/rejected": -0.6885741949081421,
      "logps/chosen": -0.680468738079071,
      "logps/rejected": -1.0603516101837158,
      "loss": 0.7915,
      "nll_loss": 0.7652343511581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06805419921875,
      "rewards/margins": 0.03795776516199112,
      "rewards/rejected": -0.10600586235523224,
      "step": 24450
    },
    {
      "epoch": 0.9166026493788761,
      "grad_norm": 2.8391863011021607,
      "learning_rate": 5.115189890962512e-07,
      "log_odds_chosen": 0.4971679747104645,
      "log_odds_ratio": -0.599414050579071,
      "logits/chosen": -0.771289050579071,
      "logits/rejected": -0.696484386920929,
      "logps/chosen": -0.678906261920929,
      "logps/rejected": -1.0089843273162842,
      "loss": 0.7741,
      "nll_loss": 0.7398437261581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06785888969898224,
      "rewards/margins": 0.03303069993853569,
      "rewards/rejected": -0.10085449367761612,
      "step": 24460
    },
    {
      "epoch": 0.91697738472204,
      "grad_norm": 2.798762592176477,
      "learning_rate": 5.114144588020982e-07,
      "log_odds_chosen": 0.53009033203125,
      "log_odds_ratio": -0.5780273675918579,
      "logits/chosen": -0.8148437738418579,
      "logits/rejected": -0.6922851800918579,
      "logps/chosen": -0.6753906011581421,
      "logps/rejected": -1.0076172351837158,
      "loss": 0.7665,
      "nll_loss": 0.7076171636581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06752929836511612,
      "rewards/margins": 0.03321228176355362,
      "rewards/rejected": -0.10076904296875,
      "step": 24470
    },
    {
      "epoch": 0.917352120065204,
      "grad_norm": 2.4792301352642276,
      "learning_rate": 5.113099925649136e-07,
      "log_odds_chosen": 0.5083984136581421,
      "log_odds_ratio": -0.567675769329071,
      "logits/chosen": -0.725781261920929,
      "logits/rejected": -0.6361328363418579,
      "logps/chosen": -0.6314452886581421,
      "logps/rejected": -0.939648449420929,
      "loss": 0.7881,
      "nll_loss": 0.7601562738418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06315918266773224,
      "rewards/margins": 0.03076782263815403,
      "rewards/rejected": -0.09396972507238388,
      "step": 24480
    },
    {
      "epoch": 0.9177268554083678,
      "grad_norm": 2.4609322964952045,
      "learning_rate": 5.112055903192996e-07,
      "log_odds_chosen": 0.716503918170929,
      "log_odds_ratio": -0.5179687738418579,
      "logits/chosen": -0.760546863079071,
      "logits/rejected": -0.66259765625,
      "logps/chosen": -0.636914074420929,
      "logps/rejected": -1.066992163658142,
      "loss": 0.7834,
      "nll_loss": 0.6937500238418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06373290717601776,
      "rewards/margins": 0.04305267333984375,
      "rewards/rejected": -0.1068115234375,
      "step": 24490
    },
    {
      "epoch": 0.9181015907515317,
      "grad_norm": 2.5181495211870404,
      "learning_rate": 5.111012519999519e-07,
      "log_odds_chosen": 0.5089477300643921,
      "log_odds_ratio": -0.5889648199081421,
      "logits/chosen": -0.7173827886581421,
      "logits/rejected": -0.6456054449081421,
      "logps/chosen": -0.6483398675918579,
      "logps/rejected": -0.9710937738418579,
      "loss": 0.7741,
      "nll_loss": 0.70263671875,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06484375149011612,
      "rewards/margins": 0.032290268689394,
      "rewards/rejected": -0.09709472954273224,
      "step": 24500
    },
    {
      "epoch": 0.9184763260946956,
      "grad_norm": 2.8501447171624745,
      "learning_rate": 5.109969775416598e-07,
      "log_odds_chosen": 0.655102550983429,
      "log_odds_ratio": -0.545703113079071,
      "logits/chosen": -0.7626953125,
      "logits/rejected": -0.61279296875,
      "logps/chosen": -0.6456054449081421,
      "logps/rejected": -1.072656273841858,
      "loss": 0.7839,
      "nll_loss": 0.7513672113418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06459961086511612,
      "rewards/margins": 0.04259796068072319,
      "rewards/rejected": -0.10718993842601776,
      "step": 24510
    },
    {
      "epoch": 0.9188510614378596,
      "grad_norm": 2.7990485365308584,
      "learning_rate": 5.108927668793053e-07,
      "log_odds_chosen": 0.537304699420929,
      "log_odds_ratio": -0.57958984375,
      "logits/chosen": -0.8033202886581421,
      "logits/rejected": -0.6815429925918579,
      "logps/chosen": -0.664843738079071,
      "logps/rejected": -1.0134766101837158,
      "loss": 0.7775,
      "nll_loss": 0.7222656011581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06650390475988388,
      "rewards/margins": 0.03487548977136612,
      "rewards/rejected": -0.10134277492761612,
      "step": 24520
    },
    {
      "epoch": 0.9192257967810235,
      "grad_norm": 2.9616587817288447,
      "learning_rate": 5.107886199478635e-07,
      "log_odds_chosen": 0.40272217988967896,
      "log_odds_ratio": -0.6641601324081421,
      "logits/chosen": -0.792187511920929,
      "logits/rejected": -0.65380859375,
      "logps/chosen": -0.7247070074081421,
      "logps/rejected": -0.997851550579071,
      "loss": 0.7873,
      "nll_loss": 0.7025390863418579,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07248535007238388,
      "rewards/margins": 0.02725982666015625,
      "rewards/rejected": -0.09969482570886612,
      "step": 24530
    },
    {
      "epoch": 0.9196005321241872,
      "grad_norm": 2.342402024237575,
      "learning_rate": 5.106845366824023e-07,
      "log_odds_chosen": 0.593676745891571,
      "log_odds_ratio": -0.5599609613418579,
      "logits/chosen": -0.8285156488418579,
      "logits/rejected": -0.7076171636581421,
      "logps/chosen": -0.64892578125,
      "logps/rejected": -1.001562476158142,
      "loss": 0.7803,
      "nll_loss": 0.7025390863418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06495361030101776,
      "rewards/margins": 0.03508453443646431,
      "rewards/rejected": -0.10009765625,
      "step": 24540
    },
    {
      "epoch": 0.9199752674673511,
      "grad_norm": 2.3621063841558056,
      "learning_rate": 5.105805170180822e-07,
      "log_odds_chosen": 0.5212768316268921,
      "log_odds_ratio": -0.5855957269668579,
      "logits/chosen": -0.789843738079071,
      "logits/rejected": -0.6781250238418579,
      "logps/chosen": -0.6205078363418579,
      "logps/rejected": -0.9634765386581421,
      "loss": 0.7822,
      "nll_loss": 0.6724609136581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06210937350988388,
      "rewards/margins": 0.03426513820886612,
      "rewards/rejected": -0.0963134765625,
      "step": 24550
    },
    {
      "epoch": 0.920350002810515,
      "grad_norm": 2.653736524936608,
      "learning_rate": 5.104765608901559e-07,
      "log_odds_chosen": 0.4970703125,
      "log_odds_ratio": -0.597851574420929,
      "logits/chosen": -0.776562511920929,
      "logits/rejected": -0.6947265863418579,
      "logps/chosen": -0.6958984136581421,
      "logps/rejected": -1.014062523841858,
      "loss": 0.7955,
      "nll_loss": 0.759960949420929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06959228217601776,
      "rewards/margins": 0.03180694580078125,
      "rewards/rejected": -0.1014404296875,
      "step": 24560
    },
    {
      "epoch": 0.920724738153679,
      "grad_norm": 2.657043704522225,
      "learning_rate": 5.103726682339685e-07,
      "log_odds_chosen": 0.59765625,
      "log_odds_ratio": -0.5615234375,
      "logits/chosen": -0.7681640386581421,
      "logits/rejected": -0.6895507574081421,
      "logps/chosen": -0.6494140625,
      "logps/rejected": -1.0021483898162842,
      "loss": 0.777,
      "nll_loss": 0.731640636920929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06495361030101776,
      "rewards/margins": 0.035247802734375,
      "rewards/rejected": -0.1002197265625,
      "step": 24570
    },
    {
      "epoch": 0.9210994734968428,
      "grad_norm": 2.5783527638683275,
      "learning_rate": 5.102688389849571e-07,
      "log_odds_chosen": 0.7270873785018921,
      "log_odds_ratio": -0.49980467557907104,
      "logits/chosen": -0.798046886920929,
      "logits/rejected": -0.6537109613418579,
      "logps/chosen": -0.691210925579071,
      "logps/rejected": -1.1589844226837158,
      "loss": 0.7994,
      "nll_loss": 0.745312511920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.069091796875,
      "rewards/margins": 0.0467987060546875,
      "rewards/rejected": -0.11589355766773224,
      "step": 24580
    },
    {
      "epoch": 0.9214742088400067,
      "grad_norm": 2.596987649639151,
      "learning_rate": 5.101650730786509e-07,
      "log_odds_chosen": 0.51708984375,
      "log_odds_ratio": -0.581103503704071,
      "logits/chosen": -0.764453113079071,
      "logits/rejected": -0.6507812738418579,
      "logps/chosen": -0.655078113079071,
      "logps/rejected": -0.974902331829071,
      "loss": 0.781,
      "nll_loss": 0.7162109613418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06550292670726776,
      "rewards/margins": 0.03193511813879013,
      "rewards/rejected": -0.09742431342601776,
      "step": 24590
    },
    {
      "epoch": 0.9218489441831706,
      "grad_norm": 2.7117871382677685,
      "learning_rate": 5.100613704506706e-07,
      "log_odds_chosen": 0.4956298768520355,
      "log_odds_ratio": -0.597949206829071,
      "logits/chosen": -0.7816406488418579,
      "logits/rejected": -0.693359375,
      "logps/chosen": -0.7349609136581421,
      "logps/rejected": -1.0744140148162842,
      "loss": 0.7943,
      "nll_loss": 0.731249988079071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07347412407398224,
      "rewards/margins": 0.03398437425494194,
      "rewards/rejected": -0.10747070610523224,
      "step": 24600
    },
    {
      "epoch": 0.9222236795263345,
      "grad_norm": 2.7880238587088506,
      "learning_rate": 5.099577310367286e-07,
      "log_odds_chosen": 0.39027100801467896,
      "log_odds_ratio": -0.615917980670929,
      "logits/chosen": -0.7461913824081421,
      "logits/rejected": -0.6641601324081421,
      "logps/chosen": -0.623242199420929,
      "logps/rejected": -0.852734386920929,
      "loss": 0.7808,
      "nll_loss": 0.684374988079071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06228027492761612,
      "rewards/margins": 0.02297668531537056,
      "rewards/rejected": -0.08530273288488388,
      "step": 24610
    },
    {
      "epoch": 0.9225984148694985,
      "grad_norm": 2.6572748674249995,
      "learning_rate": 5.098541547726285e-07,
      "log_odds_chosen": 0.591796875,
      "log_odds_ratio": -0.581005871295929,
      "logits/chosen": -0.756640613079071,
      "logits/rejected": -0.689453125,
      "logps/chosen": -0.6451171636581421,
      "logps/rejected": -1.042382836341858,
      "loss": 0.7697,
      "nll_loss": 0.71435546875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06455077975988388,
      "rewards/margins": 0.03969268873333931,
      "rewards/rejected": -0.10427246242761612,
      "step": 24620
    },
    {
      "epoch": 0.9229731502126624,
      "grad_norm": 2.482663378259853,
      "learning_rate": 5.097506415942655e-07,
      "log_odds_chosen": 0.5191894769668579,
      "log_odds_ratio": -0.562304675579071,
      "logits/chosen": -0.7826172113418579,
      "logits/rejected": -0.653027355670929,
      "logps/chosen": -0.693554699420929,
      "logps/rejected": -1.0068359375,
      "loss": 0.7797,
      "nll_loss": 0.7357422113418579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06939697265625,
      "rewards/margins": 0.03129119798541069,
      "rewards/rejected": -0.1007080078125,
      "step": 24630
    },
    {
      "epoch": 0.9233478855558263,
      "grad_norm": 2.3185429753045965,
      "learning_rate": 5.096471914376254e-07,
      "log_odds_chosen": 0.6090332269668579,
      "log_odds_ratio": -0.546142578125,
      "logits/chosen": -0.7764648199081421,
      "logits/rejected": -0.691113293170929,
      "logps/chosen": -0.6167968511581421,
      "logps/rejected": -0.972851574420929,
      "loss": 0.787,
      "nll_loss": 0.6908203363418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06162109225988388,
      "rewards/margins": 0.03563842922449112,
      "rewards/rejected": -0.09726562350988388,
      "step": 24640
    },
    {
      "epoch": 0.92372262089899,
      "grad_norm": 2.4911256269067494,
      "learning_rate": 5.095438042387856e-07,
      "log_odds_chosen": 0.5328735113143921,
      "log_odds_ratio": -0.5873047113418579,
      "logits/chosen": -0.8115234375,
      "logits/rejected": -0.688281238079071,
      "logps/chosen": -0.683398425579071,
      "logps/rejected": -1.004296898841858,
      "loss": 0.8123,
      "nll_loss": 0.7232421636581421,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06833495944738388,
      "rewards/margins": 0.032108306884765625,
      "rewards/rejected": -0.10045166313648224,
      "step": 24650
    },
    {
      "epoch": 0.9240973562421539,
      "grad_norm": 2.421408438861924,
      "learning_rate": 5.094404799339134e-07,
      "log_odds_chosen": 0.4942871034145355,
      "log_odds_ratio": -0.606738269329071,
      "logits/chosen": -0.7562500238418579,
      "logits/rejected": -0.665820300579071,
      "logps/chosen": -0.655468761920929,
      "logps/rejected": -0.988085925579071,
      "loss": 0.7909,
      "nll_loss": 0.7261718511581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0655517578125,
      "rewards/margins": 0.03327331691980362,
      "rewards/rejected": -0.09877929836511612,
      "step": 24660
    },
    {
      "epoch": 0.9244720915853178,
      "grad_norm": 2.7494920944702237,
      "learning_rate": 5.093372184592671e-07,
      "log_odds_chosen": 0.5464721918106079,
      "log_odds_ratio": -0.5869140625,
      "logits/chosen": -0.7646484375,
      "logits/rejected": -0.6587890386581421,
      "logps/chosen": -0.6641601324081421,
      "logps/rejected": -1.0031249523162842,
      "loss": 0.7845,
      "nll_loss": 0.7173827886581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06645508110523224,
      "rewards/margins": 0.03393859788775444,
      "rewards/rejected": -0.100341796875,
      "step": 24670
    },
    {
      "epoch": 0.9248468269284817,
      "grad_norm": 2.5692245361660295,
      "learning_rate": 5.092340197511956e-07,
      "log_odds_chosen": 0.5297485589981079,
      "log_odds_ratio": -0.548046886920929,
      "logits/chosen": -0.8232421875,
      "logits/rejected": -0.712695300579071,
      "logps/chosen": -0.6436523199081421,
      "logps/rejected": -0.9476562738418579,
      "loss": 0.7655,
      "nll_loss": 0.7177734375,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06431885063648224,
      "rewards/margins": 0.030373383313417435,
      "rewards/rejected": -0.09478759765625,
      "step": 24680
    },
    {
      "epoch": 0.9252215622716456,
      "grad_norm": 2.669873274042771,
      "learning_rate": 5.091308837461377e-07,
      "log_odds_chosen": 0.630175769329071,
      "log_odds_ratio": -0.553906261920929,
      "logits/chosen": -0.7823241949081421,
      "logits/rejected": -0.673828125,
      "logps/chosen": -0.6402343511581421,
      "logps/rejected": -1.018945336341858,
      "loss": 0.7667,
      "nll_loss": 0.6646484136581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06397704780101776,
      "rewards/margins": 0.037810515612363815,
      "rewards/rejected": -0.10187987983226776,
      "step": 24690
    },
    {
      "epoch": 0.9255962976148095,
      "grad_norm": 2.4596383459901783,
      "learning_rate": 5.090278103806222e-07,
      "log_odds_chosen": 0.4843505918979645,
      "log_odds_ratio": -0.615527331829071,
      "logits/chosen": -0.7333984375,
      "logits/rejected": -0.675488293170929,
      "logps/chosen": -0.656542956829071,
      "logps/rejected": -0.929882824420929,
      "loss": 0.8007,
      "nll_loss": 0.748828113079071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06563720852136612,
      "rewards/margins": 0.02736968919634819,
      "rewards/rejected": -0.09304199367761612,
      "step": 24700
    },
    {
      "epoch": 0.9259710329579735,
      "grad_norm": 2.429396691437173,
      "learning_rate": 5.089247995912683e-07,
      "log_odds_chosen": 0.6368408203125,
      "log_odds_ratio": -0.516796886920929,
      "logits/chosen": -0.70458984375,
      "logits/rejected": -0.6044921875,
      "logps/chosen": -0.603515625,
      "logps/rejected": -0.9609375,
      "loss": 0.7878,
      "nll_loss": 0.684374988079071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06032714992761612,
      "rewards/margins": 0.03581085056066513,
      "rewards/rejected": -0.09608153998851776,
      "step": 24710
    },
    {
      "epoch": 0.9263457683011374,
      "grad_norm": 2.912208686849794,
      "learning_rate": 5.088218513147844e-07,
      "log_odds_chosen": 0.7015136480331421,
      "log_odds_ratio": -0.5331054925918579,
      "logits/chosen": -0.794140636920929,
      "logits/rejected": -0.6644531488418579,
      "logps/chosen": -0.67041015625,
      "logps/rejected": -1.1228516101837158,
      "loss": 0.7841,
      "nll_loss": 0.7572265863418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06707763671875,
      "rewards/margins": 0.04519348219037056,
      "rewards/rejected": -0.11223144829273224,
      "step": 24720
    },
    {
      "epoch": 0.9267205036443013,
      "grad_norm": 2.9229293438410493,
      "learning_rate": 5.087189654879688e-07,
      "log_odds_chosen": 0.44941407442092896,
      "log_odds_ratio": -0.605273425579071,
      "logits/chosen": -0.7623046636581421,
      "logits/rejected": -0.6812499761581421,
      "logps/chosen": -0.6747070550918579,
      "logps/rejected": -0.9515625238418579,
      "loss": 0.8009,
      "nll_loss": 0.730664074420929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06748046725988388,
      "rewards/margins": 0.02770080603659153,
      "rewards/rejected": -0.09517822414636612,
      "step": 24730
    },
    {
      "epoch": 0.9270952389874652,
      "grad_norm": 2.749927504665628,
      "learning_rate": 5.086161420477092e-07,
      "log_odds_chosen": 0.513378918170929,
      "log_odds_ratio": -0.571484386920929,
      "logits/chosen": -0.7734375,
      "logits/rejected": -0.6705077886581421,
      "logps/chosen": -0.6722656488418579,
      "logps/rejected": -1.000585913658142,
      "loss": 0.7881,
      "nll_loss": 0.7611328363418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06724853813648224,
      "rewards/margins": 0.032831571996212006,
      "rewards/rejected": -0.10009765625,
      "step": 24740
    },
    {
      "epoch": 0.9274699743306289,
      "grad_norm": 4.0318113808813045,
      "learning_rate": 5.085133809309825e-07,
      "log_odds_chosen": 0.582958996295929,
      "log_odds_ratio": -0.534472644329071,
      "logits/chosen": -0.793749988079071,
      "logits/rejected": -0.6675781011581421,
      "logps/chosen": -0.663867175579071,
      "logps/rejected": -1.001367211341858,
      "loss": 0.7893,
      "nll_loss": 0.718554675579071,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06636963039636612,
      "rewards/margins": 0.033811189234256744,
      "rewards/rejected": -0.1002197265625,
      "step": 24750
    },
    {
      "epoch": 0.9278447096737928,
      "grad_norm": 2.6933204353724385,
      "learning_rate": 5.084106820748547e-07,
      "log_odds_chosen": 0.503100574016571,
      "log_odds_ratio": -0.599609375,
      "logits/chosen": -0.772265613079071,
      "logits/rejected": -0.655468761920929,
      "logps/chosen": -0.64453125,
      "logps/rejected": -0.9569336175918579,
      "loss": 0.7965,
      "nll_loss": 0.689160168170929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06451416015625,
      "rewards/margins": 0.031190108507871628,
      "rewards/rejected": -0.09572754055261612,
      "step": 24760
    },
    {
      "epoch": 0.9282194450169567,
      "grad_norm": 3.5250995893394514,
      "learning_rate": 5.083080454164808e-07,
      "log_odds_chosen": 0.4500732421875,
      "log_odds_ratio": -0.6255859136581421,
      "logits/chosen": -0.7757812738418579,
      "logits/rejected": -0.665332019329071,
      "logps/chosen": -0.704296886920929,
      "logps/rejected": -0.9869140386581421,
      "loss": 0.7893,
      "nll_loss": 0.7603515386581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07042236626148224,
      "rewards/margins": 0.02818756178021431,
      "rewards/rejected": -0.09860839694738388,
      "step": 24770
    },
    {
      "epoch": 0.9285941803601206,
      "grad_norm": 2.6315129784485096,
      "learning_rate": 5.082054708931043e-07,
      "log_odds_chosen": 0.45045167207717896,
      "log_odds_ratio": -0.5977538824081421,
      "logits/chosen": -0.7798827886581421,
      "logits/rejected": -0.692578136920929,
      "logps/chosen": -0.6734374761581421,
      "logps/rejected": -0.959179699420929,
      "loss": 0.7915,
      "nll_loss": 0.719921886920929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06735839694738388,
      "rewards/margins": 0.0286102294921875,
      "rewards/rejected": -0.09593506157398224,
      "step": 24780
    },
    {
      "epoch": 0.9289689157032845,
      "grad_norm": 2.7131979238997497,
      "learning_rate": 5.081029584420579e-07,
      "log_odds_chosen": 0.531603991985321,
      "log_odds_ratio": -0.585156261920929,
      "logits/chosen": -0.777539074420929,
      "logits/rejected": -0.6773437261581421,
      "logps/chosen": -0.677929699420929,
      "logps/rejected": -1.016015648841858,
      "loss": 0.7936,
      "nll_loss": 0.7347656488418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06785888969898224,
      "rewards/margins": 0.0337066650390625,
      "rewards/rejected": -0.10152588039636612,
      "step": 24790
    },
    {
      "epoch": 0.9293436510464484,
      "grad_norm": 3.09312327623164,
      "learning_rate": 5.08000508000762e-07,
      "log_odds_chosen": 0.6636596918106079,
      "log_odds_ratio": -0.525585949420929,
      "logits/chosen": -0.7352539300918579,
      "logits/rejected": -0.660351574420929,
      "logps/chosen": -0.6302734613418579,
      "logps/rejected": -1.0441405773162842,
      "loss": 0.7774,
      "nll_loss": 0.657910168170929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06303711235523224,
      "rewards/margins": 0.041353605687618256,
      "rewards/rejected": -0.10441894829273224,
      "step": 24800
    },
    {
      "epoch": 0.9297183863896124,
      "grad_norm": 2.5909448921775113,
      "learning_rate": 5.078981195067258e-07,
      "log_odds_chosen": 0.37456053495407104,
      "log_odds_ratio": -0.630175769329071,
      "logits/chosen": -0.827929675579071,
      "logits/rejected": -0.7035156488418579,
      "logps/chosen": -0.64453125,
      "logps/rejected": -0.872265636920929,
      "loss": 0.7782,
      "nll_loss": 0.721386730670929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06444092094898224,
      "rewards/margins": 0.022853851318359375,
      "rewards/rejected": -0.08735351264476776,
      "step": 24810
    },
    {
      "epoch": 0.9300931217327763,
      "grad_norm": 2.8677494299645163,
      "learning_rate": 5.077957928975466e-07,
      "log_odds_chosen": 0.4728637635707855,
      "log_odds_ratio": -0.603320300579071,
      "logits/chosen": -0.7193359136581421,
      "logits/rejected": -0.6373046636581421,
      "logps/chosen": -0.6802734136581421,
      "logps/rejected": -0.968945324420929,
      "loss": 0.789,
      "nll_loss": 0.751171886920929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06802978366613388,
      "rewards/margins": 0.02890625037252903,
      "rewards/rejected": -0.096923828125,
      "step": 24820
    },
    {
      "epoch": 0.9304678570759402,
      "grad_norm": 2.3452820381664723,
      "learning_rate": 5.076935281109094e-07,
      "log_odds_chosen": 0.4894042909145355,
      "log_odds_ratio": -0.6109374761581421,
      "logits/chosen": -0.787304699420929,
      "logits/rejected": -0.697460949420929,
      "logps/chosen": -0.680859386920929,
      "logps/rejected": -1.0154297351837158,
      "loss": 0.7978,
      "nll_loss": 0.7533203363418579,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06805419921875,
      "rewards/margins": 0.03352203220129013,
      "rewards/rejected": -0.10162353515625,
      "step": 24830
    },
    {
      "epoch": 0.930842592419104,
      "grad_norm": 3.1439292350867123,
      "learning_rate": 5.075913250845874e-07,
      "log_odds_chosen": 0.41168212890625,
      "log_odds_ratio": -0.6163085699081421,
      "logits/chosen": -0.821093738079071,
      "logits/rejected": -0.709765613079071,
      "logps/chosen": -0.698925793170929,
      "logps/rejected": -0.942578136920929,
      "loss": 0.7939,
      "nll_loss": 0.733593761920929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06993408501148224,
      "rewards/margins": 0.02434234693646431,
      "rewards/rejected": -0.09428711235523224,
      "step": 24840
    },
    {
      "epoch": 0.9312173277622678,
      "grad_norm": 3.1412633169887494,
      "learning_rate": 5.074891837564409e-07,
      "log_odds_chosen": 0.574694812297821,
      "log_odds_ratio": -0.559863269329071,
      "logits/chosen": -0.801562488079071,
      "logits/rejected": -0.710742175579071,
      "logps/chosen": -0.686328113079071,
      "logps/rejected": -1.0685546398162842,
      "loss": 0.7959,
      "nll_loss": 0.7416015863418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06868896633386612,
      "rewards/margins": 0.03818969801068306,
      "rewards/rejected": -0.10683593899011612,
      "step": 24850
    },
    {
      "epoch": 0.9315920631054317,
      "grad_norm": 2.7023979670102998,
      "learning_rate": 5.073871040644184e-07,
      "log_odds_chosen": 0.584667980670929,
      "log_odds_ratio": -0.5728515386581421,
      "logits/chosen": -0.7548828125,
      "logits/rejected": -0.6109374761581421,
      "logps/chosen": -0.663281261920929,
      "logps/rejected": -1.0427734851837158,
      "loss": 0.7857,
      "nll_loss": 0.71484375,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06636963039636612,
      "rewards/margins": 0.03790893405675888,
      "rewards/rejected": -0.10429687798023224,
      "step": 24860
    },
    {
      "epoch": 0.9319667984485956,
      "grad_norm": 2.6813768498724535,
      "learning_rate": 5.072850859465551e-07,
      "log_odds_chosen": 0.6070190668106079,
      "log_odds_ratio": -0.544140636920929,
      "logits/chosen": -0.754687488079071,
      "logits/rejected": -0.643750011920929,
      "logps/chosen": -0.608203113079071,
      "logps/rejected": -0.963671863079071,
      "loss": 0.8047,
      "nll_loss": 0.7060546875,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06081543117761612,
      "rewards/margins": 0.03553466871380806,
      "rewards/rejected": -0.09636230766773224,
      "step": 24870
    },
    {
      "epoch": 0.9323415337917595,
      "grad_norm": 2.5986530730238266,
      "learning_rate": 5.071831293409737e-07,
      "log_odds_chosen": 0.39082032442092896,
      "log_odds_ratio": -0.6329101324081421,
      "logits/chosen": -0.743945300579071,
      "logits/rejected": -0.6407226324081421,
      "logps/chosen": -0.66845703125,
      "logps/rejected": -0.893750011920929,
      "loss": 0.8089,
      "nll_loss": 0.723828136920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06689453125,
      "rewards/margins": 0.02251892164349556,
      "rewards/rejected": -0.08942870795726776,
      "step": 24880
    },
    {
      "epoch": 0.9327162691349234,
      "grad_norm": 2.4977270367992985,
      "learning_rate": 5.07081234185884e-07,
      "log_odds_chosen": 0.45122069120407104,
      "log_odds_ratio": -0.626269519329071,
      "logits/chosen": -0.767578125,
      "logits/rejected": -0.667187511920929,
      "logps/chosen": -0.665332019329071,
      "logps/rejected": -0.96484375,
      "loss": 0.7805,
      "nll_loss": 0.7250000238418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06656493991613388,
      "rewards/margins": 0.02985992468893528,
      "rewards/rejected": -0.09632568061351776,
      "step": 24890
    },
    {
      "epoch": 0.9330910044780873,
      "grad_norm": 3.109578704848666,
      "learning_rate": 5.069794004195823e-07,
      "log_odds_chosen": 0.60321044921875,
      "log_odds_ratio": -0.5785156488418579,
      "logits/chosen": -0.7650390863418579,
      "logits/rejected": -0.6597656011581421,
      "logps/chosen": -0.6348632574081421,
      "logps/rejected": -1.0314452648162842,
      "loss": 0.7717,
      "nll_loss": 0.695019543170929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06354980170726776,
      "rewards/margins": 0.03966064378619194,
      "rewards/rejected": -0.10318603366613388,
      "step": 24900
    },
    {
      "epoch": 0.9334657398212513,
      "grad_norm": 2.3336331009328934,
      "learning_rate": 5.06877627980452e-07,
      "log_odds_chosen": 0.6214355230331421,
      "log_odds_ratio": -0.56494140625,
      "logits/chosen": -0.763671875,
      "logits/rejected": -0.6498047113418579,
      "logps/chosen": -0.6734374761581421,
      "logps/rejected": -1.0750000476837158,
      "loss": 0.7752,
      "nll_loss": 0.6822265386581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06739501655101776,
      "rewards/margins": 0.04015808179974556,
      "rewards/rejected": -0.10758056491613388,
      "step": 24910
    },
    {
      "epoch": 0.9338404751644152,
      "grad_norm": 2.6739839599026487,
      "learning_rate": 5.067759168069628e-07,
      "log_odds_chosen": 0.524487316608429,
      "log_odds_ratio": -0.5936034917831421,
      "logits/chosen": -0.757617175579071,
      "logits/rejected": -0.654980480670929,
      "logps/chosen": -0.665820300579071,
      "logps/rejected": -0.9791015386581421,
      "loss": 0.7769,
      "nll_loss": 0.6968749761581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06657715141773224,
      "rewards/margins": 0.03131561353802681,
      "rewards/rejected": -0.09792480617761612,
      "step": 24920
    },
    {
      "epoch": 0.934215210507579,
      "grad_norm": 2.6755759558392977,
      "learning_rate": 5.066742668376709e-07,
      "log_odds_chosen": 0.4552246034145355,
      "log_odds_ratio": -0.6153320074081421,
      "logits/chosen": -0.772265613079071,
      "logits/rejected": -0.6600586175918579,
      "logps/chosen": -0.64013671875,
      "logps/rejected": -0.9349609613418579,
      "loss": 0.768,
      "nll_loss": 0.6982421875,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06401367485523224,
      "rewards/margins": 0.02946777269244194,
      "rewards/rejected": -0.093505859375,
      "step": 24930
    },
    {
      "epoch": 0.934589945850743,
      "grad_norm": 2.4317936313915895,
      "learning_rate": 5.065726780112187e-07,
      "log_odds_chosen": 0.5289551019668579,
      "log_odds_ratio": -0.587890625,
      "logits/chosen": -0.767382800579071,
      "logits/rejected": -0.665332019329071,
      "logps/chosen": -0.6539062261581421,
      "logps/rejected": -0.9791015386581421,
      "loss": 0.8074,
      "nll_loss": 0.748046875,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06536865234375,
      "rewards/margins": 0.03258972242474556,
      "rewards/rejected": -0.09794922173023224,
      "step": 24940
    },
    {
      "epoch": 0.9349646811939069,
      "grad_norm": 2.652026736404324,
      "learning_rate": 5.064711502663347e-07,
      "log_odds_chosen": 0.439697265625,
      "log_odds_ratio": -0.61181640625,
      "logits/chosen": -0.7789062261581421,
      "logits/rejected": -0.6751953363418579,
      "logps/chosen": -0.637402355670929,
      "logps/rejected": -0.906445324420929,
      "loss": 0.7619,
      "nll_loss": 0.7115234136581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06375732272863388,
      "rewards/margins": 0.02687225304543972,
      "rewards/rejected": -0.090576171875,
      "step": 24950
    },
    {
      "epoch": 0.9353394165370706,
      "grad_norm": 2.2553932803386005,
      "learning_rate": 5.063696835418333e-07,
      "log_odds_chosen": 0.635180652141571,
      "log_odds_ratio": -0.5580078363418579,
      "logits/chosen": -0.783007800579071,
      "logits/rejected": -0.7025390863418579,
      "logps/chosen": -0.66845703125,
      "logps/rejected": -1.051171898841858,
      "loss": 0.802,
      "nll_loss": 0.847460925579071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06683349609375,
      "rewards/margins": 0.03832549974322319,
      "rewards/rejected": -0.1051025390625,
      "step": 24960
    },
    {
      "epoch": 0.9357141518802345,
      "grad_norm": 2.476725728147945,
      "learning_rate": 5.062682777766146e-07,
      "log_odds_chosen": 0.513623058795929,
      "log_odds_ratio": -0.58740234375,
      "logits/chosen": -0.739453136920929,
      "logits/rejected": -0.6666015386581421,
      "logps/chosen": -0.669726550579071,
      "logps/rejected": -1.0011718273162842,
      "loss": 0.7896,
      "nll_loss": 0.71826171875,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06691894680261612,
      "rewards/margins": 0.03321991115808487,
      "rewards/rejected": -0.10009765625,
      "step": 24970
    },
    {
      "epoch": 0.9360888872233984,
      "grad_norm": 2.565539239121757,
      "learning_rate": 5.061669329096646e-07,
      "log_odds_chosen": 0.501660168170929,
      "log_odds_ratio": -0.59423828125,
      "logits/chosen": -0.787890613079071,
      "logits/rejected": -0.7025390863418579,
      "logps/chosen": -0.6695312261581421,
      "logps/rejected": -1.0011718273162842,
      "loss": 0.7882,
      "nll_loss": 0.705273449420929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06690673530101776,
      "rewards/margins": 0.03325042873620987,
      "rewards/rejected": -0.10007324069738388,
      "step": 24980
    },
    {
      "epoch": 0.9364636225665623,
      "grad_norm": 2.664732709119137,
      "learning_rate": 5.060656488800544e-07,
      "log_odds_chosen": 0.49071043729782104,
      "log_odds_ratio": -0.5810546875,
      "logits/chosen": -0.7386718988418579,
      "logits/rejected": -0.681445300579071,
      "logps/chosen": -0.676562488079071,
      "logps/rejected": -0.9755859375,
      "loss": 0.776,
      "nll_loss": 0.690625011920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06766357272863388,
      "rewards/margins": 0.029904937371611595,
      "rewards/rejected": -0.09757079929113388,
      "step": 24990
    },
    {
      "epoch": 0.9368383579097262,
      "grad_norm": 2.4823943445774446,
      "learning_rate": 5.059644256269407e-07,
      "log_odds_chosen": 0.72271728515625,
      "log_odds_ratio": -0.5333007574081421,
      "logits/chosen": -0.7515624761581421,
      "logits/rejected": -0.637890636920929,
      "logps/chosen": -0.64990234375,
      "logps/rejected": -1.1318359375,
      "loss": 0.7809,
      "nll_loss": 0.7132812738418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06496582180261612,
      "rewards/margins": 0.048187255859375,
      "rewards/rejected": -0.11318359524011612,
      "step": 25000
    },
    {
      "epoch": 0.9368383579097262,
      "eval_log_odds_chosen": 0.41629868745803833,
      "eval_log_odds_ratio": -0.6274687051773071,
      "eval_logits/chosen": -0.7034713625907898,
      "eval_logits/rejected": -0.6044302582740784,
      "eval_logps/chosen": -0.7482655048370361,
      "eval_logps/rejected": -1.0310636758804321,
      "eval_loss": 1.130112886428833,
      "eval_nll_loss": 1.0701721906661987,
      "eval_rewards/accuracies": 0.5954146981239319,
      "eval_rewards/chosen": -0.07482869178056717,
      "eval_rewards/margins": 0.028280578553676605,
      "eval_rewards/rejected": -0.10311097651720047,
      "eval_runtime": 2437.8108,
      "eval_samples_per_second": 77.865,
      "eval_steps_per_second": 1.217,
      "step": 25000
    },
    {
      "epoch": 0.9372130932528902,
      "grad_norm": 3.304250964478706,
      "learning_rate": 5.058632630895651e-07,
      "log_odds_chosen": 0.5103515386581421,
      "log_odds_ratio": -0.5752929449081421,
      "logits/chosen": -0.79638671875,
      "logits/rejected": -0.653613269329071,
      "logps/chosen": -0.6558593511581421,
      "logps/rejected": -0.9466797113418579,
      "loss": 0.7898,
      "nll_loss": 0.7513672113418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06556396186351776,
      "rewards/margins": 0.029109954833984375,
      "rewards/rejected": -0.09467773139476776,
      "step": 25010
    },
    {
      "epoch": 0.937587828596054,
      "grad_norm": 2.6990354605180666,
      "learning_rate": 5.057621612072543e-07,
      "log_odds_chosen": 0.5576171875,
      "log_odds_ratio": -0.557421863079071,
      "logits/chosen": -0.789257824420929,
      "logits/rejected": -0.7138671875,
      "logps/chosen": -0.682812511920929,
      "logps/rejected": -1.0333983898162842,
      "loss": 0.7959,
      "nll_loss": 0.72314453125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06822510063648224,
      "rewards/margins": 0.03498535230755806,
      "rewards/rejected": -0.10324706882238388,
      "step": 25020
    },
    {
      "epoch": 0.937962563939218,
      "grad_norm": 2.5726688535583544,
      "learning_rate": 5.0566111991942e-07,
      "log_odds_chosen": 0.4580444395542145,
      "log_odds_ratio": -0.6029297113418579,
      "logits/chosen": -0.745898425579071,
      "logits/rejected": -0.6368163824081421,
      "logps/chosen": -0.686230480670929,
      "logps/rejected": -0.9765625,
      "loss": 0.7892,
      "nll_loss": 0.7250000238418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.068603515625,
      "rewards/margins": 0.029108429327607155,
      "rewards/rejected": -0.09765625,
      "step": 25030
    },
    {
      "epoch": 0.9383372992823819,
      "grad_norm": 2.3811110707657392,
      "learning_rate": 5.055601391655586e-07,
      "log_odds_chosen": 0.4923461973667145,
      "log_odds_ratio": -0.606640636920929,
      "logits/chosen": -0.750781238079071,
      "logits/rejected": -0.6463867425918579,
      "logps/chosen": -0.6898437738418579,
      "logps/rejected": -1.0244140625,
      "loss": 0.7956,
      "nll_loss": 0.728515625,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06901855766773224,
      "rewards/margins": 0.03339996188879013,
      "rewards/rejected": -0.10244140774011612,
      "step": 25040
    },
    {
      "epoch": 0.9387120346255458,
      "grad_norm": 3.0698562366468662,
      "learning_rate": 5.054592188852508e-07,
      "log_odds_chosen": 0.5748046636581421,
      "log_odds_ratio": -0.5829101800918579,
      "logits/chosen": -0.7710937261581421,
      "logits/rejected": -0.6871093511581421,
      "logps/chosen": -0.646777331829071,
      "logps/rejected": -1.0027344226837158,
      "loss": 0.7797,
      "nll_loss": 0.6922851800918579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06466064602136612,
      "rewards/margins": 0.035518646240234375,
      "rewards/rejected": -0.1002197265625,
      "step": 25050
    },
    {
      "epoch": 0.9390867699687095,
      "grad_norm": 2.7393433677325603,
      "learning_rate": 5.053583590181615e-07,
      "log_odds_chosen": 0.59722900390625,
      "log_odds_ratio": -0.562695324420929,
      "logits/chosen": -0.761035144329071,
      "logits/rejected": -0.6322265863418579,
      "logps/chosen": -0.662792980670929,
      "logps/rejected": -1.0382812023162842,
      "loss": 0.7937,
      "nll_loss": 0.7083984613418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06625976413488388,
      "rewards/margins": 0.03754272311925888,
      "rewards/rejected": -0.10380859673023224,
      "step": 25060
    },
    {
      "epoch": 0.9394615053118734,
      "grad_norm": 3.490620223336407,
      "learning_rate": 5.052575595040407e-07,
      "log_odds_chosen": 0.44073486328125,
      "log_odds_ratio": -0.6382812261581421,
      "logits/chosen": -0.7665039300918579,
      "logits/rejected": -0.651562511920929,
      "logps/chosen": -0.694531261920929,
      "logps/rejected": -0.998828113079071,
      "loss": 0.7779,
      "nll_loss": 0.7652343511581421,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06942138820886612,
      "rewards/margins": 0.030429840087890625,
      "rewards/rejected": -0.09986571967601776,
      "step": 25070
    },
    {
      "epoch": 0.9398362406550373,
      "grad_norm": 3.1723279537579967,
      "learning_rate": 5.051568202827215e-07,
      "log_odds_chosen": 0.4425048828125,
      "log_odds_ratio": -0.597851574420929,
      "logits/chosen": -0.774609386920929,
      "logits/rejected": -0.6576172113418579,
      "logps/chosen": -0.69580078125,
      "logps/rejected": -0.9859374761581421,
      "loss": 0.7885,
      "nll_loss": 0.784375011920929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.069580078125,
      "rewards/margins": 0.02902069129049778,
      "rewards/rejected": -0.09860839694738388,
      "step": 25080
    },
    {
      "epoch": 0.9402109759982012,
      "grad_norm": 2.4550930734393237,
      "learning_rate": 5.050561412941218e-07,
      "log_odds_chosen": 0.5278075933456421,
      "log_odds_ratio": -0.569042980670929,
      "logits/chosen": -0.7621093988418579,
      "logits/rejected": -0.6666015386581421,
      "logps/chosen": -0.673046886920929,
      "logps/rejected": -1.0027344226837158,
      "loss": 0.7767,
      "nll_loss": 0.741503894329071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06733398139476776,
      "rewards/margins": 0.03299713134765625,
      "rewards/rejected": -0.10032959282398224,
      "step": 25090
    },
    {
      "epoch": 0.9405857113413651,
      "grad_norm": 2.366599197987329,
      "learning_rate": 5.049555224782425e-07,
      "log_odds_chosen": 0.5412963628768921,
      "log_odds_ratio": -0.556347668170929,
      "logits/chosen": -0.7855468988418579,
      "logits/rejected": -0.6968749761581421,
      "logps/chosen": -0.6812499761581421,
      "logps/rejected": -1.0134766101837158,
      "loss": 0.7852,
      "nll_loss": 0.7447265386581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06810303032398224,
      "rewards/margins": 0.033264923840761185,
      "rewards/rejected": -0.101318359375,
      "step": 25100
    },
    {
      "epoch": 0.940960446684529,
      "grad_norm": 2.7014270516187704,
      "learning_rate": 5.048549637751687e-07,
      "log_odds_chosen": 0.663378894329071,
      "log_odds_ratio": -0.554394543170929,
      "logits/chosen": -0.7734375,
      "logits/rejected": -0.650585949420929,
      "logps/chosen": -0.65869140625,
      "logps/rejected": -1.0763671398162842,
      "loss": 0.773,
      "nll_loss": 0.699999988079071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06585693359375,
      "rewards/margins": 0.041796110570430756,
      "rewards/rejected": -0.10767821967601776,
      "step": 25110
    },
    {
      "epoch": 0.941335182027693,
      "grad_norm": 2.7374010044587513,
      "learning_rate": 5.047544651250687e-07,
      "log_odds_chosen": 0.41075438261032104,
      "log_odds_ratio": -0.64599609375,
      "logits/chosen": -0.737109363079071,
      "logits/rejected": -0.660937488079071,
      "logps/chosen": -0.6924804449081421,
      "logps/rejected": -0.947460949420929,
      "loss": 0.7632,
      "nll_loss": 0.746386706829071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06922607123851776,
      "rewards/margins": 0.025511931627988815,
      "rewards/rejected": -0.09468994289636612,
      "step": 25120
    },
    {
      "epoch": 0.9417099173708569,
      "grad_norm": 2.9279000802009607,
      "learning_rate": 5.046540264681944e-07,
      "log_odds_chosen": 0.3423705995082855,
      "log_odds_ratio": -0.6529296636581421,
      "logits/chosen": -0.78515625,
      "logits/rejected": -0.65771484375,
      "logps/chosen": -0.6976562738418579,
      "logps/rejected": -0.9212890863418579,
      "loss": 0.7884,
      "nll_loss": 0.717578113079071,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06973876804113388,
      "rewards/margins": 0.02234649658203125,
      "rewards/rejected": -0.09213867038488388,
      "step": 25130
    },
    {
      "epoch": 0.9420846527140208,
      "grad_norm": 2.8906350496792967,
      "learning_rate": 5.045536477448807e-07,
      "log_odds_chosen": 0.5505126714706421,
      "log_odds_ratio": -0.566113293170929,
      "logits/chosen": -0.7466796636581421,
      "logits/rejected": -0.649707019329071,
      "logps/chosen": -0.6806640625,
      "logps/rejected": -1.0138671398162842,
      "loss": 0.7841,
      "nll_loss": 0.740234375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06804199516773224,
      "rewards/margins": 0.033362578600645065,
      "rewards/rejected": -0.10141601413488388,
      "step": 25140
    },
    {
      "epoch": 0.9424593880571847,
      "grad_norm": 2.734507056931109,
      "learning_rate": 5.044533288955453e-07,
      "log_odds_chosen": 0.5327819585800171,
      "log_odds_ratio": -0.5799804925918579,
      "logits/chosen": -0.773242175579071,
      "logits/rejected": -0.6705077886581421,
      "logps/chosen": -0.6908203363418579,
      "logps/rejected": -1.015625,
      "loss": 0.777,
      "nll_loss": 0.745312511920929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06906738132238388,
      "rewards/margins": 0.03237152099609375,
      "rewards/rejected": -0.10142822563648224,
      "step": 25150
    },
    {
      "epoch": 0.9428341234003486,
      "grad_norm": 2.343193419041703,
      "learning_rate": 5.043530698606893e-07,
      "log_odds_chosen": 0.4998779296875,
      "log_odds_ratio": -0.617968738079071,
      "logits/chosen": -0.798046886920929,
      "logits/rejected": -0.691601574420929,
      "logps/chosen": -0.707714855670929,
      "logps/rejected": -1.0148437023162842,
      "loss": 0.7789,
      "nll_loss": 0.8050781488418579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07073974609375,
      "rewards/margins": 0.03082275390625,
      "rewards/rejected": -0.10152588039636612,
      "step": 25160
    },
    {
      "epoch": 0.9432088587435123,
      "grad_norm": 2.8138005867845886,
      "learning_rate": 5.042528705808965e-07,
      "log_odds_chosen": 0.7203124761581421,
      "log_odds_ratio": -0.5347656011581421,
      "logits/chosen": -0.7880859375,
      "logits/rejected": -0.665820300579071,
      "logps/chosen": -0.639355480670929,
      "logps/rejected": -1.0949218273162842,
      "loss": 0.7827,
      "nll_loss": 0.7523437738418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06395263969898224,
      "rewards/margins": 0.04550781100988388,
      "rewards/rejected": -0.10952148586511612,
      "step": 25170
    },
    {
      "epoch": 0.9435835940866762,
      "grad_norm": 2.9812442871026095,
      "learning_rate": 5.041527309968327e-07,
      "log_odds_chosen": 0.6700683832168579,
      "log_odds_ratio": -0.565625011920929,
      "logits/chosen": -0.7574218511581421,
      "logits/rejected": -0.654296875,
      "logps/chosen": -0.646777331829071,
      "logps/rejected": -1.1201171875,
      "loss": 0.7618,
      "nll_loss": 0.683398425579071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06462402641773224,
      "rewards/margins": 0.04735107347369194,
      "rewards/rejected": -0.112060546875,
      "step": 25180
    },
    {
      "epoch": 0.9439583294298401,
      "grad_norm": 2.3623029570107943,
      "learning_rate": 5.040526510492467e-07,
      "log_odds_chosen": 0.525927722454071,
      "log_odds_ratio": -0.568359375,
      "logits/chosen": -0.812304675579071,
      "logits/rejected": -0.706250011920929,
      "logps/chosen": -0.648242175579071,
      "logps/rejected": -0.966015636920929,
      "loss": 0.7832,
      "nll_loss": 0.825390636920929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06488037109375,
      "rewards/margins": 0.03181915357708931,
      "rewards/rejected": -0.09663085639476776,
      "step": 25190
    },
    {
      "epoch": 0.944333064773004,
      "grad_norm": 2.7670003502726606,
      "learning_rate": 5.039526306789695e-07,
      "log_odds_chosen": 0.5442870855331421,
      "log_odds_ratio": -0.6114257574081421,
      "logits/chosen": -0.771289050579071,
      "logits/rejected": -0.6841796636581421,
      "logps/chosen": -0.689453125,
      "logps/rejected": -1.06640625,
      "loss": 0.7873,
      "nll_loss": 0.7000976800918579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06895752251148224,
      "rewards/margins": 0.03759612888097763,
      "rewards/rejected": -0.10659179836511612,
      "step": 25200
    },
    {
      "epoch": 0.944707800116168,
      "grad_norm": 2.9065400704124844,
      "learning_rate": 5.038526698269144e-07,
      "log_odds_chosen": 0.4230590760707855,
      "log_odds_ratio": -0.622363269329071,
      "logits/chosen": -0.802929699420929,
      "logits/rejected": -0.766796886920929,
      "logps/chosen": -0.683886706829071,
      "logps/rejected": -0.9482421875,
      "loss": 0.7897,
      "nll_loss": 0.7159179449081421,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06842041015625,
      "rewards/margins": 0.02642517164349556,
      "rewards/rejected": -0.09476318210363388,
      "step": 25210
    },
    {
      "epoch": 0.9450825354593319,
      "grad_norm": 2.8190524257251837,
      "learning_rate": 5.037527684340763e-07,
      "log_odds_chosen": 0.40385740995407104,
      "log_odds_ratio": -0.634570300579071,
      "logits/chosen": -0.798828125,
      "logits/rejected": -0.706738293170929,
      "logps/chosen": -0.734179675579071,
      "logps/rejected": -1.0080077648162842,
      "loss": 0.7656,
      "nll_loss": 0.7740234136581421,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07335205376148224,
      "rewards/margins": 0.02728729322552681,
      "rewards/rejected": -0.10078124701976776,
      "step": 25220
    },
    {
      "epoch": 0.9454572708024958,
      "grad_norm": 2.5868622435234743,
      "learning_rate": 5.03652926441532e-07,
      "log_odds_chosen": 0.57568359375,
      "log_odds_ratio": -0.560839831829071,
      "logits/chosen": -0.7583984136581421,
      "logits/rejected": -0.6656249761581421,
      "logps/chosen": -0.6268554925918579,
      "logps/rejected": -0.989062488079071,
      "loss": 0.7809,
      "nll_loss": 0.682421863079071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.062744140625,
      "rewards/margins": 0.03618774563074112,
      "rewards/rejected": -0.09884033352136612,
      "step": 25230
    },
    {
      "epoch": 0.9458320061456597,
      "grad_norm": 2.3142747985006618,
      "learning_rate": 5.035531437904406e-07,
      "log_odds_chosen": 0.5321289300918579,
      "log_odds_ratio": -0.5833984613418579,
      "logits/chosen": -0.7113281488418579,
      "logits/rejected": -0.6484375,
      "logps/chosen": -0.62890625,
      "logps/rejected": -0.9498046636581421,
      "loss": 0.7685,
      "nll_loss": 0.6822265386581421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06285400688648224,
      "rewards/margins": 0.032134246081113815,
      "rewards/rejected": -0.094970703125,
      "step": 25240
    },
    {
      "epoch": 0.9462067414888236,
      "grad_norm": 2.527611160721168,
      "learning_rate": 5.03453420422042e-07,
      "log_odds_chosen": 0.44556885957717896,
      "log_odds_ratio": -0.6075195074081421,
      "logits/chosen": -0.820507824420929,
      "logits/rejected": -0.69189453125,
      "logps/chosen": -0.68017578125,
      "logps/rejected": -0.9383789300918579,
      "loss": 0.7915,
      "nll_loss": 0.725781261920929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06802978366613388,
      "rewards/margins": 0.02580413781106472,
      "rewards/rejected": -0.09379883110523224,
      "step": 25250
    },
    {
      "epoch": 0.9465814768319875,
      "grad_norm": 2.9444216235545433,
      "learning_rate": 5.033537562776582e-07,
      "log_odds_chosen": 0.6310058832168579,
      "log_odds_ratio": -0.5858398675918579,
      "logits/chosen": -0.7603515386581421,
      "logits/rejected": -0.645800769329071,
      "logps/chosen": -0.616992175579071,
      "logps/rejected": -1.013671875,
      "loss": 0.7907,
      "nll_loss": 0.69189453125,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06169433519244194,
      "rewards/margins": 0.039690397679805756,
      "rewards/rejected": -0.10134277492761612,
      "step": 25260
    },
    {
      "epoch": 0.9469562121751512,
      "grad_norm": 2.222050396703523,
      "learning_rate": 5.032541512986921e-07,
      "log_odds_chosen": 0.519848644733429,
      "log_odds_ratio": -0.5669921636581421,
      "logits/chosen": -0.78515625,
      "logits/rejected": -0.699023425579071,
      "logps/chosen": -0.656445324420929,
      "logps/rejected": -0.973339855670929,
      "loss": 0.774,
      "nll_loss": 0.693066418170929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06566162407398224,
      "rewards/margins": 0.03166045993566513,
      "rewards/rejected": -0.0972900390625,
      "step": 25270
    },
    {
      "epoch": 0.9473309475183151,
      "grad_norm": 3.004770177359187,
      "learning_rate": 5.031546054266276e-07,
      "log_odds_chosen": 0.5817626714706421,
      "log_odds_ratio": -0.5733398199081421,
      "logits/chosen": -0.8018554449081421,
      "logits/rejected": -0.7079101800918579,
      "logps/chosen": -0.626953125,
      "logps/rejected": -1.03369140625,
      "loss": 0.7697,
      "nll_loss": 0.7025390863418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06269530951976776,
      "rewards/margins": 0.04067382961511612,
      "rewards/rejected": -0.10344238579273224,
      "step": 25280
    },
    {
      "epoch": 0.947705682861479,
      "grad_norm": 4.4013198655518195,
      "learning_rate": 5.0305511860303e-07,
      "log_odds_chosen": 0.605725109577179,
      "log_odds_ratio": -0.5682617425918579,
      "logits/chosen": -0.840039074420929,
      "logits/rejected": -0.705859363079071,
      "logps/chosen": -0.6786133050918579,
      "logps/rejected": -1.0802733898162842,
      "loss": 0.7879,
      "nll_loss": 0.759765625,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06783447414636612,
      "rewards/margins": 0.04009551927447319,
      "rewards/rejected": -0.10808105766773224,
      "step": 25290
    },
    {
      "epoch": 0.948080418204643,
      "grad_norm": 2.8096656903990787,
      "learning_rate": 5.029556907695452e-07,
      "log_odds_chosen": 0.4857544004917145,
      "log_odds_ratio": -0.5987304449081421,
      "logits/chosen": -0.7017577886581421,
      "logits/rejected": -0.6669921875,
      "logps/chosen": -0.677734375,
      "logps/rejected": -0.9917968511581421,
      "loss": 0.7844,
      "nll_loss": 0.6742187738418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06768798828125,
      "rewards/margins": 0.03145904466509819,
      "rewards/rejected": -0.09916992485523224,
      "step": 25300
    },
    {
      "epoch": 0.9484551535478069,
      "grad_norm": 3.508007589398057,
      "learning_rate": 5.028563218678999e-07,
      "log_odds_chosen": 0.46928709745407104,
      "log_odds_ratio": -0.6099609136581421,
      "logits/chosen": -0.786914050579071,
      "logits/rejected": -0.6639648675918579,
      "logps/chosen": -0.6888672113418579,
      "logps/rejected": -0.973437488079071,
      "loss": 0.7913,
      "nll_loss": 0.7197265625,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06885986030101776,
      "rewards/margins": 0.02844085730612278,
      "rewards/rejected": -0.09736327826976776,
      "step": 25310
    },
    {
      "epoch": 0.9488298888909708,
      "grad_norm": 2.688890677835427,
      "learning_rate": 5.02757011839901e-07,
      "log_odds_chosen": 0.4549560546875,
      "log_odds_ratio": -0.6211913824081421,
      "logits/chosen": -0.7505859136581421,
      "logits/rejected": -0.649121105670929,
      "logps/chosen": -0.671679675579071,
      "logps/rejected": -0.9883788824081421,
      "loss": 0.7899,
      "nll_loss": 0.747851550579071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06721191108226776,
      "rewards/margins": 0.03171234205365181,
      "rewards/rejected": -0.09884033352136612,
      "step": 25320
    },
    {
      "epoch": 0.9492046242341347,
      "grad_norm": 2.9717207057895765,
      "learning_rate": 5.026577606274366e-07,
      "log_odds_chosen": 0.44635009765625,
      "log_odds_ratio": -0.6117187738418579,
      "logits/chosen": -0.781445324420929,
      "logits/rejected": -0.6966797113418579,
      "logps/chosen": -0.6585937738418579,
      "logps/rejected": -0.952929675579071,
      "loss": 0.7624,
      "nll_loss": 0.7310546636581421,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06582031399011612,
      "rewards/margins": 0.0294189453125,
      "rewards/rejected": -0.09526367485523224,
      "step": 25330
    },
    {
      "epoch": 0.9495793595772986,
      "grad_norm": 2.800003390755647,
      "learning_rate": 5.025585681724742e-07,
      "log_odds_chosen": 0.577929675579071,
      "log_odds_ratio": -0.57763671875,
      "logits/chosen": -0.8177734613418579,
      "logits/rejected": -0.7328125238418579,
      "logps/chosen": -0.631542980670929,
      "logps/rejected": -0.9800781011581421,
      "loss": 0.7924,
      "nll_loss": 0.724609375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06318359076976776,
      "rewards/margins": 0.03482971340417862,
      "rewards/rejected": -0.09803466498851776,
      "step": 25340
    },
    {
      "epoch": 0.9499540949204625,
      "grad_norm": 2.99005099830517,
      "learning_rate": 5.024594344170622e-07,
      "log_odds_chosen": 0.45225828886032104,
      "log_odds_ratio": -0.599902331829071,
      "logits/chosen": -0.748242199420929,
      "logits/rejected": -0.666796863079071,
      "logps/chosen": -0.64501953125,
      "logps/rejected": -0.91796875,
      "loss": 0.7798,
      "nll_loss": 0.7027343511581421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06453857570886612,
      "rewards/margins": 0.02723083458840847,
      "rewards/rejected": -0.09173583984375,
      "step": 25350
    },
    {
      "epoch": 0.9503288302636264,
      "grad_norm": 2.8040789413102463,
      "learning_rate": 5.023603593033284e-07,
      "log_odds_chosen": 0.5628417730331421,
      "log_odds_ratio": -0.56689453125,
      "logits/chosen": -0.7451171875,
      "logits/rejected": -0.606640636920929,
      "logps/chosen": -0.67822265625,
      "logps/rejected": -1.024023413658142,
      "loss": 0.7716,
      "nll_loss": 0.703320324420929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06787109375,
      "rewards/margins": 0.034648895263671875,
      "rewards/rejected": -0.10246582329273224,
      "step": 25360
    },
    {
      "epoch": 0.9507035656067903,
      "grad_norm": 2.472743968294028,
      "learning_rate": 5.022613427734804e-07,
      "log_odds_chosen": 0.5406860113143921,
      "log_odds_ratio": -0.5712890625,
      "logits/chosen": -0.7974609136581421,
      "logits/rejected": -0.629687488079071,
      "logps/chosen": -0.6678711175918579,
      "logps/rejected": -1.0085937976837158,
      "loss": 0.7827,
      "nll_loss": 0.68798828125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06676025688648224,
      "rewards/margins": 0.03404540941119194,
      "rewards/rejected": -0.10079345852136612,
      "step": 25370
    },
    {
      "epoch": 0.951078300949954,
      "grad_norm": 3.01695629261635,
      "learning_rate": 5.021623847698063e-07,
      "log_odds_chosen": 0.46802979707717896,
      "log_odds_ratio": -0.5982421636581421,
      "logits/chosen": -0.7412109375,
      "logits/rejected": -0.66845703125,
      "logps/chosen": -0.644238293170929,
      "logps/rejected": -0.9332031011581421,
      "loss": 0.7696,
      "nll_loss": 0.7166992425918579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06439208984375,
      "rewards/margins": 0.028971100226044655,
      "rewards/rejected": -0.09335937350988388,
      "step": 25380
    },
    {
      "epoch": 0.951453036293118,
      "grad_norm": 2.6546605338942064,
      "learning_rate": 5.020634852346729e-07,
      "log_odds_chosen": 0.445556640625,
      "log_odds_ratio": -0.587695300579071,
      "logits/chosen": -0.806445300579071,
      "logits/rejected": -0.694140613079071,
      "logps/chosen": -0.6377929449081421,
      "logps/rejected": -0.885546863079071,
      "loss": 0.7939,
      "nll_loss": 0.690234363079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06376953423023224,
      "rewards/margins": 0.02481689490377903,
      "rewards/rejected": -0.08859863132238388,
      "step": 25390
    },
    {
      "epoch": 0.9518277716362818,
      "grad_norm": 2.7553773277017277,
      "learning_rate": 5.019646441105269e-07,
      "log_odds_chosen": 0.6974242925643921,
      "log_odds_ratio": -0.5391601324081421,
      "logits/chosen": -0.741406261920929,
      "logits/rejected": -0.63134765625,
      "logps/chosen": -0.62890625,
      "logps/rejected": -1.0869140625,
      "loss": 0.7708,
      "nll_loss": 0.74169921875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0628662109375,
      "rewards/margins": 0.04589385911822319,
      "rewards/rejected": -0.10872802883386612,
      "step": 25400
    },
    {
      "epoch": 0.9522025069794458,
      "grad_norm": 2.807029418150161,
      "learning_rate": 5.018658613398939e-07,
      "log_odds_chosen": 0.6794677972793579,
      "log_odds_ratio": -0.5165039300918579,
      "logits/chosen": -0.82421875,
      "logits/rejected": -0.685351550579071,
      "logps/chosen": -0.6268554925918579,
      "logps/rejected": -1.0275390148162842,
      "loss": 0.7643,
      "nll_loss": 0.7017577886581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06269530951976776,
      "rewards/margins": 0.040070343762636185,
      "rewards/rejected": -0.10275878757238388,
      "step": 25410
    },
    {
      "epoch": 0.9525772423226097,
      "grad_norm": 2.738824384687323,
      "learning_rate": 5.017671368653793e-07,
      "log_odds_chosen": 0.523681640625,
      "log_odds_ratio": -0.5909179449081421,
      "logits/chosen": -0.758984386920929,
      "logits/rejected": -0.6475585699081421,
      "logps/chosen": -0.640820324420929,
      "logps/rejected": -0.9400390386581421,
      "loss": 0.7728,
      "nll_loss": 0.731640636920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06403808295726776,
      "rewards/margins": 0.02993164025247097,
      "rewards/rejected": -0.09396972507238388,
      "step": 25420
    },
    {
      "epoch": 0.9529519776657736,
      "grad_norm": 2.4578785982009586,
      "learning_rate": 5.016684706296667e-07,
      "log_odds_chosen": 0.40974122285842896,
      "log_odds_ratio": -0.610156238079071,
      "logits/chosen": -0.849804699420929,
      "logits/rejected": -0.6949218511581421,
      "logps/chosen": -0.643847644329071,
      "logps/rejected": -0.9166015386581421,
      "loss": 0.7885,
      "nll_loss": 0.722460925579071,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06440429389476776,
      "rewards/margins": 0.02718353271484375,
      "rewards/rejected": -0.0916748046875,
      "step": 25430
    },
    {
      "epoch": 0.9533267130089375,
      "grad_norm": 3.56322746958455,
      "learning_rate": 5.015698625755191e-07,
      "log_odds_chosen": 0.695361316204071,
      "log_odds_ratio": -0.53125,
      "logits/chosen": -0.730664074420929,
      "logits/rejected": -0.6019531488418579,
      "logps/chosen": -0.6524413824081421,
      "logps/rejected": -1.1042969226837158,
      "loss": 0.7753,
      "nll_loss": 0.7251952886581421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06520996242761612,
      "rewards/margins": 0.0452117919921875,
      "rewards/rejected": -0.1103515625,
      "step": 25440
    },
    {
      "epoch": 0.9537014483521014,
      "grad_norm": 3.0980563468714784,
      "learning_rate": 5.014713126457781e-07,
      "log_odds_chosen": 0.505816638469696,
      "log_odds_ratio": -0.6102539300918579,
      "logits/chosen": -0.732617199420929,
      "logits/rejected": -0.6329101324081421,
      "logps/chosen": -0.6625000238418579,
      "logps/rejected": -0.966015636920929,
      "loss": 0.8043,
      "nll_loss": 0.7154296636581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.0662841796875,
      "rewards/margins": 0.030422210693359375,
      "rewards/rejected": -0.09666748344898224,
      "step": 25450
    },
    {
      "epoch": 0.9540761836952653,
      "grad_norm": 3.8751200945461006,
      "learning_rate": 5.013728207833639e-07,
      "log_odds_chosen": 0.4424072206020355,
      "log_odds_ratio": -0.604296863079071,
      "logits/chosen": -0.716601550579071,
      "logits/rejected": -0.636425793170929,
      "logps/chosen": -0.690136730670929,
      "logps/rejected": -0.951171875,
      "loss": 0.7821,
      "nll_loss": 0.7265625,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0689697265625,
      "rewards/margins": 0.02620544470846653,
      "rewards/rejected": -0.09516601264476776,
      "step": 25460
    },
    {
      "epoch": 0.9544509190384292,
      "grad_norm": 2.951225705813196,
      "learning_rate": 5.012743869312748e-07,
      "log_odds_chosen": 0.4521240293979645,
      "log_odds_ratio": -0.6265624761581421,
      "logits/chosen": -0.794726550579071,
      "logits/rejected": -0.681445300579071,
      "logps/chosen": -0.7108398675918579,
      "logps/rejected": -1.0128905773162842,
      "loss": 0.7905,
      "nll_loss": 0.7383788824081421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07103271782398224,
      "rewards/margins": 0.03016052208840847,
      "rewards/rejected": -0.1011962890625,
      "step": 25470
    },
    {
      "epoch": 0.954825654381593,
      "grad_norm": 2.530092142051759,
      "learning_rate": 5.011760110325881e-07,
      "log_odds_chosen": 0.5168212652206421,
      "log_odds_ratio": -0.5826171636581421,
      "logits/chosen": -0.7880859375,
      "logits/rejected": -0.677929699420929,
      "logps/chosen": -0.6693359613418579,
      "logps/rejected": -0.9912109375,
      "loss": 0.7829,
      "nll_loss": 0.7796875238418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06688232719898224,
      "rewards/margins": 0.03231658786535263,
      "rewards/rejected": -0.09925536811351776,
      "step": 25480
    },
    {
      "epoch": 0.9552003897247568,
      "grad_norm": 3.0177169428720076,
      "learning_rate": 5.010776930304587e-07,
      "log_odds_chosen": 0.4632324278354645,
      "log_odds_ratio": -0.591113269329071,
      "logits/chosen": -0.8167968988418579,
      "logits/rejected": -0.687792956829071,
      "logps/chosen": -0.660937488079071,
      "logps/rejected": -0.9481445550918579,
      "loss": 0.7662,
      "nll_loss": 0.7411133050918579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06611327826976776,
      "rewards/margins": 0.02874298021197319,
      "rewards/rejected": -0.09481201320886612,
      "step": 25490
    },
    {
      "epoch": 0.9555751250679207,
      "grad_norm": 2.8312012261491604,
      "learning_rate": 5.009794328681196e-07,
      "log_odds_chosen": 0.601025402545929,
      "log_odds_ratio": -0.565625011920929,
      "logits/chosen": -0.766796886920929,
      "logits/rejected": -0.6708008050918579,
      "logps/chosen": -0.6231445074081421,
      "logps/rejected": -0.991406261920929,
      "loss": 0.7899,
      "nll_loss": 0.6563476324081421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06232910230755806,
      "rewards/margins": 0.036775968968868256,
      "rewards/rejected": -0.09906005859375,
      "step": 25500
    },
    {
      "epoch": 0.9559498604110847,
      "grad_norm": 2.104310804534428,
      "learning_rate": 5.008812304888818e-07,
      "log_odds_chosen": 0.550036609172821,
      "log_odds_ratio": -0.592089831829071,
      "logits/chosen": -0.7623046636581421,
      "logits/rejected": -0.6748046875,
      "logps/chosen": -0.6216796636581421,
      "logps/rejected": -0.9906250238418579,
      "loss": 0.7659,
      "nll_loss": 0.7181640863418579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06223144382238388,
      "rewards/margins": 0.03685150295495987,
      "rewards/rejected": -0.09913329780101776,
      "step": 25510
    },
    {
      "epoch": 0.9563245957542486,
      "grad_norm": 3.561514906169516,
      "learning_rate": 5.007830858361344e-07,
      "log_odds_chosen": 0.584381103515625,
      "log_odds_ratio": -0.5704101324081421,
      "logits/chosen": -0.769335925579071,
      "logits/rejected": -0.677929699420929,
      "logps/chosen": -0.6888672113418579,
      "logps/rejected": -1.065820336341858,
      "loss": 0.7816,
      "nll_loss": 0.727343738079071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06890869140625,
      "rewards/margins": 0.0376434326171875,
      "rewards/rejected": -0.10651855170726776,
      "step": 25520
    },
    {
      "epoch": 0.9566993310974125,
      "grad_norm": 3.0330424519796986,
      "learning_rate": 5.006849988533433e-07,
      "log_odds_chosen": 0.4998413026332855,
      "log_odds_ratio": -0.5892578363418579,
      "logits/chosen": -0.8003906011581421,
      "logits/rejected": -0.7164062261581421,
      "logps/chosen": -0.6626952886581421,
      "logps/rejected": -0.9644531011581421,
      "loss": 0.7809,
      "nll_loss": 0.7158203125,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06629638373851776,
      "rewards/margins": 0.030152130872011185,
      "rewards/rejected": -0.09647216647863388,
      "step": 25530
    },
    {
      "epoch": 0.9570740664405764,
      "grad_norm": 3.433246097644279,
      "learning_rate": 5.005869694840526e-07,
      "log_odds_chosen": 0.5404052734375,
      "log_odds_ratio": -0.581347644329071,
      "logits/chosen": -0.749218761920929,
      "logits/rejected": -0.607226550579071,
      "logps/chosen": -0.6297851800918579,
      "logps/rejected": -0.9677734375,
      "loss": 0.7498,
      "nll_loss": 0.673632800579071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06298828125,
      "rewards/margins": 0.03380126878619194,
      "rewards/rejected": -0.09683837741613388,
      "step": 25540
    },
    {
      "epoch": 0.9574488017837403,
      "grad_norm": 2.6520847068908107,
      "learning_rate": 5.004889976718836e-07,
      "log_odds_chosen": 0.642016589641571,
      "log_odds_ratio": -0.546875,
      "logits/chosen": -0.800488293170929,
      "logits/rejected": -0.679394543170929,
      "logps/chosen": -0.672656238079071,
      "logps/rejected": -1.070898413658142,
      "loss": 0.7793,
      "nll_loss": 0.7657226324081421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06732177734375,
      "rewards/margins": 0.03987579420208931,
      "rewards/rejected": -0.10712890326976776,
      "step": 25550
    },
    {
      "epoch": 0.9578235371269042,
      "grad_norm": 3.171455676403352,
      "learning_rate": 5.003910833605343e-07,
      "log_odds_chosen": 0.6081176996231079,
      "log_odds_ratio": -0.5497070550918579,
      "logits/chosen": -0.7691406011581421,
      "logits/rejected": -0.643847644329071,
      "logps/chosen": -0.6507812738418579,
      "logps/rejected": -1.043359398841858,
      "loss": 0.7954,
      "nll_loss": 0.706738293170929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06501464545726776,
      "rewards/margins": 0.03922881931066513,
      "rewards/rejected": -0.1043701171875,
      "step": 25560
    },
    {
      "epoch": 0.9581982724700681,
      "grad_norm": 2.8922629405230227,
      "learning_rate": 5.002932264937806e-07,
      "log_odds_chosen": 0.5126098394393921,
      "log_odds_ratio": -0.6041015386581421,
      "logits/chosen": -0.833789050579071,
      "logits/rejected": -0.703906238079071,
      "logps/chosen": -0.625292956829071,
      "logps/rejected": -0.9525390863418579,
      "loss": 0.7863,
      "nll_loss": 0.693554699420929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06253661960363388,
      "rewards/margins": 0.03275756910443306,
      "rewards/rejected": -0.09527587890625,
      "step": 25570
    },
    {
      "epoch": 0.958573007813232,
      "grad_norm": 2.6877011184864923,
      "learning_rate": 5.001954270154747e-07,
      "log_odds_chosen": 0.45720213651657104,
      "log_odds_ratio": -0.5814453363418579,
      "logits/chosen": -0.78125,
      "logits/rejected": -0.703125,
      "logps/chosen": -0.6314452886581421,
      "logps/rejected": -0.9126952886581421,
      "loss": 0.7904,
      "nll_loss": 0.6826171875,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06312255561351776,
      "rewards/margins": 0.02808685228228569,
      "rewards/rejected": -0.09123535454273224,
      "step": 25580
    },
    {
      "epoch": 0.9589477431563957,
      "grad_norm": 2.7779418710548898,
      "learning_rate": 5.000976848695459e-07,
      "log_odds_chosen": 0.6143432855606079,
      "log_odds_ratio": -0.580078125,
      "logits/chosen": -0.823046863079071,
      "logits/rejected": -0.650195300579071,
      "logps/chosen": -0.6841796636581421,
      "logps/rejected": -1.063867211341858,
      "loss": 0.7637,
      "nll_loss": 0.6949218511581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06834717094898224,
      "rewards/margins": 0.03805999830365181,
      "rewards/rejected": -0.10635986179113388,
      "step": 25590
    },
    {
      "epoch": 0.9593224784995596,
      "grad_norm": 3.074678616125516,
      "learning_rate": 5e-07,
      "log_odds_chosen": 0.5830322504043579,
      "log_odds_ratio": -0.5736328363418579,
      "logits/chosen": -0.773632824420929,
      "logits/rejected": -0.6517578363418579,
      "logps/chosen": -0.701464831829071,
      "logps/rejected": -1.0849609375,
      "loss": 0.7703,
      "nll_loss": 0.705859363079071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07020263373851776,
      "rewards/margins": 0.038240812718868256,
      "rewards/rejected": -0.10834960639476776,
      "step": 25600
    },
    {
      "epoch": 0.9596972138427236,
      "grad_norm": 2.6832984481652074,
      "learning_rate": 4.999023723509194e-07,
      "log_odds_chosen": 0.5484863519668579,
      "log_odds_ratio": -0.58740234375,
      "logits/chosen": -0.769824206829071,
      "logits/rejected": -0.659960925579071,
      "logps/chosen": -0.689257800579071,
      "logps/rejected": -1.0263671875,
      "loss": 0.7844,
      "nll_loss": 0.737011730670929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06892089545726776,
      "rewards/margins": 0.03362274169921875,
      "rewards/rejected": -0.10261230170726776,
      "step": 25610
    },
    {
      "epoch": 0.9600719491858875,
      "grad_norm": 2.446882249281692,
      "learning_rate": 4.99804801866463e-07,
      "log_odds_chosen": 0.689135730266571,
      "log_odds_ratio": -0.53857421875,
      "logits/chosen": -0.8082031011581421,
      "logits/rejected": -0.704296886920929,
      "logps/chosen": -0.6102539300918579,
      "logps/rejected": -1.034570336341858,
      "loss": 0.7857,
      "nll_loss": 0.693554699420929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06098632887005806,
      "rewards/margins": 0.042479705065488815,
      "rewards/rejected": -0.10340575873851776,
      "step": 25620
    },
    {
      "epoch": 0.9604466845290514,
      "grad_norm": 2.4564187478679873,
      "learning_rate": 4.997072884908658e-07,
      "log_odds_chosen": 0.5489867925643921,
      "log_odds_ratio": -0.575878918170929,
      "logits/chosen": -0.7308593988418579,
      "logits/rejected": -0.6444336175918579,
      "logps/chosen": -0.652539074420929,
      "logps/rejected": -1.0146484375,
      "loss": 0.7792,
      "nll_loss": 0.7236328125,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06528320163488388,
      "rewards/margins": 0.036191560328006744,
      "rewards/rejected": -0.10148926079273224,
      "step": 25630
    },
    {
      "epoch": 0.9608214198722153,
      "grad_norm": 2.559743651606401,
      "learning_rate": 4.996098321684392e-07,
      "log_odds_chosen": 0.642163097858429,
      "log_odds_ratio": -0.563281238079071,
      "logits/chosen": -0.7935546636581421,
      "logits/rejected": -0.693164050579071,
      "logps/chosen": -0.665332019329071,
      "logps/rejected": -1.0750000476837158,
      "loss": 0.7598,
      "nll_loss": 0.719921886920929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06656493991613388,
      "rewards/margins": 0.04094696044921875,
      "rewards/rejected": -0.10747070610523224,
      "step": 25640
    },
    {
      "epoch": 0.9611961552153792,
      "grad_norm": 2.4755446254270312,
      "learning_rate": 4.995124328435701e-07,
      "log_odds_chosen": 0.47734373807907104,
      "log_odds_ratio": -0.5997070074081421,
      "logits/chosen": -0.7660156488418579,
      "logits/rejected": -0.658203125,
      "logps/chosen": -0.6966797113418579,
      "logps/rejected": -0.994921863079071,
      "loss": 0.8009,
      "nll_loss": 0.7671874761581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06966552883386612,
      "rewards/margins": 0.029868315905332565,
      "rewards/rejected": -0.09956054389476776,
      "step": 25650
    },
    {
      "epoch": 0.9615708905585431,
      "grad_norm": 2.7792543596412345,
      "learning_rate": 4.994150904607217e-07,
      "log_odds_chosen": 0.46673583984375,
      "log_odds_ratio": -0.617480456829071,
      "logits/chosen": -0.7115234136581421,
      "logits/rejected": -0.6499999761581421,
      "logps/chosen": -0.634472668170929,
      "logps/rejected": -0.913867175579071,
      "loss": 0.7829,
      "nll_loss": 0.6978515386581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06340332329273224,
      "rewards/margins": 0.02803802490234375,
      "rewards/rejected": -0.0914306640625,
      "step": 25660
    },
    {
      "epoch": 0.961945625901707,
      "grad_norm": 2.851439765257029,
      "learning_rate": 4.993178049644328e-07,
      "log_odds_chosen": 0.496826171875,
      "log_odds_ratio": -0.593945324420929,
      "logits/chosen": -0.795703113079071,
      "logits/rejected": -0.6871093511581421,
      "logps/chosen": -0.6622070074081421,
      "logps/rejected": -0.9703124761581421,
      "loss": 0.7681,
      "nll_loss": 0.728222668170929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06625976413488388,
      "rewards/margins": 0.03078918531537056,
      "rewards/rejected": -0.09713134914636612,
      "step": 25670
    },
    {
      "epoch": 0.9623203612448709,
      "grad_norm": 2.903690750726363,
      "learning_rate": 4.992205762993177e-07,
      "log_odds_chosen": 0.695361316204071,
      "log_odds_ratio": -0.533203125,
      "logits/chosen": -0.7583984136581421,
      "logits/rejected": -0.633984386920929,
      "logps/chosen": -0.626171886920929,
      "logps/rejected": -1.063867211341858,
      "loss": 0.7836,
      "nll_loss": 0.7464843988418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.0626220703125,
      "rewards/margins": 0.04374389722943306,
      "rewards/rejected": -0.1064453125,
      "step": 25680
    },
    {
      "epoch": 0.9626950965880346,
      "grad_norm": 2.735592077113456,
      "learning_rate": 4.991234044100666e-07,
      "log_odds_chosen": 0.561816394329071,
      "log_odds_ratio": -0.573925793170929,
      "logits/chosen": -0.810351550579071,
      "logits/rejected": -0.681445300579071,
      "logps/chosen": -0.6297851800918579,
      "logps/rejected": -0.9658203125,
      "loss": 0.8128,
      "nll_loss": 0.7435547113418579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06297607719898224,
      "rewards/margins": 0.03364715725183487,
      "rewards/rejected": -0.0965576171875,
      "step": 25690
    },
    {
      "epoch": 0.9630698319311986,
      "grad_norm": 2.5084351608461275,
      "learning_rate": 4.990262892414443e-07,
      "log_odds_chosen": 0.5382080078125,
      "log_odds_ratio": -0.5609375238418579,
      "logits/chosen": -0.788867175579071,
      "logits/rejected": -0.686718761920929,
      "logps/chosen": -0.676562488079071,
      "logps/rejected": -1.0154297351837158,
      "loss": 0.7909,
      "nll_loss": 0.740429699420929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.067626953125,
      "rewards/margins": 0.033902741968631744,
      "rewards/rejected": -0.10150146484375,
      "step": 25700
    },
    {
      "epoch": 0.9634445672743625,
      "grad_norm": 3.0695415714648187,
      "learning_rate": 4.989292307382912e-07,
      "log_odds_chosen": 0.623242199420929,
      "log_odds_ratio": -0.563281238079071,
      "logits/chosen": -0.765429675579071,
      "logits/rejected": -0.671093761920929,
      "logps/chosen": -0.63525390625,
      "logps/rejected": -1.0154297351837158,
      "loss": 0.7725,
      "nll_loss": 0.705273449420929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06352539360523224,
      "rewards/margins": 0.03796539455652237,
      "rewards/rejected": -0.10153808444738388,
      "step": 25710
    },
    {
      "epoch": 0.9638193026175264,
      "grad_norm": 2.68706399934422,
      "learning_rate": 4.98832228845523e-07,
      "log_odds_chosen": 0.8046875,
      "log_odds_ratio": -0.4922851622104645,
      "logits/chosen": -0.7896484136581421,
      "logits/rejected": -0.649121105670929,
      "logps/chosen": -0.613964855670929,
      "logps/rejected": -1.1091797351837158,
      "loss": 0.7599,
      "nll_loss": 0.702441394329071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06141357496380806,
      "rewards/margins": 0.04949340969324112,
      "rewards/rejected": -0.1109619140625,
      "step": 25720
    },
    {
      "epoch": 0.9641940379606903,
      "grad_norm": 2.462410640710504,
      "learning_rate": 4.987352835081298e-07,
      "log_odds_chosen": 0.6148681640625,
      "log_odds_ratio": -0.5528320074081421,
      "logits/chosen": -0.7533203363418579,
      "logits/rejected": -0.660449206829071,
      "logps/chosen": -0.647265613079071,
      "logps/rejected": -0.993945300579071,
      "loss": 0.8071,
      "nll_loss": 0.682324230670929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06472168117761612,
      "rewards/margins": 0.034645844250917435,
      "rewards/rejected": -0.09930419921875,
      "step": 25730
    },
    {
      "epoch": 0.9645687733038542,
      "grad_norm": 2.3369500631111406,
      "learning_rate": 4.986383946711772e-07,
      "log_odds_chosen": 0.5399414300918579,
      "log_odds_ratio": -0.578320324420929,
      "logits/chosen": -0.7826172113418579,
      "logits/rejected": -0.671093761920929,
      "logps/chosen": -0.6849609613418579,
      "logps/rejected": -1.0451171398162842,
      "loss": 0.7828,
      "nll_loss": 0.7240234613418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0684814453125,
      "rewards/margins": 0.0360107421875,
      "rewards/rejected": -0.10451660305261612,
      "step": 25740
    },
    {
      "epoch": 0.9649435086470181,
      "grad_norm": 3.1851150966833304,
      "learning_rate": 4.985415622798046e-07,
      "log_odds_chosen": 0.54833984375,
      "log_odds_ratio": -0.5677734613418579,
      "logits/chosen": -0.787304699420929,
      "logits/rejected": -0.727734386920929,
      "logps/chosen": -0.648730456829071,
      "logps/rejected": -0.9876953363418579,
      "loss": 0.7836,
      "nll_loss": 0.7080078125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06486816704273224,
      "rewards/margins": 0.03390808030962944,
      "rewards/rejected": -0.09871826320886612,
      "step": 25750
    },
    {
      "epoch": 0.965318243990182,
      "grad_norm": 2.577991995371174,
      "learning_rate": 4.984447862792267e-07,
      "log_odds_chosen": 0.548291027545929,
      "log_odds_ratio": -0.568066418170929,
      "logits/chosen": -0.782421886920929,
      "logits/rejected": -0.671191394329071,
      "logps/chosen": -0.67578125,
      "logps/rejected": -1.033789038658142,
      "loss": 0.7713,
      "nll_loss": 0.7049804925918579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06751708686351776,
      "rewards/margins": 0.03589172288775444,
      "rewards/rejected": -0.10340575873851776,
      "step": 25760
    },
    {
      "epoch": 0.9656929793333459,
      "grad_norm": 2.8694430328548592,
      "learning_rate": 4.983480666147325e-07,
      "log_odds_chosen": 0.554394543170929,
      "log_odds_ratio": -0.5770508050918579,
      "logits/chosen": -0.745312511920929,
      "logits/rejected": -0.6656249761581421,
      "logps/chosen": -0.67919921875,
      "logps/rejected": -1.040429711341858,
      "loss": 0.7676,
      "nll_loss": 0.7216796875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06790771335363388,
      "rewards/margins": 0.03617248684167862,
      "rewards/rejected": -0.1041259765625,
      "step": 25770
    },
    {
      "epoch": 0.9660677146765098,
      "grad_norm": 2.560288926450745,
      "learning_rate": 4.982514032316846e-07,
      "log_odds_chosen": 0.418975830078125,
      "log_odds_ratio": -0.625781238079071,
      "logits/chosen": -0.7964843511581421,
      "logits/rejected": -0.660839855670929,
      "logps/chosen": -0.6927734613418579,
      "logps/rejected": -0.958203136920929,
      "loss": 0.778,
      "nll_loss": 0.7354491949081421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06929931789636612,
      "rewards/margins": 0.02655181847512722,
      "rewards/rejected": -0.09581299126148224,
      "step": 25780
    },
    {
      "epoch": 0.9664424500196737,
      "grad_norm": 2.6335584591342793,
      "learning_rate": 4.98154796075521e-07,
      "log_odds_chosen": 0.4443725645542145,
      "log_odds_ratio": -0.6292968988418579,
      "logits/chosen": -0.7906249761581421,
      "logits/rejected": -0.7105468511581421,
      "logps/chosen": -0.6634765863418579,
      "logps/rejected": -0.9349609613418579,
      "loss": 0.7707,
      "nll_loss": 0.7513672113418579,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.06633301079273224,
      "rewards/margins": 0.02710266038775444,
      "rewards/rejected": -0.09343262016773224,
      "step": 25790
    },
    {
      "epoch": 0.9668171853628375,
      "grad_norm": 2.6303987129336166,
      "learning_rate": 4.980582450917523e-07,
      "log_odds_chosen": 0.5423828363418579,
      "log_odds_ratio": -0.590624988079071,
      "logits/chosen": -0.855664074420929,
      "logits/rejected": -0.724609375,
      "logps/chosen": -0.6656249761581421,
      "logps/rejected": -0.991992175579071,
      "loss": 0.7896,
      "nll_loss": 0.6890624761581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06660155951976776,
      "rewards/margins": 0.03267364576458931,
      "rewards/rejected": -0.09929199516773224,
      "step": 25800
    },
    {
      "epoch": 0.9671919207060014,
      "grad_norm": 2.992911370433323,
      "learning_rate": 4.979617502259639e-07,
      "log_odds_chosen": 0.5088745355606079,
      "log_odds_ratio": -0.570605456829071,
      "logits/chosen": -0.7876952886581421,
      "logits/rejected": -0.65576171875,
      "logps/chosen": -0.6719726324081421,
      "logps/rejected": -0.9935547113418579,
      "loss": 0.7869,
      "nll_loss": 0.7490234375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06718750298023224,
      "rewards/margins": 0.03205261379480362,
      "rewards/rejected": -0.09929199516773224,
      "step": 25810
    },
    {
      "epoch": 0.9675666560491653,
      "grad_norm": 3.1500449161308715,
      "learning_rate": 4.978653114238151e-07,
      "log_odds_chosen": 0.530871570110321,
      "log_odds_ratio": -0.598339855670929,
      "logits/chosen": -0.7583984136581421,
      "logits/rejected": -0.6454101800918579,
      "logps/chosen": -0.716015636920929,
      "logps/rejected": -1.0632812976837158,
      "loss": 0.776,
      "nll_loss": 0.7314453125,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07149658352136612,
      "rewards/margins": 0.034680940210819244,
      "rewards/rejected": -0.10625000298023224,
      "step": 25820
    },
    {
      "epoch": 0.9679413913923292,
      "grad_norm": 2.596615675454656,
      "learning_rate": 4.977689286310382e-07,
      "log_odds_chosen": 0.5471435785293579,
      "log_odds_ratio": -0.5835937261581421,
      "logits/chosen": -0.7828124761581421,
      "logits/rejected": -0.6629883050918579,
      "logps/chosen": -0.6796875,
      "logps/rejected": -1.0291016101837158,
      "loss": 0.7832,
      "nll_loss": 0.704882800579071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0679931640625,
      "rewards/margins": 0.03500824049115181,
      "rewards/rejected": -0.10299072414636612,
      "step": 25830
    },
    {
      "epoch": 0.968316126735493,
      "grad_norm": 3.148178484760972,
      "learning_rate": 4.976726017934395e-07,
      "log_odds_chosen": 0.6514648199081421,
      "log_odds_ratio": -0.5582031011581421,
      "logits/chosen": -0.73828125,
      "logits/rejected": -0.6597656011581421,
      "logps/chosen": -0.6953125,
      "logps/rejected": -1.083593726158142,
      "loss": 0.793,
      "nll_loss": 0.694628894329071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06954345852136612,
      "rewards/margins": 0.038818359375,
      "rewards/rejected": -0.10834960639476776,
      "step": 25840
    },
    {
      "epoch": 0.968690862078657,
      "grad_norm": 2.647124631717101,
      "learning_rate": 4.975763308568984e-07,
      "log_odds_chosen": 0.505139172077179,
      "log_odds_ratio": -0.5767577886581421,
      "logits/chosen": -0.786914050579071,
      "logits/rejected": -0.648242175579071,
      "logps/chosen": -0.68359375,
      "logps/rejected": -0.9916015863418579,
      "loss": 0.7603,
      "nll_loss": 0.67431640625,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06840820610523224,
      "rewards/margins": 0.03080444410443306,
      "rewards/rejected": -0.09921874850988388,
      "step": 25850
    },
    {
      "epoch": 0.9690655974218209,
      "grad_norm": 2.762386714735485,
      "learning_rate": 4.974801157673675e-07,
      "log_odds_chosen": 0.6631835699081421,
      "log_odds_ratio": -0.526660144329071,
      "logits/chosen": -0.7806640863418579,
      "logits/rejected": -0.6830078363418579,
      "logps/chosen": -0.644824206829071,
      "logps/rejected": -1.0439453125,
      "loss": 0.7835,
      "nll_loss": 0.69970703125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06446532905101776,
      "rewards/margins": 0.03997802734375,
      "rewards/rejected": -0.10446777194738388,
      "step": 25860
    },
    {
      "epoch": 0.9694403327649848,
      "grad_norm": 2.3758471149083276,
      "learning_rate": 4.97383956470873e-07,
      "log_odds_chosen": 0.4418701231479645,
      "log_odds_ratio": -0.5943359136581421,
      "logits/chosen": -0.7417968511581421,
      "logits/rejected": -0.6749023199081421,
      "logps/chosen": -0.7120116949081421,
      "logps/rejected": -0.9896484613418579,
      "loss": 0.7872,
      "nll_loss": 0.7310546636581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07121582329273224,
      "rewards/margins": 0.027680207043886185,
      "rewards/rejected": -0.09895019233226776,
      "step": 25870
    },
    {
      "epoch": 0.9698150681081487,
      "grad_norm": 3.394819308636749,
      "learning_rate": 4.972878529135135e-07,
      "log_odds_chosen": 0.45329588651657104,
      "log_odds_ratio": -0.5938476324081421,
      "logits/chosen": -0.760449230670929,
      "logits/rejected": -0.6558593511581421,
      "logps/chosen": -0.6366211175918579,
      "logps/rejected": -0.9292968511581421,
      "loss": 0.7632,
      "nll_loss": 0.703320324420929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06361083686351776,
      "rewards/margins": 0.0293121337890625,
      "rewards/rejected": -0.09296874701976776,
      "step": 25880
    },
    {
      "epoch": 0.9701898034513126,
      "grad_norm": 5.097570151225006,
      "learning_rate": 4.971918050414611e-07,
      "log_odds_chosen": 0.644238293170929,
      "log_odds_ratio": -0.5589843988418579,
      "logits/chosen": -0.7763671875,
      "logits/rejected": -0.6678711175918579,
      "logps/chosen": -0.635937511920929,
      "logps/rejected": -1.050390601158142,
      "loss": 0.7623,
      "nll_loss": 0.6888672113418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06361083686351776,
      "rewards/margins": 0.0413970947265625,
      "rewards/rejected": -0.10505370795726776,
      "step": 25890
    },
    {
      "epoch": 0.9705645387944764,
      "grad_norm": 2.547160985080854,
      "learning_rate": 4.9709581280096e-07,
      "log_odds_chosen": 0.614941418170929,
      "log_odds_ratio": -0.556347668170929,
      "logits/chosen": -0.765625,
      "logits/rejected": -0.6371093988418579,
      "logps/chosen": -0.671093761920929,
      "logps/rejected": -1.052148461341858,
      "loss": 0.764,
      "nll_loss": 0.69287109375,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06716308742761612,
      "rewards/margins": 0.03800048679113388,
      "rewards/rejected": -0.10515137016773224,
      "step": 25900
    },
    {
      "epoch": 0.9709392741376403,
      "grad_norm": 2.6373983733129633,
      "learning_rate": 4.969998761383275e-07,
      "log_odds_chosen": 0.5596679449081421,
      "log_odds_ratio": -0.5770508050918579,
      "logits/chosen": -0.7933593988418579,
      "logits/rejected": -0.630175769329071,
      "logps/chosen": -0.682812511920929,
      "logps/rejected": -1.05078125,
      "loss": 0.7673,
      "nll_loss": 0.7044922113418579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06822510063648224,
      "rewards/margins": 0.03681793063879013,
      "rewards/rejected": -0.10507812350988388,
      "step": 25910
    },
    {
      "epoch": 0.9713140094808042,
      "grad_norm": 3.059890709894124,
      "learning_rate": 4.969039949999532e-07,
      "log_odds_chosen": 0.40874022245407104,
      "log_odds_ratio": -0.590527355670929,
      "logits/chosen": -0.7847656011581421,
      "logits/rejected": -0.7044922113418579,
      "logps/chosen": -0.657421886920929,
      "logps/rejected": -0.9058593511581421,
      "loss": 0.7889,
      "nll_loss": 0.6939452886581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06568603217601776,
      "rewards/margins": 0.024860382080078125,
      "rewards/rejected": -0.09055175632238388,
      "step": 25920
    },
    {
      "epoch": 0.971688744823968,
      "grad_norm": 3.0085589073280157,
      "learning_rate": 4.968081693322991e-07,
      "log_odds_chosen": 0.32044678926467896,
      "log_odds_ratio": -0.6478515863418579,
      "logits/chosen": -0.775585949420929,
      "logits/rejected": -0.7093750238418579,
      "logps/chosen": -0.7044922113418579,
      "logps/rejected": -0.897265613079071,
      "loss": 0.7782,
      "nll_loss": 0.766796886920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07041015475988388,
      "rewards/margins": 0.019304657354950905,
      "rewards/rejected": -0.08974609524011612,
      "step": 25930
    },
    {
      "epoch": 0.972063480167132,
      "grad_norm": 3.217279261336988,
      "learning_rate": 4.967123990818995e-07,
      "log_odds_chosen": 0.629687488079071,
      "log_odds_ratio": -0.5635741949081421,
      "logits/chosen": -0.8125,
      "logits/rejected": -0.69091796875,
      "logps/chosen": -0.6883789300918579,
      "logps/rejected": -1.0939452648162842,
      "loss": 0.7932,
      "nll_loss": 0.7313476800918579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06883545219898224,
      "rewards/margins": 0.04064636304974556,
      "rewards/rejected": -0.109375,
      "step": 25940
    },
    {
      "epoch": 0.9724382155102959,
      "grad_norm": 2.367456158508081,
      "learning_rate": 4.966166841953606e-07,
      "log_odds_chosen": 0.532397449016571,
      "log_odds_ratio": -0.5831054449081421,
      "logits/chosen": -0.7896484136581421,
      "logits/rejected": -0.666015625,
      "logps/chosen": -0.663867175579071,
      "logps/rejected": -0.9906250238418579,
      "loss": 0.7741,
      "nll_loss": 0.702832043170929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06635741889476776,
      "rewards/margins": 0.03272705152630806,
      "rewards/rejected": -0.09907226264476776,
      "step": 25950
    },
    {
      "epoch": 0.9728129508534598,
      "grad_norm": 2.474245908409096,
      "learning_rate": 4.965210246193609e-07,
      "log_odds_chosen": 0.6563720703125,
      "log_odds_ratio": -0.557812511920929,
      "logits/chosen": -0.8031250238418579,
      "logits/rejected": -0.693164050579071,
      "logps/chosen": -0.63671875,
      "logps/rejected": -1.0810546875,
      "loss": 0.7641,
      "nll_loss": 0.7177734375,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06368408352136612,
      "rewards/margins": 0.044403076171875,
      "rewards/rejected": -0.10811767727136612,
      "step": 25960
    },
    {
      "epoch": 0.9731876861966237,
      "grad_norm": 2.751526956547046,
      "learning_rate": 4.964254203006508e-07,
      "log_odds_chosen": 0.6026855707168579,
      "log_odds_ratio": -0.5640624761581421,
      "logits/chosen": -0.8119140863418579,
      "logits/rejected": -0.7105468511581421,
      "logps/chosen": -0.6615234613418579,
      "logps/rejected": -1.040429711341858,
      "loss": 0.7591,
      "nll_loss": 0.7041015625,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06617431342601776,
      "rewards/margins": 0.03796691820025444,
      "rewards/rejected": -0.10411377251148224,
      "step": 25970
    },
    {
      "epoch": 0.9735624215397876,
      "grad_norm": 2.590816748081946,
      "learning_rate": 4.963298711860517e-07,
      "log_odds_chosen": 0.613818347454071,
      "log_odds_ratio": -0.567089855670929,
      "logits/chosen": -0.7574218511581421,
      "logits/rejected": -0.6490234136581421,
      "logps/chosen": -0.7173827886581421,
      "logps/rejected": -1.1105468273162842,
      "loss": 0.7691,
      "nll_loss": 0.7359374761581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07177734375,
      "rewards/margins": 0.03925170749425888,
      "rewards/rejected": -0.111083984375,
      "step": 25980
    },
    {
      "epoch": 0.9739371568829515,
      "grad_norm": 2.855906146039251,
      "learning_rate": 4.962343772224578e-07,
      "log_odds_chosen": 0.432861328125,
      "log_odds_ratio": -0.611621081829071,
      "logits/chosen": -0.802929699420929,
      "logits/rejected": -0.6771484613418579,
      "logps/chosen": -0.6796875,
      "logps/rejected": -0.9330078363418579,
      "loss": 0.7804,
      "nll_loss": 0.716015636920929,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06794433295726776,
      "rewards/margins": 0.02538299560546875,
      "rewards/rejected": -0.09332275390625,
      "step": 25990
    },
    {
      "epoch": 0.9743118922261154,
      "grad_norm": 2.62225504955661,
      "learning_rate": 4.961389383568338e-07,
      "log_odds_chosen": 0.5316406488418579,
      "log_odds_ratio": -0.578808605670929,
      "logits/chosen": -0.78076171875,
      "logits/rejected": -0.711132824420929,
      "logps/chosen": -0.6728515625,
      "logps/rejected": -0.9947265386581421,
      "loss": 0.7687,
      "nll_loss": 0.697949230670929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06721191108226776,
      "rewards/margins": 0.03214874118566513,
      "rewards/rejected": -0.09940185397863388,
      "step": 26000
    },
    {
      "epoch": 0.9746866275692792,
      "grad_norm": 2.6037560286219334,
      "learning_rate": 4.960435545362163e-07,
      "log_odds_chosen": 0.5927734375,
      "log_odds_ratio": -0.5511718988418579,
      "logits/chosen": -0.83349609375,
      "logits/rejected": -0.6883789300918579,
      "logps/chosen": -0.657519519329071,
      "logps/rejected": -1.020898461341858,
      "loss": 0.7719,
      "nll_loss": 0.736132800579071,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06572265923023224,
      "rewards/margins": 0.03643493726849556,
      "rewards/rejected": -0.10219726711511612,
      "step": 26010
    },
    {
      "epoch": 0.975061362912443,
      "grad_norm": 3.20057567957263,
      "learning_rate": 4.959482257077131e-07,
      "log_odds_chosen": 0.41486817598342896,
      "log_odds_ratio": -0.6357421875,
      "logits/chosen": -0.727343738079071,
      "logits/rejected": -0.6617187261581421,
      "logps/chosen": -0.674023449420929,
      "logps/rejected": -0.955859363079071,
      "loss": 0.7634,
      "nll_loss": 0.7518554925918579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06739501655101776,
      "rewards/margins": 0.02820282056927681,
      "rewards/rejected": -0.0955810546875,
      "step": 26020
    },
    {
      "epoch": 0.975436098255607,
      "grad_norm": 2.901256433592597,
      "learning_rate": 4.958529518185029e-07,
      "log_odds_chosen": 0.5870116949081421,
      "log_odds_ratio": -0.5677734613418579,
      "logits/chosen": -0.777539074420929,
      "logits/rejected": -0.6412109136581421,
      "logps/chosen": -0.690625011920929,
      "logps/rejected": -1.0763671398162842,
      "loss": 0.7784,
      "nll_loss": 0.7373046875,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06905517727136612,
      "rewards/margins": 0.03867492824792862,
      "rewards/rejected": -0.10773925483226776,
      "step": 26030
    },
    {
      "epoch": 0.9758108335987709,
      "grad_norm": 2.9206715590921535,
      "learning_rate": 4.957577328158356e-07,
      "log_odds_chosen": 0.732714831829071,
      "log_odds_ratio": -0.535449206829071,
      "logits/chosen": -0.7679687738418579,
      "logits/rejected": -0.651562511920929,
      "logps/chosen": -0.6419922113418579,
      "logps/rejected": -1.1220703125,
      "loss": 0.8009,
      "nll_loss": 0.7152343988418579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.064208984375,
      "rewards/margins": 0.04797668382525444,
      "rewards/rejected": -0.11219482123851776,
      "step": 26040
    },
    {
      "epoch": 0.9761855689419348,
      "grad_norm": 2.6578962179775254,
      "learning_rate": 4.956625686470318e-07,
      "log_odds_chosen": 0.5711425542831421,
      "log_odds_ratio": -0.563769519329071,
      "logits/chosen": -0.8052734136581421,
      "logits/rejected": -0.688671886920929,
      "logps/chosen": -0.625292956829071,
      "logps/rejected": -0.979687511920929,
      "loss": 0.7637,
      "nll_loss": 0.725878894329071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.0625,
      "rewards/margins": 0.035385895520448685,
      "rewards/rejected": -0.097900390625,
      "step": 26050
    },
    {
      "epoch": 0.9765603042850987,
      "grad_norm": 2.729911206360954,
      "learning_rate": 4.95567459259483e-07,
      "log_odds_chosen": 0.553515613079071,
      "log_odds_ratio": -0.59716796875,
      "logits/chosen": -0.7835937738418579,
      "logits/rejected": -0.6747070550918579,
      "logps/chosen": -0.655566394329071,
      "logps/rejected": -1.02734375,
      "loss": 0.7821,
      "nll_loss": 0.716015636920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06557617336511612,
      "rewards/margins": 0.03708343580365181,
      "rewards/rejected": -0.10268554836511612,
      "step": 26060
    },
    {
      "epoch": 0.9769350396282626,
      "grad_norm": 3.098986407701719,
      "learning_rate": 4.954724046006516e-07,
      "log_odds_chosen": 0.4664306640625,
      "log_odds_ratio": -0.61083984375,
      "logits/chosen": -0.7720702886581421,
      "logits/rejected": -0.69091796875,
      "logps/chosen": -0.691210925579071,
      "logps/rejected": -0.980273425579071,
      "loss": 0.7861,
      "nll_loss": 0.7852538824081421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06905517727136612,
      "rewards/margins": 0.028966521844267845,
      "rewards/rejected": -0.09803466498851776,
      "step": 26070
    },
    {
      "epoch": 0.9773097749714265,
      "grad_norm": 3.04702028486125,
      "learning_rate": 4.953774046180699e-07,
      "log_odds_chosen": 0.653576672077179,
      "log_odds_ratio": -0.5376952886581421,
      "logits/chosen": -0.744140625,
      "logits/rejected": -0.631640613079071,
      "logps/chosen": -0.680957019329071,
      "logps/rejected": -1.0763671398162842,
      "loss": 0.7868,
      "nll_loss": 0.7457031011581421,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06800536811351776,
      "rewards/margins": 0.03957824781537056,
      "rewards/rejected": -0.10764160007238388,
      "step": 26080
    },
    {
      "epoch": 0.9776845103145904,
      "grad_norm": 2.888417143823001,
      "learning_rate": 4.952824592593411e-07,
      "log_odds_chosen": 0.520263671875,
      "log_odds_ratio": -0.609667956829071,
      "logits/chosen": -0.7523437738418579,
      "logits/rejected": -0.6688476800918579,
      "logps/chosen": -0.7099609375,
      "logps/rejected": -1.0498046875,
      "loss": 0.7749,
      "nll_loss": 0.701367199420929,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07100830227136612,
      "rewards/margins": 0.033931732177734375,
      "rewards/rejected": -0.10498046875,
      "step": 26090
    },
    {
      "epoch": 0.9780592456577543,
      "grad_norm": 2.977229351946179,
      "learning_rate": 4.951875684721383e-07,
      "log_odds_chosen": 0.47535401582717896,
      "log_odds_ratio": -0.596875011920929,
      "logits/chosen": -0.814648449420929,
      "logits/rejected": -0.6973632574081421,
      "logps/chosen": -0.704785168170929,
      "logps/rejected": -1.00390625,
      "loss": 0.7848,
      "nll_loss": 0.7396484613418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07048340141773224,
      "rewards/margins": 0.02987975999712944,
      "rewards/rejected": -0.10039062798023224,
      "step": 26100
    },
    {
      "epoch": 0.978433981000918,
      "grad_norm": 2.9150361854466404,
      "learning_rate": 4.950927322042051e-07,
      "log_odds_chosen": 0.37895506620407104,
      "log_odds_ratio": -0.6109374761581421,
      "logits/chosen": -0.7847656011581421,
      "logits/rejected": -0.7178710699081421,
      "logps/chosen": -0.657031238079071,
      "logps/rejected": -0.886914074420929,
      "loss": 0.783,
      "nll_loss": 0.7061523199081421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06574706733226776,
      "rewards/margins": 0.02295684814453125,
      "rewards/rejected": -0.0887451171875,
      "step": 26110
    },
    {
      "epoch": 0.978808716344082,
      "grad_norm": 2.8509556132176006,
      "learning_rate": 4.949979504033549e-07,
      "log_odds_chosen": 0.582958996295929,
      "log_odds_ratio": -0.571972668170929,
      "logits/chosen": -0.8111327886581421,
      "logits/rejected": -0.6820312738418579,
      "logps/chosen": -0.630175769329071,
      "logps/rejected": -1.0,
      "loss": 0.7541,
      "nll_loss": 0.6600586175918579,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06302490085363388,
      "rewards/margins": 0.03702697902917862,
      "rewards/rejected": -0.10000000149011612,
      "step": 26120
    },
    {
      "epoch": 0.9791834516872459,
      "grad_norm": 3.1824204360156787,
      "learning_rate": 4.949032230174711e-07,
      "log_odds_chosen": 0.55462646484375,
      "log_odds_ratio": -0.566699206829071,
      "logits/chosen": -0.798828125,
      "logits/rejected": -0.6981445550918579,
      "logps/chosen": -0.6226562261581421,
      "logps/rejected": -0.9564453363418579,
      "loss": 0.7526,
      "nll_loss": 0.693164050579071,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06228027492761612,
      "rewards/margins": 0.03333129733800888,
      "rewards/rejected": -0.09571532905101776,
      "step": 26130
    },
    {
      "epoch": 0.9795581870304098,
      "grad_norm": 2.3249796342107545,
      "learning_rate": 4.948085499945068e-07,
      "log_odds_chosen": 0.578906238079071,
      "log_odds_ratio": -0.5624023675918579,
      "logits/chosen": -0.851757824420929,
      "logits/rejected": -0.694140613079071,
      "logps/chosen": -0.678417980670929,
      "logps/rejected": -1.0496094226837158,
      "loss": 0.7642,
      "nll_loss": 0.7298828363418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06781005859375,
      "rewards/margins": 0.03714294359087944,
      "rewards/rejected": -0.10498046875,
      "step": 26140
    },
    {
      "epoch": 0.9799329223735737,
      "grad_norm": 2.7611385940162068,
      "learning_rate": 4.947139312824851e-07,
      "log_odds_chosen": 0.4395507872104645,
      "log_odds_ratio": -0.6103515625,
      "logits/chosen": -0.7828124761581421,
      "logits/rejected": -0.6529296636581421,
      "logps/chosen": -0.671582043170929,
      "logps/rejected": -0.970898449420929,
      "loss": 0.7504,
      "nll_loss": 0.706250011920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06716308742761612,
      "rewards/margins": 0.02991333045065403,
      "rewards/rejected": -0.0970458984375,
      "step": 26150
    },
    {
      "epoch": 0.9803076577167376,
      "grad_norm": 2.98952531814747,
      "learning_rate": 4.946193668294979e-07,
      "log_odds_chosen": 0.5150085687637329,
      "log_odds_ratio": -0.61083984375,
      "logits/chosen": -0.7767578363418579,
      "logits/rejected": -0.6827148199081421,
      "logps/chosen": -0.673144519329071,
      "logps/rejected": -1.0060546398162842,
      "loss": 0.7608,
      "nll_loss": 0.7447265386581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0673828125,
      "rewards/margins": 0.03321685642004013,
      "rewards/rejected": -0.10057373344898224,
      "step": 26160
    },
    {
      "epoch": 0.9806823930599015,
      "grad_norm": 2.680969869590159,
      "learning_rate": 4.945248565837074e-07,
      "log_odds_chosen": 0.5696045160293579,
      "log_odds_ratio": -0.584667980670929,
      "logits/chosen": -0.790820300579071,
      "logits/rejected": -0.7240234613418579,
      "logps/chosen": -0.6458984613418579,
      "logps/rejected": -1.0224609375,
      "loss": 0.7761,
      "nll_loss": 0.6871093511581421,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06463623046875,
      "rewards/margins": 0.037574004381895065,
      "rewards/rejected": -0.10228271782398224,
      "step": 26170
    },
    {
      "epoch": 0.9810571284030654,
      "grad_norm": 2.5799308412887667,
      "learning_rate": 4.944304004933445e-07,
      "log_odds_chosen": 0.39409178495407104,
      "log_odds_ratio": -0.629199206829071,
      "logits/chosen": -0.7279297113418579,
      "logits/rejected": -0.6812499761581421,
      "logps/chosen": -0.6396484375,
      "logps/rejected": -0.873828113079071,
      "loss": 0.7732,
      "nll_loss": 0.7298828363418579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06395263969898224,
      "rewards/margins": 0.02339935302734375,
      "rewards/rejected": -0.08735351264476776,
      "step": 26180
    },
    {
      "epoch": 0.9814318637462293,
      "grad_norm": 2.6276818119892567,
      "learning_rate": 4.943359985067098e-07,
      "log_odds_chosen": 0.4276184141635895,
      "log_odds_ratio": -0.6166015863418579,
      "logits/chosen": -0.7710937261581421,
      "logits/rejected": -0.6929687261581421,
      "logps/chosen": -0.6748046875,
      "logps/rejected": -0.941210925579071,
      "loss": 0.7821,
      "nll_loss": 0.73828125,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06748046725988388,
      "rewards/margins": 0.02663574181497097,
      "rewards/rejected": -0.09414062649011612,
      "step": 26190
    },
    {
      "epoch": 0.9818065990893932,
      "grad_norm": 2.945869293251328,
      "learning_rate": 4.942416505721723e-07,
      "log_odds_chosen": 0.5022338628768921,
      "log_odds_ratio": -0.579394519329071,
      "logits/chosen": -0.780078113079071,
      "logits/rejected": -0.6766601800918579,
      "logps/chosen": -0.65380859375,
      "logps/rejected": -0.9585937261581421,
      "loss": 0.759,
      "nll_loss": 0.6917968988418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06535644829273224,
      "rewards/margins": 0.03051299974322319,
      "rewards/rejected": -0.09584961086511612,
      "step": 26200
    },
    {
      "epoch": 0.982181334432557,
      "grad_norm": 3.1078496410457617,
      "learning_rate": 4.941473566381706e-07,
      "log_odds_chosen": 0.5812133550643921,
      "log_odds_ratio": -0.5748046636581421,
      "logits/chosen": -0.804492175579071,
      "logits/rejected": -0.685546875,
      "logps/chosen": -0.658007800579071,
      "logps/rejected": -1.0417969226837158,
      "loss": 0.7675,
      "nll_loss": 0.692187488079071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06577148288488388,
      "rewards/margins": 0.03840332105755806,
      "rewards/rejected": -0.1041259765625,
      "step": 26210
    },
    {
      "epoch": 0.9825560697757209,
      "grad_norm": 3.8672788265169102,
      "learning_rate": 4.940531166532117e-07,
      "log_odds_chosen": 0.47552490234375,
      "log_odds_ratio": -0.5707031488418579,
      "logits/chosen": -0.818164050579071,
      "logits/rejected": -0.703320324420929,
      "logps/chosen": -0.65380859375,
      "logps/rejected": -0.9300781488418579,
      "loss": 0.7758,
      "nll_loss": 0.7176758050918579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06533203274011612,
      "rewards/margins": 0.02761382982134819,
      "rewards/rejected": -0.09304199367761612,
      "step": 26220
    },
    {
      "epoch": 0.9829308051188848,
      "grad_norm": 2.4023019901893585,
      "learning_rate": 4.939589305658716e-07,
      "log_odds_chosen": 0.706103503704071,
      "log_odds_ratio": -0.5455566644668579,
      "logits/chosen": -0.735546886920929,
      "logits/rejected": -0.666699230670929,
      "logps/chosen": -0.6583007574081421,
      "logps/rejected": -1.0929687023162842,
      "loss": 0.7681,
      "nll_loss": 0.703320324420929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06580810248851776,
      "rewards/margins": 0.043470002710819244,
      "rewards/rejected": -0.10932616889476776,
      "step": 26230
    },
    {
      "epoch": 0.9833055404620487,
      "grad_norm": 2.932755438780389,
      "learning_rate": 4.938647983247948e-07,
      "log_odds_chosen": 0.5699218511581421,
      "log_odds_ratio": -0.5772460699081421,
      "logits/chosen": -0.766796886920929,
      "logits/rejected": -0.6656249761581421,
      "logps/chosen": -0.61572265625,
      "logps/rejected": -0.956347644329071,
      "loss": 0.7612,
      "nll_loss": 0.6807616949081421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06165771558880806,
      "rewards/margins": 0.03395233303308487,
      "rewards/rejected": -0.0955810546875,
      "step": 26240
    },
    {
      "epoch": 0.9836802758052126,
      "grad_norm": 3.13018219987805,
      "learning_rate": 4.937707198786941e-07,
      "log_odds_chosen": 0.6224120855331421,
      "log_odds_ratio": -0.54296875,
      "logits/chosen": -0.7757812738418579,
      "logits/rejected": -0.65673828125,
      "logps/chosen": -0.651074230670929,
      "logps/rejected": -1.020898461341858,
      "loss": 0.7636,
      "nll_loss": 0.6552734375,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06512451171875,
      "rewards/margins": 0.03700561448931694,
      "rewards/rejected": -0.10219726711511612,
      "step": 26250
    },
    {
      "epoch": 0.9840550111483765,
      "grad_norm": 3.1972337196852116,
      "learning_rate": 4.936766951763509e-07,
      "log_odds_chosen": 0.6044067144393921,
      "log_odds_ratio": -0.6097656488418579,
      "logits/chosen": -0.751171886920929,
      "logits/rejected": -0.615527331829071,
      "logps/chosen": -0.658203125,
      "logps/rejected": -1.0916016101837158,
      "loss": 0.7585,
      "nll_loss": 0.740039050579071,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06585693359375,
      "rewards/margins": 0.04342346265912056,
      "rewards/rejected": -0.10930176079273224,
      "step": 26260
    },
    {
      "epoch": 0.9844297464915404,
      "grad_norm": 2.7630694334381003,
      "learning_rate": 4.935827241666148e-07,
      "log_odds_chosen": 0.6861816644668579,
      "log_odds_ratio": -0.5420898199081421,
      "logits/chosen": -0.7081054449081421,
      "logits/rejected": -0.613574206829071,
      "logps/chosen": -0.644238293170929,
      "logps/rejected": -1.071679711341858,
      "loss": 0.766,
      "nll_loss": 0.660937488079071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06437988579273224,
      "rewards/margins": 0.042711637914180756,
      "rewards/rejected": -0.10711669921875,
      "step": 26270
    },
    {
      "epoch": 0.9848044818347043,
      "grad_norm": 3.204354808884691,
      "learning_rate": 4.934888067984035e-07,
      "log_odds_chosen": 0.6567627191543579,
      "log_odds_ratio": -0.539843738079071,
      "logits/chosen": -0.753125011920929,
      "logits/rejected": -0.6493164300918579,
      "logps/chosen": -0.668164074420929,
      "logps/rejected": -1.0673828125,
      "loss": 0.7538,
      "nll_loss": 0.761523425579071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06684570014476776,
      "rewards/margins": 0.03991546481847763,
      "rewards/rejected": -0.106689453125,
      "step": 26280
    },
    {
      "epoch": 0.9851792171778682,
      "grad_norm": 2.913790769840308,
      "learning_rate": 4.933949430207026e-07,
      "log_odds_chosen": 0.566943347454071,
      "log_odds_ratio": -0.5501953363418579,
      "logits/chosen": -0.7613281011581421,
      "logits/rejected": -0.661816418170929,
      "logps/chosen": -0.6290038824081421,
      "logps/rejected": -0.983203113079071,
      "loss": 0.764,
      "nll_loss": 0.6683593988418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06291504204273224,
      "rewards/margins": 0.035395048558712006,
      "rewards/rejected": -0.09831543266773224,
      "step": 26290
    },
    {
      "epoch": 0.9855539525210321,
      "grad_norm": 2.741188507171496,
      "learning_rate": 4.933011327825659e-07,
      "log_odds_chosen": 0.7344970703125,
      "log_odds_ratio": -0.5269531011581421,
      "logits/chosen": -0.740234375,
      "logits/rejected": -0.635058581829071,
      "logps/chosen": -0.64453125,
      "logps/rejected": -1.095117211341858,
      "loss": 0.7645,
      "nll_loss": 0.6904296875,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06439208984375,
      "rewards/margins": 0.04512329027056694,
      "rewards/rejected": -0.10959472507238388,
      "step": 26300
    },
    {
      "epoch": 0.985928687864196,
      "grad_norm": 2.8336850186966207,
      "learning_rate": 4.932073760331147e-07,
      "log_odds_chosen": 0.48322755098342896,
      "log_odds_ratio": -0.591601550579071,
      "logits/chosen": -0.7777343988418579,
      "logits/rejected": -0.6689453125,
      "logps/chosen": -0.65234375,
      "logps/rejected": -0.966015636920929,
      "loss": 0.7651,
      "nll_loss": 0.6910156011581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06525878608226776,
      "rewards/margins": 0.03137359768152237,
      "rewards/rejected": -0.09663085639476776,
      "step": 26310
    },
    {
      "epoch": 0.9863034232073598,
      "grad_norm": 3.047085172836358,
      "learning_rate": 4.931136727215378e-07,
      "log_odds_chosen": 0.600659191608429,
      "log_odds_ratio": -0.57275390625,
      "logits/chosen": -0.8177734613418579,
      "logits/rejected": -0.673828125,
      "logps/chosen": -0.6527343988418579,
      "logps/rejected": -1.0499999523162842,
      "loss": 0.7716,
      "nll_loss": 0.683886706829071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06529541313648224,
      "rewards/margins": 0.039734650403261185,
      "rewards/rejected": -0.10509033501148224,
      "step": 26320
    },
    {
      "epoch": 0.9866781585505237,
      "grad_norm": 3.5250710086394115,
      "learning_rate": 4.930200227970923e-07,
      "log_odds_chosen": 0.588787853717804,
      "log_odds_ratio": -0.565625011920929,
      "logits/chosen": -0.7529296875,
      "logits/rejected": -0.6288086175918579,
      "logps/chosen": -0.6451171636581421,
      "logps/rejected": -1.0250976085662842,
      "loss": 0.7882,
      "nll_loss": 0.743945300579071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06452636420726776,
      "rewards/margins": 0.037970732897520065,
      "rewards/rejected": -0.1025390625,
      "step": 26330
    },
    {
      "epoch": 0.9870528938936876,
      "grad_norm": 3.0772008928505343,
      "learning_rate": 4.929264262091018e-07,
      "log_odds_chosen": 0.53729248046875,
      "log_odds_ratio": -0.5711914300918579,
      "logits/chosen": -0.7542968988418579,
      "logits/rejected": -0.611328125,
      "logps/chosen": -0.6504882574081421,
      "logps/rejected": -0.990234375,
      "loss": 0.7822,
      "nll_loss": 0.6822265386581421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06503906100988388,
      "rewards/margins": 0.0340728759765625,
      "rewards/rejected": -0.09909667819738388,
      "step": 26340
    },
    {
      "epoch": 0.9874276292368515,
      "grad_norm": 2.629878333890411,
      "learning_rate": 4.928328829069579e-07,
      "log_odds_chosen": 0.5499267578125,
      "log_odds_ratio": -0.595507800579071,
      "logits/chosen": -0.790234386920929,
      "logits/rejected": -0.7027343511581421,
      "logps/chosen": -0.669726550579071,
      "logps/rejected": -1.058007836341858,
      "loss": 0.7628,
      "nll_loss": 0.693359375,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.06695556640625,
      "rewards/margins": 0.038878630846738815,
      "rewards/rejected": -0.10581054538488388,
      "step": 26350
    },
    {
      "epoch": 0.9878023645800154,
      "grad_norm": 3.8430606332905963,
      "learning_rate": 4.92739392840119e-07,
      "log_odds_chosen": 0.4759521484375,
      "log_odds_ratio": -0.6009765863418579,
      "logits/chosen": -0.8076171875,
      "logits/rejected": -0.6792968511581421,
      "logps/chosen": -0.702929675579071,
      "logps/rejected": -0.986132800579071,
      "loss": 0.773,
      "nll_loss": 0.7162109613418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07030029594898224,
      "rewards/margins": 0.028336334973573685,
      "rewards/rejected": -0.09860839694738388,
      "step": 26360
    },
    {
      "epoch": 0.9881770999231793,
      "grad_norm": 3.3216716222727087,
      "learning_rate": 4.926459559581109e-07,
      "log_odds_chosen": 0.39543455839157104,
      "log_odds_ratio": -0.624707043170929,
      "logits/chosen": -0.720410168170929,
      "logits/rejected": -0.63671875,
      "logps/chosen": -0.6400390863418579,
      "logps/rejected": -0.88525390625,
      "loss": 0.7816,
      "nll_loss": 0.740527331829071,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.0640869140625,
      "rewards/margins": 0.02453308179974556,
      "rewards/rejected": -0.08853759616613388,
      "step": 26370
    },
    {
      "epoch": 0.9885518352663432,
      "grad_norm": 2.7224397119791552,
      "learning_rate": 4.92552572210526e-07,
      "log_odds_chosen": 0.43621826171875,
      "log_odds_ratio": -0.624316394329071,
      "logits/chosen": -0.72314453125,
      "logits/rejected": -0.666015625,
      "logps/chosen": -0.7137695550918579,
      "logps/rejected": -0.9740234613418579,
      "loss": 0.7689,
      "nll_loss": 0.768750011920929,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0714111328125,
      "rewards/margins": 0.02596893347799778,
      "rewards/rejected": -0.09743652492761612,
      "step": 26380
    },
    {
      "epoch": 0.9889265706095071,
      "grad_norm": 2.463274468079282,
      "learning_rate": 4.924592415470239e-07,
      "log_odds_chosen": 0.6171509027481079,
      "log_odds_ratio": -0.548828125,
      "logits/chosen": -0.7447265386581421,
      "logits/rejected": -0.662890613079071,
      "logps/chosen": -0.6431640386581421,
      "logps/rejected": -1.029296875,
      "loss": 0.7709,
      "nll_loss": 0.67578125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06429443508386612,
      "rewards/margins": 0.03864746168255806,
      "rewards/rejected": -0.10295410454273224,
      "step": 26390
    },
    {
      "epoch": 0.989301305952671,
      "grad_norm": 2.9815499912444743,
      "learning_rate": 4.923659639173309e-07,
      "log_odds_chosen": 0.5165771245956421,
      "log_odds_ratio": -0.5843750238418579,
      "logits/chosen": -0.7833007574081421,
      "logits/rejected": -0.653613269329071,
      "logps/chosen": -0.6978515386581421,
      "logps/rejected": -1.017187476158142,
      "loss": 0.7706,
      "nll_loss": 0.772265613079071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06977538764476776,
      "rewards/margins": 0.031982421875,
      "rewards/rejected": -0.10178222507238388,
      "step": 26400
    },
    {
      "epoch": 0.9896760412958349,
      "grad_norm": 3.0528572288332168,
      "learning_rate": 4.922727392712398e-07,
      "log_odds_chosen": 0.5445922613143921,
      "log_odds_ratio": -0.5953124761581421,
      "logits/chosen": -0.7513672113418579,
      "logits/rejected": -0.674609363079071,
      "logps/chosen": -0.6572265625,
      "logps/rejected": -0.994824230670929,
      "loss": 0.7724,
      "nll_loss": 0.700878918170929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06575927883386612,
      "rewards/margins": 0.033712007105350494,
      "rewards/rejected": -0.09941406548023224,
      "step": 26410
    },
    {
      "epoch": 0.9900507766389987,
      "grad_norm": 2.919868609181986,
      "learning_rate": 4.921795675586098e-07,
      "log_odds_chosen": 0.43513184785842896,
      "log_odds_ratio": -0.6044921875,
      "logits/chosen": -0.744335949420929,
      "logits/rejected": -0.6357421875,
      "logps/chosen": -0.6429687738418579,
      "logps/rejected": -0.9330078363418579,
      "loss": 0.7809,
      "nll_loss": 0.748046875,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06423339992761612,
      "rewards/margins": 0.02904205396771431,
      "rewards/rejected": -0.09333495795726776,
      "step": 26420
    },
    {
      "epoch": 0.9904255119821626,
      "grad_norm": 3.1313806172876677,
      "learning_rate": 4.920864487293668e-07,
      "log_odds_chosen": 0.4169067442417145,
      "log_odds_ratio": -0.6075195074081421,
      "logits/chosen": -0.790234386920929,
      "logits/rejected": -0.6610351800918579,
      "logps/chosen": -0.665820300579071,
      "logps/rejected": -0.9369140863418579,
      "loss": 0.7691,
      "nll_loss": 0.734179675579071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06656493991613388,
      "rewards/margins": 0.02712402306497097,
      "rewards/rejected": -0.09375,
      "step": 26430
    },
    {
      "epoch": 0.9908002473253265,
      "grad_norm": 2.8650144226742746,
      "learning_rate": 4.919933827335031e-07,
      "log_odds_chosen": 0.7266601324081421,
      "log_odds_ratio": -0.517285168170929,
      "logits/chosen": -0.756054699420929,
      "logits/rejected": -0.628613293170929,
      "logps/chosen": -0.671875,
      "logps/rejected": -1.1318359375,
      "loss": 0.7696,
      "nll_loss": 0.737500011920929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06717529147863388,
      "rewards/margins": 0.04608154296875,
      "rewards/rejected": -0.11318359524011612,
      "step": 26440
    },
    {
      "epoch": 0.9911749826684904,
      "grad_norm": 3.8466536671507647,
      "learning_rate": 4.919003695210765e-07,
      "log_odds_chosen": 0.3908447325229645,
      "log_odds_ratio": -0.630566418170929,
      "logits/chosen": -0.745898425579071,
      "logits/rejected": -0.634570300579071,
      "logps/chosen": -0.630859375,
      "logps/rejected": -0.8778320550918579,
      "loss": 0.7695,
      "nll_loss": 0.679980456829071,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0631103515625,
      "rewards/margins": 0.024703215807676315,
      "rewards/rejected": -0.08782958984375,
      "step": 26450
    },
    {
      "epoch": 0.9915497180116543,
      "grad_norm": 2.259062643854322,
      "learning_rate": 4.918074090422117e-07,
      "log_odds_chosen": 0.45427244901657104,
      "log_odds_ratio": -0.6014648675918579,
      "logits/chosen": -0.7640625238418579,
      "logits/rejected": -0.6460937261581421,
      "logps/chosen": -0.64501953125,
      "logps/rejected": -0.905078113079071,
      "loss": 0.7508,
      "nll_loss": 0.668164074420929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06452636420726776,
      "rewards/margins": 0.02596435509622097,
      "rewards/rejected": -0.09050293266773224,
      "step": 26460
    },
    {
      "epoch": 0.9919244533548182,
      "grad_norm": 2.3558039707008063,
      "learning_rate": 4.917145012470986e-07,
      "log_odds_chosen": 0.33935546875,
      "log_odds_ratio": -0.65771484375,
      "logits/chosen": -0.783984363079071,
      "logits/rejected": -0.6973632574081421,
      "logps/chosen": -0.6832031011581421,
      "logps/rejected": -0.88671875,
      "loss": 0.7662,
      "nll_loss": 0.7193359136581421,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06832275539636612,
      "rewards/margins": 0.02037963829934597,
      "rewards/rejected": -0.08865966647863388,
      "step": 26470
    },
    {
      "epoch": 0.9922991886979821,
      "grad_norm": 2.7457685952510937,
      "learning_rate": 4.916216460859938e-07,
      "log_odds_chosen": 0.49560546875,
      "log_odds_ratio": -0.595019519329071,
      "logits/chosen": -0.7582031488418579,
      "logits/rejected": -0.651171863079071,
      "logps/chosen": -0.699023425579071,
      "logps/rejected": -1.026953101158142,
      "loss": 0.7704,
      "nll_loss": 0.7378906011581421,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06983642280101776,
      "rewards/margins": 0.03288726881146431,
      "rewards/rejected": -0.10274658352136612,
      "step": 26480
    },
    {
      "epoch": 0.992673924041146,
      "grad_norm": 2.855587021850054,
      "learning_rate": 4.915288435092186e-07,
      "log_odds_chosen": 0.537463366985321,
      "log_odds_ratio": -0.602832019329071,
      "logits/chosen": -0.7572265863418579,
      "logits/rejected": -0.668164074420929,
      "logps/chosen": -0.6578124761581421,
      "logps/rejected": -0.979687511920929,
      "loss": 0.7901,
      "nll_loss": 0.7041015625,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06578369438648224,
      "rewards/margins": 0.03216094896197319,
      "rewards/rejected": -0.09792480617761612,
      "step": 26490
    },
    {
      "epoch": 0.9930486593843099,
      "grad_norm": 3.0289950960528396,
      "learning_rate": 4.914360934671609e-07,
      "log_odds_chosen": 0.4758056700229645,
      "log_odds_ratio": -0.59765625,
      "logits/chosen": -0.7308593988418579,
      "logits/rejected": -0.6337890625,
      "logps/chosen": -0.63037109375,
      "logps/rejected": -0.9271484613418579,
      "loss": 0.7607,
      "nll_loss": 0.660351574420929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06308593600988388,
      "rewards/margins": 0.02960510179400444,
      "rewards/rejected": -0.0926513671875,
      "step": 26500
    },
    {
      "epoch": 0.9934233947274738,
      "grad_norm": 2.717027572382723,
      "learning_rate": 4.913433959102734e-07,
      "log_odds_chosen": 0.4775024354457855,
      "log_odds_ratio": -0.5931640863418579,
      "logits/chosen": -0.766406238079071,
      "logits/rejected": -0.6849609613418579,
      "logps/chosen": -0.6690429449081421,
      "logps/rejected": -0.960156261920929,
      "loss": 0.781,
      "nll_loss": 0.706738293170929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06695556640625,
      "rewards/margins": 0.02908935584127903,
      "rewards/rejected": -0.09603271633386612,
      "step": 26510
    },
    {
      "epoch": 0.9937981300706377,
      "grad_norm": 3.063229792289311,
      "learning_rate": 4.912507507890746e-07,
      "log_odds_chosen": 0.5741211175918579,
      "log_odds_ratio": -0.5516601800918579,
      "logits/chosen": -0.751171886920929,
      "logits/rejected": -0.6424804925918579,
      "logps/chosen": -0.6292968988418579,
      "logps/rejected": -0.985546886920929,
      "loss": 0.7438,
      "nll_loss": 0.691210925579071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06292724609375,
      "rewards/margins": 0.035697173327207565,
      "rewards/rejected": -0.09864501655101776,
      "step": 26520
    },
    {
      "epoch": 0.9941728654138015,
      "grad_norm": 2.9098830922322896,
      "learning_rate": 4.911581580541482e-07,
      "log_odds_chosen": 0.5486205816268921,
      "log_odds_ratio": -0.5703125,
      "logits/chosen": -0.7295898199081421,
      "logits/rejected": -0.644335925579071,
      "logps/chosen": -0.619140625,
      "logps/rejected": -0.9544922113418579,
      "loss": 0.7592,
      "nll_loss": 0.64990234375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06202392652630806,
      "rewards/margins": 0.03348999097943306,
      "rewards/rejected": -0.09544678032398224,
      "step": 26530
    },
    {
      "epoch": 0.9945476007569654,
      "grad_norm": 2.840685026218449,
      "learning_rate": 4.910656176561429e-07,
      "log_odds_chosen": 0.527148425579071,
      "log_odds_ratio": -0.597851574420929,
      "logits/chosen": -0.7720702886581421,
      "logits/rejected": -0.68115234375,
      "logps/chosen": -0.6578124761581421,
      "logps/rejected": -0.946093738079071,
      "loss": 0.7654,
      "nll_loss": 0.7251952886581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06583251804113388,
      "rewards/margins": 0.02882537804543972,
      "rewards/rejected": -0.09470214694738388,
      "step": 26540
    },
    {
      "epoch": 0.9949223361001293,
      "grad_norm": 2.49929497990558,
      "learning_rate": 4.909731295457724e-07,
      "log_odds_chosen": 0.609912097454071,
      "log_odds_ratio": -0.5633789300918579,
      "logits/chosen": -0.7222656011581421,
      "logits/rejected": -0.61376953125,
      "logps/chosen": -0.667285144329071,
      "logps/rejected": -1.074804663658142,
      "loss": 0.7714,
      "nll_loss": 0.718945324420929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06671142578125,
      "rewards/margins": 0.04067077487707138,
      "rewards/rejected": -0.10737304389476776,
      "step": 26550
    },
    {
      "epoch": 0.9952970714432932,
      "grad_norm": 3.1900408950657444,
      "learning_rate": 4.90880693673816e-07,
      "log_odds_chosen": 0.457763671875,
      "log_odds_ratio": -0.597460925579071,
      "logits/chosen": -0.75830078125,
      "logits/rejected": -0.666796863079071,
      "logps/chosen": -0.645800769329071,
      "logps/rejected": -0.9398437738418579,
      "loss": 0.7653,
      "nll_loss": 0.704296886920929,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06464843451976776,
      "rewards/margins": 0.02935028076171875,
      "rewards/rejected": -0.09396972507238388,
      "step": 26560
    },
    {
      "epoch": 0.9956718067864571,
      "grad_norm": 2.555845639771438,
      "learning_rate": 4.907883099911169e-07,
      "log_odds_chosen": 0.41459959745407104,
      "log_odds_ratio": -0.6158202886581421,
      "logits/chosen": -0.727343738079071,
      "logits/rejected": -0.640429675579071,
      "logps/chosen": -0.65478515625,
      "logps/rejected": -0.91796875,
      "loss": 0.7529,
      "nll_loss": 0.7154296636581421,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06552734225988388,
      "rewards/margins": 0.02628784254193306,
      "rewards/rejected": -0.09176025539636612,
      "step": 26570
    },
    {
      "epoch": 0.996046542129621,
      "grad_norm": 2.945797278482257,
      "learning_rate": 4.90695978448584e-07,
      "log_odds_chosen": 0.4608520567417145,
      "log_odds_ratio": -0.6376953125,
      "logits/chosen": -0.766796886920929,
      "logits/rejected": -0.69921875,
      "logps/chosen": -0.7310546636581421,
      "logps/rejected": -1.0412108898162842,
      "loss": 0.7786,
      "nll_loss": 0.736035168170929,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07309570163488388,
      "rewards/margins": 0.03091583214700222,
      "rewards/rejected": -0.10395507514476776,
      "step": 26580
    },
    {
      "epoch": 0.9964212774727849,
      "grad_norm": 3.253393356260141,
      "learning_rate": 4.906036989971901e-07,
      "log_odds_chosen": 0.28242188692092896,
      "log_odds_ratio": -0.665234386920929,
      "logits/chosen": -0.7188476324081421,
      "logits/rejected": -0.6695312261581421,
      "logps/chosen": -0.6675781011581421,
      "logps/rejected": -0.8392578363418579,
      "loss": 0.7748,
      "nll_loss": 0.7254883050918579,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06671142578125,
      "rewards/margins": 0.01723175123333931,
      "rewards/rejected": -0.08393554389476776,
      "step": 26590
    },
    {
      "epoch": 0.9967960128159488,
      "grad_norm": 2.3052058943088993,
      "learning_rate": 4.905114715879726e-07,
      "log_odds_chosen": 0.39519041776657104,
      "log_odds_ratio": -0.6385742425918579,
      "logits/chosen": -0.7142578363418579,
      "logits/rejected": -0.6465820074081421,
      "logps/chosen": -0.681640625,
      "logps/rejected": -0.940234363079071,
      "loss": 0.7701,
      "nll_loss": 0.751171886920929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06817626953125,
      "rewards/margins": 0.02592620812356472,
      "rewards/rejected": -0.09403076022863388,
      "step": 26600
    },
    {
      "epoch": 0.9971707481591127,
      "grad_norm": 3.1968142291681767,
      "learning_rate": 4.904192961720338e-07,
      "log_odds_chosen": 0.612438976764679,
      "log_odds_ratio": -0.5440429449081421,
      "logits/chosen": -0.745800793170929,
      "logits/rejected": -0.648730456829071,
      "logps/chosen": -0.6595703363418579,
      "logps/rejected": -1.0361328125,
      "loss": 0.7614,
      "nll_loss": 0.7132812738418579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06594238430261612,
      "rewards/margins": 0.037644196301698685,
      "rewards/rejected": -0.10368652641773224,
      "step": 26610
    },
    {
      "epoch": 0.9975454835022766,
      "grad_norm": 2.340580731420119,
      "learning_rate": 4.903271727005396e-07,
      "log_odds_chosen": 0.38151854276657104,
      "log_odds_ratio": -0.623339831829071,
      "logits/chosen": -0.801953136920929,
      "logits/rejected": -0.678906261920929,
      "logps/chosen": -0.655566394329071,
      "logps/rejected": -0.8955078125,
      "loss": 0.765,
      "nll_loss": 0.688671886920929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06556396186351776,
      "rewards/margins": 0.02399597130715847,
      "rewards/rejected": -0.08957519382238388,
      "step": 26620
    },
    {
      "epoch": 0.9979202188454404,
      "grad_norm": 2.7233461796383973,
      "learning_rate": 4.902351011247209e-07,
      "log_odds_chosen": 0.40642088651657104,
      "log_odds_ratio": -0.60986328125,
      "logits/chosen": -0.7330077886581421,
      "logits/rejected": -0.6537109613418579,
      "logps/chosen": -0.6812499761581421,
      "logps/rejected": -0.901171863079071,
      "loss": 0.7728,
      "nll_loss": 0.747363269329071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.06812743842601776,
      "rewards/margins": 0.02205505408346653,
      "rewards/rejected": -0.09019775688648224,
      "step": 26630
    },
    {
      "epoch": 0.9982949541886043,
      "grad_norm": 2.8774861924241266,
      "learning_rate": 4.901430813958718e-07,
      "log_odds_chosen": 0.558947741985321,
      "log_odds_ratio": -0.5718749761581421,
      "logits/chosen": -0.7330077886581421,
      "logits/rejected": -0.630859375,
      "logps/chosen": -0.6065429449081421,
      "logps/rejected": -0.951367199420929,
      "loss": 0.7776,
      "nll_loss": 0.6675781011581421,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06065673753619194,
      "rewards/margins": 0.03444213792681694,
      "rewards/rejected": -0.09506835788488388,
      "step": 26640
    },
    {
      "epoch": 0.9986696895317682,
      "grad_norm": 2.4232542146288703,
      "learning_rate": 4.90051113465351e-07,
      "log_odds_chosen": 0.620922863483429,
      "log_odds_ratio": -0.5497070550918579,
      "logits/chosen": -0.75390625,
      "logits/rejected": -0.6661132574081421,
      "logps/chosen": -0.691113293170929,
      "logps/rejected": -1.0798828601837158,
      "loss": 0.7855,
      "nll_loss": 0.71728515625,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06912841647863388,
      "rewards/margins": 0.03881530836224556,
      "rewards/rejected": -0.10797119140625,
      "step": 26650
    },
    {
      "epoch": 0.9990444248749321,
      "grad_norm": 2.3376148312083096,
      "learning_rate": 4.899591972845809e-07,
      "log_odds_chosen": 0.752636730670929,
      "log_odds_ratio": -0.544921875,
      "logits/chosen": -0.790234386920929,
      "logits/rejected": -0.6571289300918579,
      "logps/chosen": -0.6836913824081421,
      "logps/rejected": -1.1904296875,
      "loss": 0.7628,
      "nll_loss": 0.7373046875,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06831054389476776,
      "rewards/margins": 0.05070800706744194,
      "rewards/rejected": -0.11909179389476776,
      "step": 26660
    },
    {
      "epoch": 0.999419160218096,
      "grad_norm": 3.2270969394537925,
      "learning_rate": 4.898673328050477e-07,
      "log_odds_chosen": 0.5020507574081421,
      "log_odds_ratio": -0.59716796875,
      "logits/chosen": -0.7476562261581421,
      "logits/rejected": -0.63671875,
      "logps/chosen": -0.704882800579071,
      "logps/rejected": -1.029882788658142,
      "loss": 0.7738,
      "nll_loss": 0.73876953125,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07053222507238388,
      "rewards/margins": 0.03243865817785263,
      "rewards/rejected": -0.10297851264476776,
      "step": 26670
    },
    {
      "epoch": 0.9997938955612599,
      "grad_norm": 2.570887225219163,
      "learning_rate": 4.897755199783008e-07,
      "log_odds_chosen": 0.6326659917831421,
      "log_odds_ratio": -0.5394531488418579,
      "logits/chosen": -0.7562500238418579,
      "logits/rejected": -0.6175781488418579,
      "logps/chosen": -0.671679675579071,
      "logps/rejected": -1.0613281726837158,
      "loss": 0.7775,
      "nll_loss": 0.735546886920929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06715087592601776,
      "rewards/margins": 0.03895263746380806,
      "rewards/rejected": -0.10612793266773224,
      "step": 26680
    },
    {
      "epoch": 0.9999812632328418,
      "step": 26685,
      "total_flos": 0.0,
      "train_loss": 0.0,
      "train_runtime": 23.1532,
      "train_samples_per_second": 147525.725,
      "train_steps_per_second": 1152.538
    }
  ],
  "logging_steps": 10,
  "max_steps": 26685,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 5000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}