{
  "best_metric": 1.1802747249603271,
  "best_model_checkpoint": "models/llama3.2-3b-orpo-coarse/checkpoint-5000",
  "epoch": 1.0,
  "eval_steps": 5000,
  "global_step": 13178,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0007588404917286386,
      "grad_norm": 16.28905678970287,
      "learning_rate": 8e-07,
      "log_odds_chosen": 0.6555420160293579,
      "log_odds_ratio": -0.68408203125,
      "logits/chosen": -1.0910155773162842,
      "logits/rejected": -1.1103515625,
      "logps/chosen": -2.21875,
      "logps/rejected": -2.825000047683716,
      "loss": 2.518,
      "nll_loss": 2.332812547683716,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.22182616591453552,
      "rewards/margins": 0.060699462890625,
      "rewards/rejected": -0.28266602754592896,
      "step": 10
    },
    {
      "epoch": 0.0015176809834572772,
      "grad_norm": 8.977044251578562,
      "learning_rate": 1.6e-06,
      "log_odds_chosen": 0.5552002191543579,
      "log_odds_ratio": -0.7049804925918579,
      "logits/chosen": -1.138085961341858,
      "logits/rejected": -1.131445288658142,
      "logps/chosen": -2.3125,
      "logps/rejected": -2.8218750953674316,
      "loss": 2.3322,
      "nll_loss": 2.2734375,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.23154297471046448,
      "rewards/margins": 0.0509796142578125,
      "rewards/rejected": -0.28271484375,
      "step": 20
    },
    {
      "epoch": 0.002276521475185916,
      "grad_norm": 3.4584431033537384,
      "learning_rate": 2.4e-06,
      "log_odds_chosen": 0.6031738519668579,
      "log_odds_ratio": -0.614941418170929,
      "logits/chosen": -1.235742211341858,
      "logits/rejected": -1.194726586341858,
      "logps/chosen": -1.784765601158142,
      "logps/rejected": -2.30078125,
      "loss": 1.9747,
      "nll_loss": 1.8878905773162842,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.178466796875,
      "rewards/margins": 0.05157470703125,
      "rewards/rejected": -0.22998046875,
      "step": 30
    },
    {
      "epoch": 0.0030353619669145544,
      "grad_norm": 2.779230061008798,
      "learning_rate": 3.2e-06,
      "log_odds_chosen": 0.37614136934280396,
      "log_odds_ratio": -0.7298828363418579,
      "logits/chosen": -1.0929687023162842,
      "logits/rejected": -1.0183594226837158,
      "logps/chosen": -1.552734375,
      "logps/rejected": -1.8781249523162842,
      "loss": 1.795,
      "nll_loss": 1.745703101158142,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15510253608226776,
      "rewards/margins": 0.03264465183019638,
      "rewards/rejected": -0.187744140625,
      "step": 40
    },
    {
      "epoch": 0.0037942024586431933,
      "grad_norm": 2.5592134050477653,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.559399425983429,
      "log_odds_ratio": -0.5757812261581421,
      "logits/chosen": -0.9652343988418579,
      "logits/rejected": -0.919726550579071,
      "logps/chosen": -1.413671851158142,
      "logps/rejected": -1.8718750476837158,
      "loss": 1.7396,
      "nll_loss": 1.596093773841858,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.141357421875,
      "rewards/margins": 0.04580993577837944,
      "rewards/rejected": -0.18730469048023224,
      "step": 50
    },
    {
      "epoch": 0.004553042950371832,
      "grad_norm": 2.8450488945476566,
      "learning_rate": 4.8e-06,
      "log_odds_chosen": 0.5027710199356079,
      "log_odds_ratio": -0.5972656011581421,
      "logits/chosen": -1.0324218273162842,
      "logits/rejected": -0.975390613079071,
      "logps/chosen": -1.277929663658142,
      "logps/rejected": -1.692968726158142,
      "loss": 1.6585,
      "nll_loss": 1.4763672351837158,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.12788085639476776,
      "rewards/margins": 0.04137573391199112,
      "rewards/rejected": -0.16914062201976776,
      "step": 60
    },
    {
      "epoch": 0.005311883442100471,
      "grad_norm": 2.4981511166293764,
      "learning_rate": 5.6e-06,
      "log_odds_chosen": 0.5220092535018921,
      "log_odds_ratio": -0.5845702886581421,
      "logits/chosen": -1.106054663658142,
      "logits/rejected": -1.019140601158142,
      "logps/chosen": -1.224218726158142,
      "logps/rejected": -1.671484351158142,
      "loss": 1.6213,
      "nll_loss": 1.501562476158142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.12246093899011612,
      "rewards/margins": 0.0447998046875,
      "rewards/rejected": -0.167236328125,
      "step": 70
    },
    {
      "epoch": 0.006070723933829109,
      "grad_norm": 2.6645809181218567,
      "learning_rate": 6.4e-06,
      "log_odds_chosen": 0.555920422077179,
      "log_odds_ratio": -0.568652331829071,
      "logits/chosen": -1.048437476158142,
      "logits/rejected": -0.991992175579071,
      "logps/chosen": -1.229882836341858,
      "logps/rejected": -1.6925780773162842,
      "loss": 1.5998,
      "nll_loss": 1.505273461341858,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.12290038913488388,
      "rewards/margins": 0.04631652683019638,
      "rewards/rejected": -0.16923828423023224,
      "step": 80
    },
    {
      "epoch": 0.006829564425557748,
      "grad_norm": 2.467324514034339,
      "learning_rate": 7.2e-06,
      "log_odds_chosen": 0.3549560606479645,
      "log_odds_ratio": -0.6615234613418579,
      "logits/chosen": -1.115625023841858,
      "logits/rejected": -1.0144531726837158,
      "logps/chosen": -1.228906273841858,
      "logps/rejected": -1.5417969226837158,
      "loss": 1.51,
      "nll_loss": 1.5164062976837158,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.1229248046875,
      "rewards/margins": 0.031341552734375,
      "rewards/rejected": -0.154296875,
      "step": 90
    },
    {
      "epoch": 0.007588404917286387,
      "grad_norm": 2.47471355596972,
      "learning_rate": 8e-06,
      "log_odds_chosen": 0.39606934785842896,
      "log_odds_ratio": -0.6527343988418579,
      "logits/chosen": -1.009179711341858,
      "logits/rejected": -0.9609375,
      "logps/chosen": -1.2267577648162842,
      "logps/rejected": -1.552734375,
      "loss": 1.5495,
      "nll_loss": 1.5304687023162842,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.12258300930261612,
      "rewards/margins": 0.03266448900103569,
      "rewards/rejected": -0.15522460639476776,
      "step": 100
    },
    {
      "epoch": 0.008347245409015025,
      "grad_norm": 2.02338598823769,
      "learning_rate": 7.627700713964738e-06,
      "log_odds_chosen": 0.5322631597518921,
      "log_odds_ratio": -0.566601574420929,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.888867199420929,
      "logps/chosen": -1.1130859851837158,
      "logps/rejected": -1.5285155773162842,
      "loss": 1.4784,
      "nll_loss": 1.435937523841858,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11125488579273224,
      "rewards/margins": 0.04149017482995987,
      "rewards/rejected": -0.15275879204273224,
      "step": 110
    },
    {
      "epoch": 0.009106085900743664,
      "grad_norm": 1.7600136940465556,
      "learning_rate": 7.3029674334022146e-06,
      "log_odds_chosen": 0.585436999797821,
      "log_odds_ratio": -0.55029296875,
      "logits/chosen": -1.0187499523162842,
      "logits/rejected": -0.9371093511581421,
      "logps/chosen": -1.0310547351837158,
      "logps/rejected": -1.4699218273162842,
      "loss": 1.4271,
      "nll_loss": 1.293554663658142,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.10307617485523224,
      "rewards/margins": 0.0438995361328125,
      "rewards/rejected": -0.14692382514476776,
      "step": 120
    },
    {
      "epoch": 0.009864926392472302,
      "grad_norm": 1.6794517773793187,
      "learning_rate": 7.016464154456233e-06,
      "log_odds_chosen": 0.35838621854782104,
      "log_odds_ratio": -0.6317383050918579,
      "logits/chosen": -0.9859374761581421,
      "logits/rejected": -0.9013671875,
      "logps/chosen": -1.074609398841858,
      "logps/rejected": -1.357031226158142,
      "loss": 1.4294,
      "nll_loss": 1.3332030773162842,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.10749511420726776,
      "rewards/margins": 0.02816314622759819,
      "rewards/rejected": -0.1356201171875,
      "step": 130
    },
    {
      "epoch": 0.010623766884200941,
      "grad_norm": 1.515687206113085,
      "learning_rate": 6.7612340378281325e-06,
      "log_odds_chosen": 0.49846190214157104,
      "log_odds_ratio": -0.5927734375,
      "logits/chosen": -0.960742175579071,
      "logits/rejected": -0.9111328125,
      "logps/chosen": -0.993359386920929,
      "logps/rejected": -1.381445288658142,
      "loss": 1.3889,
      "nll_loss": 1.2716796398162842,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09931640326976776,
      "rewards/margins": 0.03876953199505806,
      "rewards/rejected": -0.13808593153953552,
      "step": 140
    },
    {
      "epoch": 0.01138260737592958,
      "grad_norm": 1.5746644221408093,
      "learning_rate": 6.531972647421809e-06,
      "log_odds_chosen": 0.4237060546875,
      "log_odds_ratio": -0.609375,
      "logits/chosen": -0.996874988079071,
      "logits/rejected": -0.950976550579071,
      "logps/chosen": -0.9847656488418579,
      "logps/rejected": -1.307031273841858,
      "loss": 1.3694,
      "nll_loss": 1.302343726158142,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09843750298023224,
      "rewards/margins": 0.032186128199100494,
      "rewards/rejected": -0.13068847358226776,
      "step": 150
    },
    {
      "epoch": 0.012141447867658217,
      "grad_norm": 1.7303046396071229,
      "learning_rate": 6.3245553203367575e-06,
      "log_odds_chosen": 0.4165405333042145,
      "log_odds_ratio": -0.6290527582168579,
      "logits/chosen": -1.0310547351837158,
      "logits/rejected": -0.9527343511581421,
      "logps/chosen": -1.0068359375,
      "logps/rejected": -1.3210937976837158,
      "loss": 1.3639,
      "nll_loss": 1.32421875,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.10068359225988388,
      "rewards/margins": 0.03147735446691513,
      "rewards/rejected": -0.13212890923023224,
      "step": 160
    },
    {
      "epoch": 0.012900288359386856,
      "grad_norm": 1.5892750852136275,
      "learning_rate": 6.135719910778963e-06,
      "log_odds_chosen": 0.43085938692092896,
      "log_odds_ratio": -0.638378918170929,
      "logits/chosen": -0.9833984375,
      "logits/rejected": -0.942578136920929,
      "logps/chosen": -0.980664074420929,
      "logps/rejected": -1.329687476158142,
      "loss": 1.3067,
      "nll_loss": 1.208984375,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.09804687649011612,
      "rewards/margins": 0.03497924655675888,
      "rewards/rejected": -0.13300780951976776,
      "step": 170
    },
    {
      "epoch": 0.013659128851115495,
      "grad_norm": 1.9895227129073771,
      "learning_rate": 5.962847939999439e-06,
      "log_odds_chosen": 0.4945312440395355,
      "log_odds_ratio": -0.589648425579071,
      "logits/chosen": -0.932421863079071,
      "logits/rejected": -0.873828113079071,
      "logps/chosen": -0.9701172113418579,
      "logps/rejected": -1.3488280773162842,
      "loss": 1.3382,
      "nll_loss": 1.2804687023162842,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.09707031399011612,
      "rewards/margins": 0.03778686374425888,
      "rewards/rejected": -0.13481445610523224,
      "step": 180
    },
    {
      "epoch": 0.014417969342844134,
      "grad_norm": 1.6480729869591368,
      "learning_rate": 5.803810000880094e-06,
      "log_odds_chosen": 0.4837402403354645,
      "log_odds_ratio": -0.5904296636581421,
      "logits/chosen": -0.9173828363418579,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.9761718511581421,
      "logps/rejected": -1.346289038658142,
      "loss": 1.3017,
      "nll_loss": 1.285546898841858,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.09760741889476776,
      "rewards/margins": 0.03700103610754013,
      "rewards/rejected": -0.13461914658546448,
      "step": 190
    },
    {
      "epoch": 0.015176809834572773,
      "grad_norm": 1.503264102872236,
      "learning_rate": 5.65685424949238e-06,
      "log_odds_chosen": 0.4825683534145355,
      "log_odds_ratio": -0.582226574420929,
      "logits/chosen": -0.9466797113418579,
      "logits/rejected": -0.870898425579071,
      "logps/chosen": -0.939648449420929,
      "logps/rejected": -1.3035156726837158,
      "loss": 1.2894,
      "nll_loss": 1.217187523841858,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.09396972507238388,
      "rewards/margins": 0.036403656005859375,
      "rewards/rejected": -0.13041992485523224,
      "step": 200
    },
    {
      "epoch": 0.015935650326301412,
      "grad_norm": 1.6703995784714376,
      "learning_rate": 5.5205244747388325e-06,
      "log_odds_chosen": 0.6786133050918579,
      "log_odds_ratio": -0.579882800579071,
      "logits/chosen": -0.8832031488418579,
      "logits/rejected": -0.862109363079071,
      "logps/chosen": -0.9300781488418579,
      "logps/rejected": -1.4582030773162842,
      "loss": 1.2816,
      "nll_loss": 1.2117187976837158,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.09296874701976776,
      "rewards/margins": 0.05272827297449112,
      "rewards/rejected": -0.14572754502296448,
      "step": 210
    },
    {
      "epoch": 0.01669449081803005,
      "grad_norm": 1.5169765274707603,
      "learning_rate": 5.393598899705936e-06,
      "log_odds_chosen": 0.517822265625,
      "log_odds_ratio": -0.616503894329071,
      "logits/chosen": -0.9017578363418579,
      "logits/rejected": -0.8421875238418579,
      "logps/chosen": -0.894335925579071,
      "logps/rejected": -1.3212890625,
      "loss": 1.2526,
      "nll_loss": 1.177148461341858,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.08945312350988388,
      "rewards/margins": 0.04265594482421875,
      "rewards/rejected": -0.13203124701976776,
      "step": 220
    },
    {
      "epoch": 0.01745333130975869,
      "grad_norm": 1.4368746262060774,
      "learning_rate": 5.275043787166296e-06,
      "log_odds_chosen": 0.4863037168979645,
      "log_odds_ratio": -0.578808605670929,
      "logits/chosen": -0.8785156011581421,
      "logits/rejected": -0.7623046636581421,
      "logps/chosen": -0.9287109375,
      "logps/rejected": -1.2966797351837158,
      "loss": 1.2595,
      "nll_loss": 1.2150390148162842,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.09282226860523224,
      "rewards/margins": 0.03685913234949112,
      "rewards/rejected": -0.129638671875,
      "step": 230
    },
    {
      "epoch": 0.018212171801487327,
      "grad_norm": 1.5611472145780005,
      "learning_rate": 5.163977794943223e-06,
      "log_odds_chosen": 0.4346069395542145,
      "log_odds_ratio": -0.597363293170929,
      "logits/chosen": -0.8232421875,
      "logits/rejected": -0.8089843988418579,
      "logps/chosen": -0.871874988079071,
      "logps/rejected": -1.18359375,
      "loss": 1.2569,
      "nll_loss": 1.143164038658142,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08718261867761612,
      "rewards/margins": 0.031207656487822533,
      "rewards/rejected": -0.11833496391773224,
      "step": 240
    },
    {
      "epoch": 0.018971012293215964,
      "grad_norm": 1.460241600287341,
      "learning_rate": 5.059644256269407e-06,
      "log_odds_chosen": 0.40449219942092896,
      "log_odds_ratio": -0.616015613079071,
      "logits/chosen": -0.8539062738418579,
      "logits/rejected": -0.804492175579071,
      "logps/chosen": -0.8667968511581421,
      "logps/rejected": -1.163476586341858,
      "loss": 1.2829,
      "nll_loss": 1.1749999523162842,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08671875298023224,
      "rewards/margins": 0.02962798997759819,
      "rewards/rejected": -0.11635742336511612,
      "step": 250
    },
    {
      "epoch": 0.019729852784944605,
      "grad_norm": 1.6013195827393891,
      "learning_rate": 4.961389383568338e-06,
      "log_odds_chosen": 0.56024169921875,
      "log_odds_ratio": -0.573925793170929,
      "logits/chosen": -0.863085925579071,
      "logits/rejected": -0.7964843511581421,
      "logps/chosen": -0.834765613079071,
      "logps/rejected": -1.236328125,
      "loss": 1.2298,
      "nll_loss": 1.2373046875,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.08352050930261612,
      "rewards/margins": 0.04019775241613388,
      "rewards/rejected": -0.12368164211511612,
      "step": 260
    },
    {
      "epoch": 0.020488693276673242,
      "grad_norm": 1.5935105244091246,
      "learning_rate": 4.8686449556014755e-06,
      "log_odds_chosen": 0.693359375,
      "log_odds_ratio": -0.531054675579071,
      "logits/chosen": -0.857617199420929,
      "logits/rejected": -0.811718761920929,
      "logps/chosen": -0.777148425579071,
      "logps/rejected": -1.2861328125,
      "loss": 1.2069,
      "nll_loss": 1.126562476158142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07766113430261612,
      "rewards/margins": 0.05088958889245987,
      "rewards/rejected": -0.12858887016773224,
      "step": 270
    },
    {
      "epoch": 0.021247533768401883,
      "grad_norm": 1.3821618104135565,
      "learning_rate": 4.780914437337574e-06,
      "log_odds_chosen": 0.5213378667831421,
      "log_odds_ratio": -0.592480480670929,
      "logits/chosen": -0.8648437261581421,
      "logits/rejected": -0.8041015863418579,
      "logps/chosen": -0.811718761920929,
      "logps/rejected": -1.173437476158142,
      "loss": 1.2352,
      "nll_loss": 1.216406226158142,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.08118896186351776,
      "rewards/margins": 0.03626251220703125,
      "rewards/rejected": -0.11740722507238388,
      "step": 280
    },
    {
      "epoch": 0.02200637426013052,
      "grad_norm": 1.6247124157263477,
      "learning_rate": 4.697761756117627e-06,
      "log_odds_chosen": 0.590527355670929,
      "log_odds_ratio": -0.5450195074081421,
      "logits/chosen": -0.862109363079071,
      "logits/rejected": -0.7880859375,
      "logps/chosen": -0.7593749761581421,
      "logps/rejected": -1.153906226158142,
      "loss": 1.2422,
      "nll_loss": 1.1925780773162842,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07601318508386612,
      "rewards/margins": 0.03940429538488388,
      "rewards/rejected": -0.1153564453125,
      "step": 290
    },
    {
      "epoch": 0.02276521475185916,
      "grad_norm": 1.4425392059458433,
      "learning_rate": 4.618802153517006e-06,
      "log_odds_chosen": 0.539794921875,
      "log_odds_ratio": -0.584277331829071,
      "logits/chosen": -0.8628906011581421,
      "logits/rejected": -0.816601574420929,
      "logps/chosen": -0.7953125238418579,
      "logps/rejected": -1.162109375,
      "loss": 1.2105,
      "nll_loss": 1.14453125,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07958984375,
      "rewards/margins": 0.03663330152630806,
      "rewards/rejected": -0.11616210639476776,
      "step": 300
    },
    {
      "epoch": 0.023524055243587798,
      "grad_norm": 1.4229245984267798,
      "learning_rate": 4.543694673976518e-06,
      "log_odds_chosen": 0.553759753704071,
      "log_odds_ratio": -0.567187488079071,
      "logits/chosen": -0.891406238079071,
      "logits/rejected": -0.8333984613418579,
      "logps/chosen": -0.7958984375,
      "logps/rejected": -1.169335961341858,
      "loss": 1.2145,
      "nll_loss": 1.260351538658142,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07958984375,
      "rewards/margins": 0.03729400783777237,
      "rewards/rejected": -0.11689452826976776,
      "step": 310
    },
    {
      "epoch": 0.024282895735316435,
      "grad_norm": 1.5337475875154933,
      "learning_rate": 4.472135954999579e-06,
      "log_odds_chosen": 0.5230346918106079,
      "log_odds_ratio": -0.5665038824081421,
      "logits/chosen": -0.8939453363418579,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.8011718988418579,
      "logps/rejected": -1.1394531726837158,
      "loss": 1.2126,
      "nll_loss": 1.1082031726837158,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08017577975988388,
      "rewards/margins": 0.03389739990234375,
      "rewards/rejected": -0.11411132663488388,
      "step": 320
    },
    {
      "epoch": 0.025041736227045076,
      "grad_norm": 1.4941611010961415,
      "learning_rate": 4.403855060505443e-06,
      "log_odds_chosen": 0.5206054449081421,
      "log_odds_ratio": -0.6089843511581421,
      "logits/chosen": -0.9638671875,
      "logits/rejected": -0.9195312261581421,
      "logps/chosen": -0.776562511920929,
      "logps/rejected": -1.1339843273162842,
      "loss": 1.1647,
      "nll_loss": 1.0783202648162842,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07766113430261612,
      "rewards/margins": 0.03580169752240181,
      "rewards/rejected": -0.11337890475988388,
      "step": 330
    },
    {
      "epoch": 0.025800576718773713,
      "grad_norm": 8.234299330038246,
      "learning_rate": 4.338609156373123e-06,
      "log_odds_chosen": 0.559277355670929,
      "log_odds_ratio": -0.579296886920929,
      "logits/chosen": -0.9681640863418579,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.7923828363418579,
      "logps/rejected": -1.1804687976837158,
      "loss": 1.1985,
      "nll_loss": 1.120703101158142,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07926025241613388,
      "rewards/margins": 0.03888397291302681,
      "rewards/rejected": -0.11806640774011612,
      "step": 340
    },
    {
      "epoch": 0.026559417210502353,
      "grad_norm": 1.4986198531651644,
      "learning_rate": 4.27617987059879e-06,
      "log_odds_chosen": 0.5880126953125,
      "log_odds_ratio": -0.5694335699081421,
      "logits/chosen": -0.9203125238418579,
      "logits/rejected": -0.850390613079071,
      "logps/chosen": -0.798828125,
      "logps/rejected": -1.200781226158142,
      "loss": 1.1852,
      "nll_loss": 1.150976538658142,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07993163913488388,
      "rewards/margins": 0.04015808179974556,
      "rewards/rejected": -0.12001953274011612,
      "step": 350
    },
    {
      "epoch": 0.02731825770223099,
      "grad_norm": 1.4802049838530484,
      "learning_rate": 4.216370213557839e-06,
      "log_odds_chosen": 0.5188354253768921,
      "log_odds_ratio": -0.607617199420929,
      "logits/chosen": -0.9332031011581421,
      "logits/rejected": -0.862500011920929,
      "logps/chosen": -0.8257812261581421,
      "logps/rejected": -1.1884765625,
      "loss": 1.2022,
      "nll_loss": 1.246484398841858,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.08256836235523224,
      "rewards/margins": 0.03617553785443306,
      "rewards/rejected": -0.1187744140625,
      "step": 360
    },
    {
      "epoch": 0.02807709819395963,
      "grad_norm": 1.4330692363937183,
      "learning_rate": 4.15900195928029e-06,
      "log_odds_chosen": 0.586865246295929,
      "log_odds_ratio": -0.60498046875,
      "logits/chosen": -0.9009765386581421,
      "logits/rejected": -0.858203113079071,
      "logps/chosen": -0.797656238079071,
      "logps/rejected": -1.1896483898162842,
      "loss": 1.2002,
      "nll_loss": 1.1806640625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07974853366613388,
      "rewards/margins": 0.03930206224322319,
      "rewards/rejected": -0.11899413913488388,
      "step": 370
    },
    {
      "epoch": 0.02883593868568827,
      "grad_norm": 1.4912577480503046,
      "learning_rate": 4.103913408340617e-06,
      "log_odds_chosen": 0.540576159954071,
      "log_odds_ratio": -0.6109863519668579,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.9087890386581421,
      "logps/chosen": -0.785937488079071,
      "logps/rejected": -1.1513671875,
      "loss": 1.1835,
      "nll_loss": 1.194726586341858,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07860107719898224,
      "rewards/margins": 0.03663025051355362,
      "rewards/rejected": -0.11527099460363388,
      "step": 380
    },
    {
      "epoch": 0.029594779177416906,
      "grad_norm": 1.344518207009396,
      "learning_rate": 4.050957468334666e-06,
      "log_odds_chosen": 0.5313720703125,
      "log_odds_ratio": -0.606249988079071,
      "logits/chosen": -0.9189453125,
      "logits/rejected": -0.8687499761581421,
      "logps/chosen": -0.779101550579071,
      "logps/rejected": -1.145117163658142,
      "loss": 1.1957,
      "nll_loss": 1.102929711341858,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07797851413488388,
      "rewards/margins": 0.03666992112994194,
      "rewards/rejected": -0.11447753757238388,
      "step": 390
    },
    {
      "epoch": 0.030353619669145546,
      "grad_norm": 4.3130074404618135,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.45671385526657104,
      "log_odds_ratio": -0.6246093511581421,
      "logits/chosen": -0.918164074420929,
      "logits/rejected": -0.857617199420929,
      "logps/chosen": -0.7632812261581421,
      "logps/rejected": -1.0556640625,
      "loss": 1.1896,
      "nll_loss": 1.123632788658142,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07639160007238388,
      "rewards/margins": 0.02919769287109375,
      "rewards/rejected": -0.10543213039636612,
      "step": 400
    },
    {
      "epoch": 0.031112460160874184,
      "grad_norm": 1.3095212395183442,
      "learning_rate": 3.950918386598359e-06,
      "log_odds_chosen": 0.7026733160018921,
      "log_odds_ratio": -0.5523437261581421,
      "logits/chosen": -0.970507800579071,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.783007800579071,
      "logps/rejected": -1.248437523841858,
      "loss": 1.1875,
      "nll_loss": 1.142187476158142,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07823486626148224,
      "rewards/margins": 0.04652862623333931,
      "rewards/rejected": -0.12485351413488388,
      "step": 410
    },
    {
      "epoch": 0.031871300652602824,
      "grad_norm": 1.538770613712715,
      "learning_rate": 3.903600291794132e-06,
      "log_odds_chosen": 0.9018310308456421,
      "log_odds_ratio": -0.4969726502895355,
      "logits/chosen": -0.8892577886581421,
      "logits/rejected": -0.81640625,
      "logps/chosen": -0.752734363079071,
      "logps/rejected": -1.372656226158142,
      "loss": 1.1881,
      "nll_loss": 1.146875023841858,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07523193210363388,
      "rewards/margins": 0.06215057522058487,
      "rewards/rejected": -0.13740234076976776,
      "step": 420
    },
    {
      "epoch": 0.032630141144331465,
      "grad_norm": 1.5135129410829358,
      "learning_rate": 3.857942577363297e-06,
      "log_odds_chosen": 0.5770508050918579,
      "log_odds_ratio": -0.5787109136581421,
      "logits/chosen": -0.9457031488418579,
      "logits/rejected": -0.872851550579071,
      "logps/chosen": -0.789257824420929,
      "logps/rejected": -1.173828125,
      "loss": 1.1877,
      "nll_loss": 1.115234375,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07891845703125,
      "rewards/margins": 0.03848876804113388,
      "rewards/rejected": -0.11740722507238388,
      "step": 430
    },
    {
      "epoch": 0.0333889816360601,
      "grad_norm": 1.5495511200199545,
      "learning_rate": 3.813850356982369e-06,
      "log_odds_chosen": 0.7400146722793579,
      "log_odds_ratio": -0.5404297113418579,
      "logits/chosen": -0.9517577886581421,
      "logits/rejected": -0.914257824420929,
      "logps/chosen": -0.779101550579071,
      "logps/rejected": -1.2970702648162842,
      "loss": 1.1755,
      "nll_loss": 1.0986328125,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07794189453125,
      "rewards/margins": 0.05182952806353569,
      "rewards/rejected": -0.12973633408546448,
      "step": 440
    },
    {
      "epoch": 0.03414782212778874,
      "grad_norm": 1.434457768263082,
      "learning_rate": 3.7712361663282537e-06,
      "log_odds_chosen": 0.7149658203125,
      "log_odds_ratio": -0.53125,
      "logits/chosen": -0.892382800579071,
      "logits/rejected": -0.848437488079071,
      "logps/chosen": -0.7767578363418579,
      "logps/rejected": -1.2646484375,
      "loss": 1.1691,
      "nll_loss": 1.088476538658142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07757568359375,
      "rewards/margins": 0.04888000339269638,
      "rewards/rejected": -0.12656250596046448,
      "step": 450
    },
    {
      "epoch": 0.03490666261951738,
      "grad_norm": 1.4647624075894172,
      "learning_rate": 3.730019232961255e-06,
      "log_odds_chosen": 0.598742663860321,
      "log_odds_ratio": -0.5794922113418579,
      "logits/chosen": -0.846875011920929,
      "logits/rejected": -0.7669922113418579,
      "logps/chosen": -0.788281261920929,
      "logps/rejected": -1.1970703601837158,
      "loss": 1.1697,
      "nll_loss": 1.101171851158142,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07880859076976776,
      "rewards/margins": 0.04080658033490181,
      "rewards/rejected": -0.11972656100988388,
      "step": 460
    },
    {
      "epoch": 0.035665503111246014,
      "grad_norm": 1.460032211323527,
      "learning_rate": 3.6901248321155403e-06,
      "log_odds_chosen": 0.535937488079071,
      "log_odds_ratio": -0.59423828125,
      "logits/chosen": -0.857226550579071,
      "logits/rejected": -0.783984363079071,
      "logps/chosen": -0.793164074420929,
      "logps/rejected": -1.162695288658142,
      "loss": 1.1848,
      "nll_loss": 1.1906249523162842,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07929687201976776,
      "rewards/margins": 0.03700103610754013,
      "rewards/rejected": -0.11623535305261612,
      "step": 470
    },
    {
      "epoch": 0.036424343602974654,
      "grad_norm": 1.3524653524950496,
      "learning_rate": 3.6514837167011073e-06,
      "log_odds_chosen": 0.646289050579071,
      "log_odds_ratio": -0.5396484136581421,
      "logits/chosen": -0.888867199420929,
      "logits/rejected": -0.8472656011581421,
      "logps/chosen": -0.755664050579071,
      "logps/rejected": -1.179296851158142,
      "loss": 1.1176,
      "nll_loss": 1.028906226158142,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.0755615234375,
      "rewards/margins": 0.04240112379193306,
      "rewards/rejected": -0.11789550632238388,
      "step": 480
    },
    {
      "epoch": 0.037183184094703295,
      "grad_norm": 1.575634651364888,
      "learning_rate": 3.6140316116210052e-06,
      "log_odds_chosen": 0.5822998285293579,
      "log_odds_ratio": -0.582324206829071,
      "logits/chosen": -0.8666015863418579,
      "logits/rejected": -0.790820300579071,
      "logps/chosen": -0.757617175579071,
      "logps/rejected": -1.1457030773162842,
      "loss": 1.1698,
      "nll_loss": 1.113671898841858,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07574462890625,
      "rewards/margins": 0.03886108472943306,
      "rewards/rejected": -0.1146240234375,
      "step": 490
    },
    {
      "epoch": 0.03794202458643193,
      "grad_norm": 1.4509304015827453,
      "learning_rate": 3.5777087639996634e-06,
      "log_odds_chosen": 0.682324230670929,
      "log_odds_ratio": -0.526074230670929,
      "logits/chosen": -0.888867199420929,
      "logits/rejected": -0.8194335699081421,
      "logps/chosen": -0.7503906488418579,
      "logps/rejected": -1.1746094226837158,
      "loss": 1.1828,
      "nll_loss": 1.1652343273162842,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.0750732421875,
      "rewards/margins": 0.0423583984375,
      "rewards/rejected": -0.117431640625,
      "step": 500
    },
    {
      "epoch": 0.03870086507816057,
      "grad_norm": 1.8399283965320115,
      "learning_rate": 3.5424595421603814e-06,
      "log_odds_chosen": 0.82373046875,
      "log_odds_ratio": -0.5107421875,
      "logits/chosen": -0.8783203363418579,
      "logits/rejected": -0.8070312738418579,
      "logps/chosen": -0.736328125,
      "logps/rejected": -1.3039062023162842,
      "loss": 1.1434,
      "nll_loss": 1.0751953125,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07369384914636612,
      "rewards/margins": 0.05676879733800888,
      "rewards/rejected": -0.13034668564796448,
      "step": 510
    },
    {
      "epoch": 0.03945970556988921,
      "grad_norm": 1.476828967448819,
      "learning_rate": 3.5082320772281165e-06,
      "log_odds_chosen": 0.8194335699081421,
      "log_odds_ratio": -0.5054687261581421,
      "logits/chosen": -0.9585937261581421,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.7845703363418579,
      "logps/rejected": -1.348046898841858,
      "loss": 1.1622,
      "nll_loss": 1.1238281726837158,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07841797173023224,
      "rewards/margins": 0.05634155124425888,
      "rewards/rejected": -0.13469238579273224,
      "step": 520
    },
    {
      "epoch": 0.04021854606161785,
      "grad_norm": 1.539593920271273,
      "learning_rate": 3.474977942104555e-06,
      "log_odds_chosen": 0.5797973871231079,
      "log_odds_ratio": -0.5848633050918579,
      "logits/chosen": -0.947460949420929,
      "logits/rejected": -0.8843749761581421,
      "logps/chosen": -0.773632824420929,
      "logps/rejected": -1.160742163658142,
      "loss": 1.1431,
      "nll_loss": 1.1005859375,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07738037407398224,
      "rewards/margins": 0.03878479078412056,
      "rewards/rejected": -0.11613769829273224,
      "step": 530
    },
    {
      "epoch": 0.040977386553346484,
      "grad_norm": 1.4842674812940488,
      "learning_rate": 3.442651863295481e-06,
      "log_odds_chosen": 0.652294933795929,
      "log_odds_ratio": -0.5572265386581421,
      "logits/chosen": -0.9189453125,
      "logits/rejected": -0.8451172113418579,
      "logps/chosen": -0.749804675579071,
      "logps/rejected": -1.199609398841858,
      "loss": 1.1613,
      "nll_loss": 1.100976586341858,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0750732421875,
      "rewards/margins": 0.04497985914349556,
      "rewards/rejected": -0.12001953274011612,
      "step": 540
    },
    {
      "epoch": 0.041736227045075125,
      "grad_norm": 1.5239315494051218,
      "learning_rate": 3.4112114616897665e-06,
      "log_odds_chosen": 0.730712890625,
      "log_odds_ratio": -0.532910168170929,
      "logits/chosen": -0.9126952886581421,
      "logits/rejected": -0.8607422113418579,
      "logps/chosen": -0.740039050579071,
      "logps/rejected": -1.240234375,
      "loss": 1.1523,
      "nll_loss": 1.071874976158142,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07404784858226776,
      "rewards/margins": 0.04995117336511612,
      "rewards/rejected": -0.12397460639476776,
      "step": 550
    },
    {
      "epoch": 0.042495067536803766,
      "grad_norm": 1.4941924034303562,
      "learning_rate": 3.3806170189140663e-06,
      "log_odds_chosen": 0.6442626714706421,
      "log_odds_ratio": -0.5541015863418579,
      "logits/chosen": -0.9439452886581421,
      "logits/rejected": -0.8671875,
      "logps/chosen": -0.7408202886581421,
      "logps/rejected": -1.167578101158142,
      "loss": 1.1318,
      "nll_loss": 1.0128905773162842,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07408447563648224,
      "rewards/margins": 0.04265136644244194,
      "rewards/rejected": -0.11665038764476776,
      "step": 560
    },
    {
      "epoch": 0.0432539080285324,
      "grad_norm": 1.5068041723722188,
      "learning_rate": 3.350831266333564e-06,
      "log_odds_chosen": 0.572314441204071,
      "log_odds_ratio": -0.5923827886581421,
      "logits/chosen": -0.942187488079071,
      "logits/rejected": -0.8658202886581421,
      "logps/chosen": -0.798632800579071,
      "logps/rejected": -1.197656273841858,
      "loss": 1.1688,
      "nll_loss": 1.111718773841858,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.079833984375,
      "rewards/margins": 0.03993072360754013,
      "rewards/rejected": -0.11982421576976776,
      "step": 570
    },
    {
      "epoch": 0.04401274852026104,
      "grad_norm": 1.4143791261814025,
      "learning_rate": 3.3218191941495984e-06,
      "log_odds_chosen": 0.768115222454071,
      "log_odds_ratio": -0.4981445372104645,
      "logits/chosen": -0.9248046875,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.746874988079071,
      "logps/rejected": -1.2615234851837158,
      "loss": 1.1714,
      "nll_loss": 1.0695312023162842,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07470703125,
      "rewards/margins": 0.05145568773150444,
      "rewards/rejected": -0.12619629502296448,
      "step": 580
    },
    {
      "epoch": 0.04477158901198968,
      "grad_norm": 1.5758760245545085,
      "learning_rate": 3.293547878370473e-06,
      "log_odds_chosen": 0.5545409917831421,
      "log_odds_ratio": -0.612597644329071,
      "logits/chosen": -0.8935546875,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.765429675579071,
      "logps/rejected": -1.1183593273162842,
      "loss": 1.1337,
      "nll_loss": 1.1160156726837158,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07655028998851776,
      "rewards/margins": 0.03520049899816513,
      "rewards/rejected": -0.11186523735523224,
      "step": 590
    },
    {
      "epoch": 0.04553042950371832,
      "grad_norm": 1.4028074979920369,
      "learning_rate": 3.2659863237109044e-06,
      "log_odds_chosen": 0.966503918170929,
      "log_odds_ratio": -0.4481445252895355,
      "logits/chosen": -0.9169921875,
      "logits/rejected": -0.8544921875,
      "logps/chosen": -0.7320312261581421,
      "logps/rejected": -1.3779296875,
      "loss": 1.1331,
      "nll_loss": 1.105859398841858,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07319335639476776,
      "rewards/margins": 0.06447754055261612,
      "rewards/rejected": -0.1376953125,
      "step": 600
    },
    {
      "epoch": 0.046289269995446955,
      "grad_norm": 1.49643970687548,
      "learning_rate": 3.239105320715664e-06,
      "log_odds_chosen": 0.6585937738418579,
      "log_odds_ratio": -0.5478515625,
      "logits/chosen": -0.9150390625,
      "logits/rejected": -0.8597656488418579,
      "logps/chosen": -0.741015613079071,
      "logps/rejected": -1.168554663658142,
      "loss": 1.1114,
      "nll_loss": 1.037109375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07413329929113388,
      "rewards/margins": 0.04278564453125,
      "rewards/rejected": -0.11684570461511612,
      "step": 610
    },
    {
      "epoch": 0.047048110487175596,
      "grad_norm": 1.4603778539467505,
      "learning_rate": 3.2128773156099956e-06,
      "log_odds_chosen": 0.7294921875,
      "log_odds_ratio": -0.5185546875,
      "logits/chosen": -0.9248046875,
      "logits/rejected": -0.872265636920929,
      "logps/chosen": -0.7431640625,
      "logps/rejected": -1.24609375,
      "loss": 1.1383,
      "nll_loss": 1.102148413658142,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07432861626148224,
      "rewards/margins": 0.05032043531537056,
      "rewards/rejected": -0.12458495795726776,
      "step": 620
    },
    {
      "epoch": 0.047806950978904236,
      "grad_norm": 1.4270703213144766,
      "learning_rate": 3.187276291558383e-06,
      "log_odds_chosen": 0.694042980670929,
      "log_odds_ratio": -0.52392578125,
      "logits/chosen": -0.9664062261581421,
      "logits/rejected": -0.9027343988418579,
      "logps/chosen": -0.7904297113418579,
      "logps/rejected": -1.267578125,
      "loss": 1.1549,
      "nll_loss": 1.1359374523162842,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07899169623851776,
      "rewards/margins": 0.047760009765625,
      "rewards/rejected": -0.126708984375,
      "step": 630
    },
    {
      "epoch": 0.04856579147063287,
      "grad_norm": 1.3640593722657062,
      "learning_rate": 3.1622776601683788e-06,
      "log_odds_chosen": 0.6767578125,
      "log_odds_ratio": -0.5350586175918579,
      "logits/chosen": -0.9214843511581421,
      "logits/rejected": -0.876171886920929,
      "logps/chosen": -0.781054675579071,
      "logps/rejected": -1.224218726158142,
      "loss": 1.1219,
      "nll_loss": 1.0398437976837158,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.078125,
      "rewards/margins": 0.04420776292681694,
      "rewards/rejected": -0.12226562201976776,
      "step": 640
    },
    {
      "epoch": 0.04932463196236151,
      "grad_norm": 1.39289555170604,
      "learning_rate": 3.1378581622109444e-06,
      "log_odds_chosen": 0.5647217035293579,
      "log_odds_ratio": -0.6126953363418579,
      "logits/chosen": -1.0076172351837158,
      "logits/rejected": -0.9320312738418579,
      "logps/chosen": -0.8109375238418579,
      "logps/rejected": -1.2166016101837158,
      "loss": 1.1219,
      "nll_loss": 1.067773461341858,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.08111572265625,
      "rewards/margins": 0.040557097643613815,
      "rewards/rejected": -0.12163086235523224,
      "step": 650
    },
    {
      "epoch": 0.05008347245409015,
      "grad_norm": 1.5575966456743369,
      "learning_rate": 3.113995776646092e-06,
      "log_odds_chosen": 0.9054199457168579,
      "log_odds_ratio": -0.4915527403354645,
      "logits/chosen": -0.9623047113418579,
      "logits/rejected": -0.898242175579071,
      "logps/chosen": -0.7110351324081421,
      "logps/rejected": -1.308203101158142,
      "loss": 1.1272,
      "nll_loss": 1.0681641101837158,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07109375298023224,
      "rewards/margins": 0.05971984937787056,
      "rewards/rejected": -0.13078613579273224,
      "step": 660
    },
    {
      "epoch": 0.05084231294581879,
      "grad_norm": 1.5820498982958486,
      "learning_rate": 3.090669637145023e-06,
      "log_odds_chosen": 0.920214831829071,
      "log_odds_ratio": -0.48076170682907104,
      "logits/chosen": -0.999218761920929,
      "logits/rejected": -0.9287109375,
      "logps/chosen": -0.7611328363418579,
      "logps/rejected": -1.382421851158142,
      "loss": 1.1265,
      "nll_loss": 1.099023461341858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07607422024011612,
      "rewards/margins": 0.0622406005859375,
      "rewards/rejected": -0.13835449516773224,
      "step": 670
    },
    {
      "epoch": 0.051601153437547426,
      "grad_norm": 1.412160791122473,
      "learning_rate": 3.0678599553894814e-06,
      "log_odds_chosen": 0.636523425579071,
      "log_odds_ratio": -0.5533202886581421,
      "logits/chosen": -0.931640625,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.754687488079071,
      "logps/rejected": -1.1794922351837158,
      "loss": 1.1396,
      "nll_loss": 1.0632812976837158,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07546386867761612,
      "rewards/margins": 0.042449951171875,
      "rewards/rejected": -0.11796875298023224,
      "step": 680
    },
    {
      "epoch": 0.052359993929276066,
      "grad_norm": 1.6469723394316966,
      "learning_rate": 3.0455479505075235e-06,
      "log_odds_chosen": 0.900097668170929,
      "log_odds_ratio": -0.47685545682907104,
      "logits/chosen": -0.951171875,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.7466796636581421,
      "logps/rejected": -1.344140648841858,
      "loss": 1.1208,
      "nll_loss": 1.0744140148162842,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07470703125,
      "rewards/margins": 0.05967559665441513,
      "rewards/rejected": -0.13437500596046448,
      "step": 690
    },
    {
      "epoch": 0.05311883442100471,
      "grad_norm": 2.299409754878046,
      "learning_rate": 3.0237157840738173e-06,
      "log_odds_chosen": 0.818359375,
      "log_odds_ratio": -0.502246081829071,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.8402343988418579,
      "logps/chosen": -0.728515625,
      "logps/rejected": -1.2646484375,
      "loss": 1.1038,
      "nll_loss": 1.045507788658142,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07291259616613388,
      "rewards/margins": 0.05359496921300888,
      "rewards/rejected": -0.12663574516773224,
      "step": 700
    },
    {
      "epoch": 0.05387767491273334,
      "grad_norm": 1.4366530156011492,
      "learning_rate": 3.002346500163206e-06,
      "log_odds_chosen": 0.676806628704071,
      "log_odds_ratio": -0.568066418170929,
      "logits/chosen": -0.892578125,
      "logits/rejected": -0.8421875238418579,
      "logps/chosen": -0.7298828363418579,
      "logps/rejected": -1.1785156726837158,
      "loss": 1.0838,
      "nll_loss": 1.046875,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.072998046875,
      "rewards/margins": 0.04483947902917862,
      "rewards/rejected": -0.1177978515625,
      "step": 710
    },
    {
      "epoch": 0.05463651540446198,
      "grad_norm": 1.4091415441420312,
      "learning_rate": 2.9814239699997195e-06,
      "log_odds_chosen": 0.730639636516571,
      "log_odds_ratio": -0.5448242425918579,
      "logits/chosen": -0.9156249761581421,
      "logits/rejected": -0.869921863079071,
      "logps/chosen": -0.7748047113418579,
      "logps/rejected": -1.267968773841858,
      "loss": 1.1281,
      "nll_loss": 1.166601538658142,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07746581733226776,
      "rewards/margins": 0.04938201978802681,
      "rewards/rejected": -0.1268310546875,
      "step": 720
    },
    {
      "epoch": 0.05539535589619062,
      "grad_norm": 1.5428528298733337,
      "learning_rate": 2.9609328407904207e-06,
      "log_odds_chosen": 0.861193835735321,
      "log_odds_ratio": -0.473388671875,
      "logits/chosen": -0.983203113079071,
      "logits/rejected": -0.8863281011581421,
      "logps/chosen": -0.708984375,
      "logps/rejected": -1.2703125476837158,
      "loss": 1.0983,
      "nll_loss": 1.019921898841858,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07081298530101776,
      "rewards/margins": 0.05611877515912056,
      "rewards/rejected": -0.12700195610523224,
      "step": 730
    },
    {
      "epoch": 0.05615419638791926,
      "grad_norm": 1.3181361407151604,
      "learning_rate": 2.940858488375231e-06,
      "log_odds_chosen": 0.657470703125,
      "log_odds_ratio": -0.5528320074081421,
      "logits/chosen": -0.9462890625,
      "logits/rejected": -0.882031261920929,
      "logps/chosen": -0.7880859375,
      "logps/rejected": -1.2468750476837158,
      "loss": 1.1081,
      "nll_loss": 1.021484375,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07878418266773224,
      "rewards/margins": 0.04585418850183487,
      "rewards/rejected": -0.12468262016773224,
      "step": 740
    },
    {
      "epoch": 0.056913036879647896,
      "grad_norm": 1.4129096893123931,
      "learning_rate": 2.9211869733608857e-06,
      "log_odds_chosen": 0.70599365234375,
      "log_odds_ratio": -0.537890613079071,
      "logits/chosen": -0.947265625,
      "logits/rejected": -0.892773449420929,
      "logps/chosen": -0.759960949420929,
      "logps/rejected": -1.2421875,
      "loss": 1.1035,
      "nll_loss": 1.037695288658142,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07593993842601776,
      "rewards/margins": 0.04822998121380806,
      "rewards/rejected": -0.124267578125,
      "step": 750
    },
    {
      "epoch": 0.05767187737137654,
      "grad_norm": 1.4910014683581294,
      "learning_rate": 2.901905000440047e-06,
      "log_odds_chosen": 0.8280273675918579,
      "log_odds_ratio": -0.504443347454071,
      "logits/chosen": -0.961132824420929,
      "logits/rejected": -0.8509765863418579,
      "logps/chosen": -0.71484375,
      "logps/rejected": -1.2763671875,
      "loss": 1.1171,
      "nll_loss": 1.068750023841858,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07152099907398224,
      "rewards/margins": 0.05616455152630806,
      "rewards/rejected": -0.12775878608226776,
      "step": 760
    },
    {
      "epoch": 0.05843071786310518,
      "grad_norm": 1.367660612325947,
      "learning_rate": 2.8829998806257885e-06,
      "log_odds_chosen": 0.779040515422821,
      "log_odds_ratio": -0.5166991949081421,
      "logits/chosen": -0.934374988079071,
      "logits/rejected": -0.845507800579071,
      "logps/chosen": -0.681347668170929,
      "logps/rejected": -1.188085913658142,
      "loss": 1.1298,
      "nll_loss": 1.0185546875,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.068115234375,
      "rewards/margins": 0.05065765231847763,
      "rewards/rejected": -0.11887206882238388,
      "step": 770
    },
    {
      "epoch": 0.05918955835483381,
      "grad_norm": 1.5187404836399663,
      "learning_rate": 2.8644594961577314e-06,
      "log_odds_chosen": 0.7642577886581421,
      "log_odds_ratio": -0.5435546636581421,
      "logits/chosen": -0.9417968988418579,
      "logits/rejected": -0.8802734613418579,
      "logps/chosen": -0.768750011920929,
      "logps/rejected": -1.3263671398162842,
      "loss": 1.1197,
      "nll_loss": 1.089453101158142,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07689209282398224,
      "rewards/margins": 0.05573882907629013,
      "rewards/rejected": -0.13266602158546448,
      "step": 780
    },
    {
      "epoch": 0.05994839884656245,
      "grad_norm": 1.3998297985203276,
      "learning_rate": 2.84627226785928e-06,
      "log_odds_chosen": 0.68170166015625,
      "log_odds_ratio": -0.550976574420929,
      "logits/chosen": -0.9701172113418579,
      "logits/rejected": -0.875,
      "logps/chosen": -0.741992175579071,
      "logps/rejected": -1.1867187023162842,
      "loss": 1.1149,
      "nll_loss": 1.048242211341858,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07418213039636612,
      "rewards/margins": 0.04451598972082138,
      "rewards/rejected": -0.11870117485523224,
      "step": 790
    },
    {
      "epoch": 0.06070723933829109,
      "grad_norm": 1.5365658645460811,
      "learning_rate": 2.82842712474619e-06,
      "log_odds_chosen": 0.8926757574081421,
      "log_odds_ratio": -0.5009765625,
      "logits/chosen": -0.938281238079071,
      "logits/rejected": -0.870312511920929,
      "logps/chosen": -0.7412109375,
      "logps/rejected": -1.378320336341858,
      "loss": 1.1089,
      "nll_loss": 1.032812476158142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07413329929113388,
      "rewards/margins": 0.06382445991039276,
      "rewards/rejected": -0.13789062201976776,
      "step": 800
    },
    {
      "epoch": 0.061466079830019726,
      "grad_norm": 1.553258112910012,
      "learning_rate": 2.810913475705226e-06,
      "log_odds_chosen": 0.827832043170929,
      "log_odds_ratio": -0.4930664002895355,
      "logits/chosen": -0.9130859375,
      "logits/rejected": -0.861523449420929,
      "logps/chosen": -0.7025390863418579,
      "logps/rejected": -1.2615234851837158,
      "loss": 1.1092,
      "nll_loss": 1.0056641101837158,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07026366889476776,
      "rewards/margins": 0.0557861328125,
      "rewards/rejected": -0.1259765625,
      "step": 810
    },
    {
      "epoch": 0.06222492032174837,
      "grad_norm": 1.6754050719010933,
      "learning_rate": 2.7937211830783128e-06,
      "log_odds_chosen": 0.825244128704071,
      "log_odds_ratio": -0.513720691204071,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.9544922113418579,
      "logps/chosen": -0.761523425579071,
      "logps/rejected": -1.33203125,
      "loss": 1.1177,
      "nll_loss": 1.129296898841858,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07611083984375,
      "rewards/margins": 0.05705719068646431,
      "rewards/rejected": -0.13322754204273224,
      "step": 820
    },
    {
      "epoch": 0.06298376081347701,
      "grad_norm": 1.4397189392753629,
      "learning_rate": 2.776840538002493e-06,
      "log_odds_chosen": 0.8416992425918579,
      "log_odds_ratio": -0.51953125,
      "logits/chosen": -0.950976550579071,
      "logits/rejected": -0.9068359136581421,
      "logps/chosen": -0.737109363079071,
      "logps/rejected": -1.309960961341858,
      "loss": 1.0814,
      "nll_loss": 0.9839843511581421,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07368163764476776,
      "rewards/margins": 0.0572509765625,
      "rewards/rejected": -0.1309814453125,
      "step": 830
    },
    {
      "epoch": 0.06374260130520565,
      "grad_norm": 1.481525111906686,
      "learning_rate": 2.7602622373694163e-06,
      "log_odds_chosen": 0.6253417730331421,
      "log_odds_ratio": -0.569628894329071,
      "logits/chosen": -0.931835949420929,
      "logits/rejected": -0.8716796636581421,
      "logps/chosen": -0.7300781011581421,
      "logps/rejected": -1.1677734851837158,
      "loss": 1.1057,
      "nll_loss": 1.015234351158142,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07297363132238388,
      "rewards/margins": 0.04375610500574112,
      "rewards/rejected": -0.1168212890625,
      "step": 840
    },
    {
      "epoch": 0.06450144179693429,
      "grad_norm": 1.4528723364813,
      "learning_rate": 2.743977362280141e-06,
      "log_odds_chosen": 0.7959960699081421,
      "log_odds_ratio": -0.5220702886581421,
      "logits/chosen": -0.975390613079071,
      "logits/rejected": -0.915234386920929,
      "logps/chosen": -0.7105468511581421,
      "logps/rejected": -1.235937476158142,
      "loss": 1.1185,
      "nll_loss": 1.0314452648162842,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07106933742761612,
      "rewards/margins": 0.0523834228515625,
      "rewards/rejected": -0.12353515625,
      "step": 850
    },
    {
      "epoch": 0.06526028228866293,
      "grad_norm": 1.4155260042724327,
      "learning_rate": 2.7279773578818937e-06,
      "log_odds_chosen": 0.774218738079071,
      "log_odds_ratio": -0.54150390625,
      "logits/chosen": -0.9326171875,
      "logits/rejected": -0.8871093988418579,
      "logps/chosen": -0.7484375238418579,
      "logps/rejected": -1.286523461341858,
      "loss": 1.1145,
      "nll_loss": 0.9873046875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07486572116613388,
      "rewards/margins": 0.05366821214556694,
      "rewards/rejected": -0.12849120795726776,
      "step": 860
    },
    {
      "epoch": 0.06601912278039156,
      "grad_norm": 1.4672576973291955,
      "learning_rate": 2.7122540144832417e-06,
      "log_odds_chosen": 0.6922851800918579,
      "log_odds_ratio": -0.5411132574081421,
      "logits/chosen": -0.9400390386581421,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.7162109613418579,
      "logps/rejected": -1.152929663658142,
      "loss": 1.1071,
      "nll_loss": 1.0554687976837158,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07163085788488388,
      "rewards/margins": 0.04363097995519638,
      "rewards/rejected": -0.11533202975988388,
      "step": 870
    },
    {
      "epoch": 0.0667779632721202,
      "grad_norm": 1.4377110735801357,
      "learning_rate": 2.696799449852968e-06,
      "log_odds_chosen": 0.7808837890625,
      "log_odds_ratio": -0.517578125,
      "logits/chosen": -0.8804687261581421,
      "logits/rejected": -0.841796875,
      "logps/chosen": -0.744335949420929,
      "logps/rejected": -1.2470703125,
      "loss": 1.1171,
      "nll_loss": 1.073828101158142,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07441405951976776,
      "rewards/margins": 0.05037841945886612,
      "rewards/rejected": -0.12473144382238388,
      "step": 880
    },
    {
      "epoch": 0.06753680376384884,
      "grad_norm": 1.5523580294418937,
      "learning_rate": 2.6816060926159636e-06,
      "log_odds_chosen": 0.8277221918106079,
      "log_odds_ratio": -0.506103515625,
      "logits/chosen": -0.948046863079071,
      "logits/rejected": -0.8501952886581421,
      "logps/chosen": -0.705078125,
      "logps/rejected": -1.2453124523162842,
      "loss": 1.0915,
      "nll_loss": 1.0099608898162842,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07052002102136612,
      "rewards/margins": 0.05402832105755806,
      "rewards/rejected": -0.12456054985523224,
      "step": 890
    },
    {
      "epoch": 0.06829564425557748,
      "grad_norm": 1.3981673710781848,
      "learning_rate": 2.6666666666666664e-06,
      "log_odds_chosen": 0.8017578125,
      "log_odds_ratio": -0.513134777545929,
      "logits/chosen": -0.940625011920929,
      "logits/rejected": -0.8638671636581421,
      "logps/chosen": -0.7030273675918579,
      "logps/rejected": -1.211328148841858,
      "loss": 1.0997,
      "nll_loss": 1.0662109851837158,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07032470405101776,
      "rewards/margins": 0.050933837890625,
      "rewards/rejected": -0.12116698920726776,
      "step": 900
    },
    {
      "epoch": 0.06905448474730612,
      "grad_norm": 1.4703260458527112,
      "learning_rate": 2.6519741765271837e-06,
      "log_odds_chosen": 0.8729492425918579,
      "log_odds_ratio": -0.4708007872104645,
      "logits/chosen": -0.9691406488418579,
      "logits/rejected": -0.8560546636581421,
      "logps/chosen": -0.6851562261581421,
      "logps/rejected": -1.2585937976837158,
      "loss": 1.1185,
      "nll_loss": 1.0173828601837158,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06849364936351776,
      "rewards/margins": 0.05747680738568306,
      "rewards/rejected": -0.12590332329273224,
      "step": 910
    },
    {
      "epoch": 0.06981332523903476,
      "grad_norm": 1.7240865708769755,
      "learning_rate": 2.637521893583148e-06,
      "log_odds_chosen": 0.833544909954071,
      "log_odds_ratio": -0.522167980670929,
      "logits/chosen": -0.9603515863418579,
      "logits/rejected": -0.895703136920929,
      "logps/chosen": -0.737500011920929,
      "logps/rejected": -1.32421875,
      "loss": 1.1254,
      "nll_loss": 1.0656249523162842,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07379150390625,
      "rewards/margins": 0.05867920070886612,
      "rewards/rejected": -0.13249512016773224,
      "step": 920
    },
    {
      "epoch": 0.0705721657307634,
      "grad_norm": 1.596536890919686,
      "learning_rate": 2.6233033431358115e-06,
      "log_odds_chosen": 0.688720703125,
      "log_odds_ratio": -0.5291992425918579,
      "logits/chosen": -0.9482421875,
      "logits/rejected": -0.8775390386581421,
      "logps/chosen": -0.698046863079071,
      "logps/rejected": -1.146093726158142,
      "loss": 1.1058,
      "nll_loss": 1.027929663658142,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06984863430261612,
      "rewards/margins": 0.04474487155675888,
      "rewards/rejected": -0.11459960788488388,
      "step": 930
    },
    {
      "epoch": 0.07133100622249203,
      "grad_norm": 1.5205195253800634,
      "learning_rate": 2.6093122922137685e-06,
      "log_odds_chosen": 0.684814453125,
      "log_odds_ratio": -0.5577148199081421,
      "logits/chosen": -0.9986327886581421,
      "logits/rejected": -0.963671863079071,
      "logps/chosen": -0.7222656011581421,
      "logps/rejected": -1.1583983898162842,
      "loss": 1.1012,
      "nll_loss": 0.95703125,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07225342094898224,
      "rewards/margins": 0.04364623874425888,
      "rewards/rejected": -0.11591796576976776,
      "step": 940
    },
    {
      "epoch": 0.07208984671422067,
      "grad_norm": 1.5984518956278635,
      "learning_rate": 2.5955427380922006e-06,
      "log_odds_chosen": 0.7061523199081421,
      "log_odds_ratio": -0.5171874761581421,
      "logits/chosen": -0.9800781011581421,
      "logits/rejected": -0.8882812261581421,
      "logps/chosen": -0.6806640625,
      "logps/rejected": -1.127539038658142,
      "loss": 1.1089,
      "nll_loss": 1.0857422351837158,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06800536811351776,
      "rewards/margins": 0.04460601881146431,
      "rewards/rejected": -0.1126708984375,
      "step": 950
    },
    {
      "epoch": 0.07284868720594931,
      "grad_norm": 1.5002616023423914,
      "learning_rate": 2.5819888974716113e-06,
      "log_odds_chosen": 0.765625,
      "log_odds_ratio": -0.555371105670929,
      "logits/chosen": -0.9507812261581421,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.7378906011581421,
      "logps/rejected": -1.265234351158142,
      "loss": 1.1068,
      "nll_loss": 1.115234375,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07376708835363388,
      "rewards/margins": 0.05278320237994194,
      "rewards/rejected": -0.12646484375,
      "step": 960
    },
    {
      "epoch": 0.07360752769767795,
      "grad_norm": 1.4651067791697183,
      "learning_rate": 2.5686451962717425e-06,
      "log_odds_chosen": 0.7742065191268921,
      "log_odds_ratio": -0.534130871295929,
      "logits/chosen": -0.9517577886581421,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.68896484375,
      "logps/rejected": -1.2111327648162842,
      "loss": 1.0792,
      "nll_loss": 1.058984398841858,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06888427585363388,
      "rewards/margins": 0.0521976463496685,
      "rewards/rejected": -0.12116698920726776,
      "step": 970
    },
    {
      "epoch": 0.07436636818940659,
      "grad_norm": 1.3951266787145413,
      "learning_rate": 2.5555062599997596e-06,
      "log_odds_chosen": 0.83544921875,
      "log_odds_ratio": -0.53076171875,
      "logits/chosen": -1.0031249523162842,
      "logits/rejected": -0.918164074420929,
      "logps/chosen": -0.7308593988418579,
      "logps/rejected": -1.3103516101837158,
      "loss": 1.0967,
      "nll_loss": 1.1240234375,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07314453274011612,
      "rewards/margins": 0.05808105319738388,
      "rewards/rejected": -0.13111571967601776,
      "step": 980
    },
    {
      "epoch": 0.07512520868113523,
      "grad_norm": 1.5995079707109392,
      "learning_rate": 2.5425669046549126e-06,
      "log_odds_chosen": 0.740466296672821,
      "log_odds_ratio": -0.553906261920929,
      "logits/chosen": -0.985156238079071,
      "logits/rejected": -0.911914050579071,
      "logps/chosen": -0.7710937261581421,
      "logps/rejected": -1.286523461341858,
      "loss": 1.1016,
      "nll_loss": 1.021875023841858,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07707519829273224,
      "rewards/margins": 0.05153656005859375,
      "rewards/rejected": -0.12871094048023224,
      "step": 990
    },
    {
      "epoch": 0.07588404917286386,
      "grad_norm": 1.6759497570386803,
      "learning_rate": 2.5298221281347034e-06,
      "log_odds_chosen": 1.0483887195587158,
      "log_odds_ratio": -0.4666992127895355,
      "logits/chosen": -1.021875023841858,
      "logits/rejected": -0.919140636920929,
      "logps/chosen": -0.7113281488418579,
      "logps/rejected": -1.450585961341858,
      "loss": 1.0925,
      "nll_loss": 1.025781273841858,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.071044921875,
      "rewards/margins": 0.07399749755859375,
      "rewards/rejected": -0.14519043266773224,
      "step": 1000
    },
    {
      "epoch": 0.0766428896645925,
      "grad_norm": 1.4286639544950612,
      "learning_rate": 2.5172671021102103e-06,
      "log_odds_chosen": 0.787768542766571,
      "log_odds_ratio": -0.5474609136581421,
      "logits/chosen": -0.984179675579071,
      "logits/rejected": -0.923632800579071,
      "logps/chosen": -0.750195324420929,
      "logps/rejected": -1.2976562976837158,
      "loss": 1.0689,
      "nll_loss": 1.0556640625,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07501220703125,
      "rewards/margins": 0.05474243313074112,
      "rewards/rejected": -0.12978515028953552,
      "step": 1010
    },
    {
      "epoch": 0.07740173015632114,
      "grad_norm": 1.6143279270234796,
      "learning_rate": 2.504897164340598e-06,
      "log_odds_chosen": 0.7210693359375,
      "log_odds_ratio": -0.561718761920929,
      "logits/chosen": -0.991015613079071,
      "logits/rejected": -0.899609386920929,
      "logps/chosen": -0.723925769329071,
      "logps/rejected": -1.206640601158142,
      "loss": 1.0968,
      "nll_loss": 1.0457031726837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07241211086511612,
      "rewards/margins": 0.04823608323931694,
      "rewards/rejected": -0.12058105319738388,
      "step": 1020
    },
    {
      "epoch": 0.07816057064804978,
      "grad_norm": 1.5764818072728237,
      "learning_rate": 2.492707811399023e-06,
      "log_odds_chosen": 1.0046875476837158,
      "log_odds_ratio": -0.4649414122104645,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.9419921636581421,
      "logps/chosen": -0.6382812261581421,
      "logps/rejected": -1.292578101158142,
      "loss": 1.0769,
      "nll_loss": 0.949999988079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06381835788488388,
      "rewards/margins": 0.06541137397289276,
      "rewards/rejected": -0.12924805283546448,
      "step": 1030
    },
    {
      "epoch": 0.07891941113977842,
      "grad_norm": 1.558748180429937,
      "learning_rate": 2.480694691784169e-06,
      "log_odds_chosen": 0.866259753704071,
      "log_odds_ratio": -0.484375,
      "logits/chosen": -0.9994140863418579,
      "logits/rejected": -0.906445324420929,
      "logps/chosen": -0.678906261920929,
      "logps/rejected": -1.224218726158142,
      "loss": 1.1036,
      "nll_loss": 1.056640625,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06789550930261612,
      "rewards/margins": 0.05458679050207138,
      "rewards/rejected": -0.12250976264476776,
      "step": 1040
    },
    {
      "epoch": 0.07967825163150706,
      "grad_norm": 1.458824951812319,
      "learning_rate": 2.4688535993934706e-06,
      "log_odds_chosen": 0.820385754108429,
      "log_odds_ratio": -0.5077148675918579,
      "logits/chosen": -1.006445288658142,
      "logits/rejected": -0.925976574420929,
      "logps/chosen": -0.7259765863418579,
      "logps/rejected": -1.2607421875,
      "loss": 1.072,
      "nll_loss": 0.988476574420929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07264403998851776,
      "rewards/margins": 0.05347289890050888,
      "rewards/rejected": -0.12607422471046448,
      "step": 1050
    },
    {
      "epoch": 0.0804370921232357,
      "grad_norm": 1.6596203290820475,
      "learning_rate": 2.457180467335805e-06,
      "log_odds_chosen": 0.642260730266571,
      "log_odds_ratio": -0.6029297113418579,
      "logits/chosen": -0.9710937738418579,
      "logits/rejected": -0.8902343511581421,
      "logps/chosen": -0.794921875,
      "logps/rejected": -1.2492187023162842,
      "loss": 1.0757,
      "nll_loss": 1.033203125,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.0794677734375,
      "rewards/margins": 0.04537353664636612,
      "rewards/rejected": -0.12482909858226776,
      "step": 1060
    },
    {
      "epoch": 0.08119593261496433,
      "grad_norm": 2.2419464467091217,
      "learning_rate": 2.4456713620629725e-06,
      "log_odds_chosen": 0.9271484613418579,
      "log_odds_ratio": -0.5121093988418579,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.878125011920929,
      "logps/chosen": -0.707812488079071,
      "logps/rejected": -1.287109375,
      "loss": 1.1071,
      "nll_loss": 1.131250023841858,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07080078125,
      "rewards/margins": 0.05796203762292862,
      "rewards/rejected": -0.128662109375,
      "step": 1070
    },
    {
      "epoch": 0.08195477310669297,
      "grad_norm": 1.5134269175876427,
      "learning_rate": 2.4343224778007378e-06,
      "log_odds_chosen": 0.7594238519668579,
      "log_odds_ratio": -0.514941394329071,
      "logits/chosen": -0.9462890625,
      "logits/rejected": -0.8998047113418579,
      "logps/chosen": -0.724609375,
      "logps/rejected": -1.2138671875,
      "loss": 1.0804,
      "nll_loss": 1.0427734851837158,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07246093451976776,
      "rewards/margins": 0.04888305813074112,
      "rewards/rejected": -0.12131347507238388,
      "step": 1080
    },
    {
      "epoch": 0.08271361359842161,
      "grad_norm": 1.525216640704541,
      "learning_rate": 2.4231301312615306e-06,
      "log_odds_chosen": 0.975512683391571,
      "log_odds_ratio": -0.5089355707168579,
      "logits/chosen": -0.9750000238418579,
      "logits/rejected": -0.8974609375,
      "logps/chosen": -0.7203124761581421,
      "logps/rejected": -1.395898461341858,
      "loss": 1.0719,
      "nll_loss": 0.993359386920929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07199706882238388,
      "rewards/margins": 0.067596435546875,
      "rewards/rejected": -0.13955077528953552,
      "step": 1090
    },
    {
      "epoch": 0.08347245409015025,
      "grad_norm": 1.4027404178012934,
      "learning_rate": 2.412090756622109e-06,
      "log_odds_chosen": 0.7673705816268921,
      "log_odds_ratio": -0.5249999761581421,
      "logits/chosen": -0.951367199420929,
      "logits/rejected": -0.884960949420929,
      "logps/chosen": -0.738574206829071,
      "logps/rejected": -1.2646484375,
      "loss": 1.0735,
      "nll_loss": 0.994921863079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07377929985523224,
      "rewards/margins": 0.05257873609662056,
      "rewards/rejected": -0.12646484375,
      "step": 1100
    },
    {
      "epoch": 0.08423129458187889,
      "grad_norm": 1.6015960818651562,
      "learning_rate": 2.401200900750657e-06,
      "log_odds_chosen": 0.680188000202179,
      "log_odds_ratio": -0.5425781011581421,
      "logits/chosen": -0.969531238079071,
      "logits/rejected": -0.9166015386581421,
      "logps/chosen": -0.766406238079071,
      "logps/rejected": -1.244531273841858,
      "loss": 1.0568,
      "nll_loss": 0.9955078363418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07664795219898224,
      "rewards/margins": 0.04779357835650444,
      "rewards/rejected": -0.12448730319738388,
      "step": 1110
    },
    {
      "epoch": 0.08499013507360753,
      "grad_norm": 1.4462554991694485,
      "learning_rate": 2.390457218668787e-06,
      "log_odds_chosen": 0.910449206829071,
      "log_odds_ratio": -0.4771484434604645,
      "logits/chosen": -0.9351562261581421,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.7552734613418579,
      "logps/rejected": -1.367773413658142,
      "loss": 1.0932,
      "nll_loss": 1.1320312023162842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07550048828125,
      "rewards/margins": 0.06117553636431694,
      "rewards/rejected": -0.1365966796875,
      "step": 1120
    },
    {
      "epoch": 0.08574897556533617,
      "grad_norm": 1.3771017614094005,
      "learning_rate": 2.379856469234918e-06,
      "log_odds_chosen": 0.89666748046875,
      "log_odds_ratio": -0.4830078184604645,
      "logits/chosen": -0.9310547113418579,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.725781261920929,
      "logps/rejected": -1.322656273841858,
      "loss": 1.0739,
      "nll_loss": 1.034765601158142,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07258300483226776,
      "rewards/margins": 0.05972747877240181,
      "rewards/rejected": -0.13229981064796448,
      "step": 1130
    },
    {
      "epoch": 0.0865078160570648,
      "grad_norm": 1.5724289030509337,
      "learning_rate": 2.369395511036369e-06,
      "log_odds_chosen": 0.9185791015625,
      "log_odds_ratio": -0.48344725370407104,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.904101550579071,
      "logps/chosen": -0.7250000238418579,
      "logps/rejected": -1.345703125,
      "loss": 1.0835,
      "nll_loss": 1.1042969226837158,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07246093451976776,
      "rewards/margins": 0.0621337890625,
      "rewards/rejected": -0.13469238579273224,
      "step": 1140
    },
    {
      "epoch": 0.08726665654879344,
      "grad_norm": 1.5105580372447085,
      "learning_rate": 2.359071298478354e-06,
      "log_odds_chosen": 0.7783447504043579,
      "log_odds_ratio": -0.5380859375,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.8990234136581421,
      "logps/chosen": -0.73828125,
      "logps/rejected": -1.2673828601837158,
      "loss": 1.0531,
      "nll_loss": 1.0275390148162842,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07381591945886612,
      "rewards/margins": 0.05291900783777237,
      "rewards/rejected": -0.12678222358226776,
      "step": 1150
    },
    {
      "epoch": 0.08802549704052208,
      "grad_norm": 1.5063726205865329,
      "learning_rate": 2.3488808780588137e-06,
      "log_odds_chosen": 0.800000011920929,
      "log_odds_ratio": -0.5308593511581421,
      "logits/chosen": -0.9722656011581421,
      "logits/rejected": -0.8876953125,
      "logps/chosen": -0.73486328125,
      "logps/rejected": -1.263671875,
      "loss": 1.0574,
      "nll_loss": 1.08984375,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07351074367761612,
      "rewards/margins": 0.05285186693072319,
      "rewards/rejected": -0.12639160454273224,
      "step": 1160
    },
    {
      "epoch": 0.08878433753225072,
      "grad_norm": 1.640218129700523,
      "learning_rate": 2.3388213848187446e-06,
      "log_odds_chosen": 0.7334228754043579,
      "log_odds_ratio": -0.54052734375,
      "logits/chosen": -0.974414050579071,
      "logits/rejected": -0.880078136920929,
      "logps/chosen": -0.7416015863418579,
      "logps/rejected": -1.2433593273162842,
      "loss": 1.0795,
      "nll_loss": 0.9974609613418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07415771484375,
      "rewards/margins": 0.05020294338464737,
      "rewards/rejected": -0.12434081733226776,
      "step": 1170
    },
    {
      "epoch": 0.08954317802397936,
      "grad_norm": 1.4607546113706387,
      "learning_rate": 2.328890038958328e-06,
      "log_odds_chosen": 0.8072265386581421,
      "log_odds_ratio": -0.529101550579071,
      "logits/chosen": -0.927929699420929,
      "logits/rejected": -0.8916015625,
      "logps/chosen": -0.747265636920929,
      "logps/rejected": -1.2888672351837158,
      "loss": 1.084,
      "nll_loss": 1.023828148841858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07471923530101776,
      "rewards/margins": 0.05414123460650444,
      "rewards/rejected": -0.12890625,
      "step": 1180
    },
    {
      "epoch": 0.090302018515708,
      "grad_norm": 1.590480942561905,
      "learning_rate": 2.3190841426097937e-06,
      "log_odds_chosen": 0.7764648199081421,
      "log_odds_ratio": -0.4970703125,
      "logits/chosen": -0.935546875,
      "logits/rejected": -0.858593761920929,
      "logps/chosen": -0.7076171636581421,
      "logps/rejected": -1.2365233898162842,
      "loss": 1.0711,
      "nll_loss": 1.057031273841858,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07077636569738388,
      "rewards/margins": 0.05285339429974556,
      "rewards/rejected": -0.12360839545726776,
      "step": 1190
    },
    {
      "epoch": 0.09106085900743664,
      "grad_norm": 1.5911920566602926,
      "learning_rate": 2.309401076758503e-06,
      "log_odds_chosen": 0.7508484125137329,
      "log_odds_ratio": -0.55712890625,
      "logits/chosen": -1.025781273841858,
      "logits/rejected": -0.9849609136581421,
      "logps/chosen": -0.7730468511581421,
      "logps/rejected": -1.286718726158142,
      "loss": 1.0467,
      "nll_loss": 1.0251953601837158,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07729492336511612,
      "rewards/margins": 0.05129394680261612,
      "rewards/rejected": -0.12856444716453552,
      "step": 1200
    },
    {
      "epoch": 0.09181969949916527,
      "grad_norm": 1.6321906040324805,
      "learning_rate": 2.299838298304276e-06,
      "log_odds_chosen": 0.943359375,
      "log_odds_ratio": -0.474853515625,
      "logits/chosen": -0.9476562738418579,
      "logits/rejected": -0.896679699420929,
      "logps/chosen": -0.692578136920929,
      "logps/rejected": -1.282812476158142,
      "loss": 1.0844,
      "nll_loss": 1.0271484851837158,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06927490234375,
      "rewards/margins": 0.05902709811925888,
      "rewards/rejected": -0.12834472954273224,
      "step": 1210
    },
    {
      "epoch": 0.09257853999089391,
      "grad_norm": 1.7462695798120413,
      "learning_rate": 2.2903933372554728e-06,
      "log_odds_chosen": 0.7892090082168579,
      "log_odds_ratio": -0.5526367425918579,
      "logits/chosen": -1.009765625,
      "logits/rejected": -0.9203125238418579,
      "logps/chosen": -0.761425793170929,
      "logps/rejected": -1.3087890148162842,
      "loss": 1.0657,
      "nll_loss": 1.0515625476837158,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07618407905101776,
      "rewards/margins": 0.05482025071978569,
      "rewards/rejected": -0.1309814453125,
      "step": 1220
    },
    {
      "epoch": 0.09333738048262255,
      "grad_norm": 1.5708827607202138,
      "learning_rate": 2.281063794048804e-06,
      "log_odds_chosen": 0.9153808355331421,
      "log_odds_ratio": -0.5028320550918579,
      "logits/chosen": -0.988476574420929,
      "logits/rejected": -0.9078124761581421,
      "logps/chosen": -0.7201172113418579,
      "logps/rejected": -1.3582031726837158,
      "loss": 1.0552,
      "nll_loss": 0.9388672113418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07205810397863388,
      "rewards/margins": 0.06379318237304688,
      "rewards/rejected": -0.13588866591453552,
      "step": 1230
    },
    {
      "epoch": 0.09409622097435119,
      "grad_norm": 1.4432294029947919,
      "learning_rate": 2.271847336988259e-06,
      "log_odds_chosen": 0.806347668170929,
      "log_odds_ratio": -0.517871081829071,
      "logits/chosen": -1.0128905773162842,
      "logits/rejected": -0.901562511920929,
      "logps/chosen": -0.7451171875,
      "logps/rejected": -1.31640625,
      "loss": 1.0815,
      "nll_loss": 0.957226574420929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07449951022863388,
      "rewards/margins": 0.05713500827550888,
      "rewards/rejected": -0.13168945908546448,
      "step": 1240
    },
    {
      "epoch": 0.09485506146607983,
      "grad_norm": 1.5543647286170723,
      "learning_rate": 2.262741699796952e-06,
      "log_odds_chosen": 0.958984375,
      "log_odds_ratio": -0.501269519329071,
      "logits/chosen": -0.9544922113418579,
      "logits/rejected": -0.9039062261581421,
      "logps/chosen": -0.7734375,
      "logps/rejected": -1.426171898841858,
      "loss": 1.0468,
      "nll_loss": 1.048437476158142,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07730712741613388,
      "rewards/margins": 0.06527099758386612,
      "rewards/rejected": -0.14252929389476776,
      "step": 1250
    },
    {
      "epoch": 0.09561390195780847,
      "grad_norm": 1.674123203497309,
      "learning_rate": 2.253744679276044e-06,
      "log_odds_chosen": 0.7455810308456421,
      "log_odds_ratio": -0.524609386920929,
      "logits/chosen": -0.959179699420929,
      "logits/rejected": -0.8783203363418579,
      "logps/chosen": -0.7276366949081421,
      "logps/rejected": -1.2091796398162842,
      "loss": 1.0556,
      "nll_loss": 1.010156273841858,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07276611030101776,
      "rewards/margins": 0.04802246019244194,
      "rewards/rejected": -0.12080077826976776,
      "step": 1260
    },
    {
      "epoch": 0.09637274244953711,
      "grad_norm": 1.5730945328477453,
      "learning_rate": 2.244854133065255e-06,
      "log_odds_chosen": 0.91387939453125,
      "log_odds_ratio": -0.529248058795929,
      "logits/chosen": -0.990234375,
      "logits/rejected": -0.9085937738418579,
      "logps/chosen": -0.8076171875,
      "logps/rejected": -1.453515648841858,
      "loss": 1.0977,
      "nll_loss": 1.0802733898162842,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.08072509616613388,
      "rewards/margins": 0.06455688178539276,
      "rewards/rejected": -0.14531250298023224,
      "step": 1270
    },
    {
      "epoch": 0.09713158294126574,
      "grad_norm": 1.564022477389795,
      "learning_rate": 2.2360679774997895e-06,
      "log_odds_chosen": 0.7984374761581421,
      "log_odds_ratio": -0.5282226800918579,
      "logits/chosen": -0.9302734136581421,
      "logits/rejected": -0.863476574420929,
      "logps/chosen": -0.705859363079071,
      "logps/rejected": -1.2312500476837158,
      "loss": 1.0847,
      "nll_loss": 1.0154297351837158,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07062987983226776,
      "rewards/margins": 0.05250854417681694,
      "rewards/rejected": -0.12302245944738388,
      "step": 1280
    },
    {
      "epoch": 0.09789042343299438,
      "grad_norm": 1.498086269280391,
      "learning_rate": 2.2273841855588183e-06,
      "log_odds_chosen": 0.8680175542831421,
      "log_odds_ratio": -0.501660168170929,
      "logits/chosen": -0.9849609136581421,
      "logits/rejected": -0.880078136920929,
      "logps/chosen": -0.716601550579071,
      "logps/rejected": -1.306640625,
      "loss": 1.0609,
      "nll_loss": 1.064453125,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07170410454273224,
      "rewards/margins": 0.05905456468462944,
      "rewards/rejected": -0.1307373046875,
      "step": 1290
    },
    {
      "epoch": 0.09864926392472302,
      "grad_norm": 1.7728116233156204,
      "learning_rate": 2.2188007849009167e-06,
      "log_odds_chosen": 0.8995116949081421,
      "log_odds_ratio": -0.5074707269668579,
      "logits/chosen": -0.966992199420929,
      "logits/rejected": -0.923046886920929,
      "logps/chosen": -0.740039050579071,
      "logps/rejected": -1.3748047351837158,
      "loss": 1.0752,
      "nll_loss": 1.068750023841858,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07395019382238388,
      "rewards/margins": 0.06353454291820526,
      "rewards/rejected": -0.13752441108226776,
      "step": 1300
    },
    {
      "epoch": 0.09940810441645166,
      "grad_norm": 1.7158654014174184,
      "learning_rate": 2.2103158559821502e-06,
      "log_odds_chosen": 1.112890601158142,
      "log_odds_ratio": -0.43603515625,
      "logits/chosen": -0.960156261920929,
      "logits/rejected": -0.853320300579071,
      "logps/chosen": -0.6685546636581421,
      "logps/rejected": -1.397070288658142,
      "loss": 1.03,
      "nll_loss": 0.949999988079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06690673530101776,
      "rewards/margins": 0.072845458984375,
      "rewards/rejected": -0.13969726860523224,
      "step": 1310
    },
    {
      "epoch": 0.1001669449081803,
      "grad_norm": 1.7372509904613194,
      "learning_rate": 2.2019275302527213e-06,
      "log_odds_chosen": 1.05224609375,
      "log_odds_ratio": -0.4795898497104645,
      "logits/chosen": -0.982421875,
      "logits/rejected": -0.8744140863418579,
      "logps/chosen": -0.6839843988418579,
      "logps/rejected": -1.416406273841858,
      "loss": 1.0835,
      "nll_loss": 0.9994140863418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06842041015625,
      "rewards/margins": 0.07326050102710724,
      "rewards/rejected": -0.1417236328125,
      "step": 1320
    },
    {
      "epoch": 0.10092578539990894,
      "grad_norm": 1.5946406192046245,
      "learning_rate": 2.193633988428327e-06,
      "log_odds_chosen": 0.7843261957168579,
      "log_odds_ratio": -0.521679699420929,
      "logits/chosen": -0.9453125,
      "logits/rejected": -0.89453125,
      "logps/chosen": -0.7162109613418579,
      "logps/rejected": -1.2314453125,
      "loss": 1.0827,
      "nll_loss": 1.0789062976837158,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07159423828125,
      "rewards/margins": 0.05157775804400444,
      "rewards/rejected": -0.12312011420726776,
      "step": 1330
    },
    {
      "epoch": 0.10168462589163758,
      "grad_norm": 1.4243961328613546,
      "learning_rate": 2.185433458832612e-06,
      "log_odds_chosen": 0.6023925542831421,
      "log_odds_ratio": -0.574999988079071,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.9302734136581421,
      "logps/chosen": -0.713085949420929,
      "logps/rejected": -1.095703125,
      "loss": 1.0959,
      "nll_loss": 1.042382836341858,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.0712890625,
      "rewards/margins": 0.03832244873046875,
      "rewards/rejected": -0.10966797173023224,
      "step": 1340
    },
    {
      "epoch": 0.10244346638336621,
      "grad_norm": 1.495406382498458,
      "learning_rate": 2.177324215807269e-06,
      "log_odds_chosen": 1.001367211341858,
      "log_odds_ratio": -0.441650390625,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.881640613079071,
      "logps/chosen": -0.700488269329071,
      "logps/rejected": -1.345312476158142,
      "loss": 1.0643,
      "nll_loss": 1.0226562023162842,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07004394382238388,
      "rewards/margins": 0.064483642578125,
      "rewards/rejected": -0.13447265326976776,
      "step": 1350
    },
    {
      "epoch": 0.10320230687509485,
      "grad_norm": 1.4865252925204064,
      "learning_rate": 2.1693045781865616e-06,
      "log_odds_chosen": 1.048974633216858,
      "log_odds_ratio": -0.4736328125,
      "logits/chosen": -0.910937488079071,
      "logits/rejected": -0.861328125,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -1.437109351158142,
      "loss": 1.0463,
      "nll_loss": 1.000390648841858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07167968899011612,
      "rewards/margins": 0.07203521579504013,
      "rewards/rejected": -0.14365234971046448,
      "step": 1360
    },
    {
      "epoch": 0.10396114736682349,
      "grad_norm": 2.887947200974786,
      "learning_rate": 2.1613729078331965e-06,
      "log_odds_chosen": 0.974902331829071,
      "log_odds_ratio": -0.46269530057907104,
      "logits/chosen": -0.9169921875,
      "logits/rejected": -0.8324218988418579,
      "logps/chosen": -0.7001953125,
      "logps/rejected": -1.377343773841858,
      "loss": 1.0644,
      "nll_loss": 0.994140625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07004394382238388,
      "rewards/margins": 0.06776733696460724,
      "rewards/rejected": -0.13779297471046448,
      "step": 1370
    },
    {
      "epoch": 0.10471998785855213,
      "grad_norm": 1.5824770881296906,
      "learning_rate": 2.1535276082326617e-06,
      "log_odds_chosen": 0.932446300983429,
      "log_odds_ratio": -0.5478515625,
      "logits/chosen": -0.9814453125,
      "logits/rejected": -0.9380859136581421,
      "logps/chosen": -0.7708984613418579,
      "logps/rejected": -1.4343750476837158,
      "loss": 1.0571,
      "nll_loss": 1.02734375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07706298679113388,
      "rewards/margins": 0.06638336181640625,
      "rewards/rejected": -0.14340820908546448,
      "step": 1380
    },
    {
      "epoch": 0.10547882835028077,
      "grad_norm": 2.06581191948831,
      "learning_rate": 2.14576712314328e-06,
      "log_odds_chosen": 0.946362316608429,
      "log_odds_ratio": -0.5375000238418579,
      "logits/chosen": -0.9765625,
      "logits/rejected": -0.904101550579071,
      "logps/chosen": -0.7408202886581421,
      "logps/rejected": -1.4210937023162842,
      "loss": 1.0305,
      "nll_loss": 0.9906250238418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.0740966796875,
      "rewards/margins": 0.06792907416820526,
      "rewards/rejected": -0.14211425185203552,
      "step": 1390
    },
    {
      "epoch": 0.10623766884200941,
      "grad_norm": 1.503598713934619,
      "learning_rate": 2.138089935299395e-06,
      "log_odds_chosen": 1.009765625,
      "log_odds_ratio": -0.4737792909145355,
      "logits/chosen": -0.95703125,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.7177734375,
      "logps/rejected": -1.396875023841858,
      "loss": 1.0736,
      "nll_loss": 1.0187499523162842,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07183837890625,
      "rewards/margins": 0.06788787990808487,
      "rewards/rejected": -0.13967284560203552,
      "step": 1400
    },
    {
      "epoch": 0.10699650933373805,
      "grad_norm": 1.492803973103788,
      "learning_rate": 2.1304945651652297e-06,
      "log_odds_chosen": 0.8885742425918579,
      "log_odds_ratio": -0.538867175579071,
      "logits/chosen": -0.9580078125,
      "logits/rejected": -0.864062488079071,
      "logps/chosen": -0.7298828363418579,
      "logps/rejected": -1.323632836341858,
      "loss": 1.0644,
      "nll_loss": 1.1648437976837158,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07294921576976776,
      "rewards/margins": 0.05940551683306694,
      "rewards/rejected": -0.13237304985523224,
      "step": 1410
    },
    {
      "epoch": 0.10775534982546668,
      "grad_norm": 1.665221406365988,
      "learning_rate": 2.122979569737101e-06,
      "log_odds_chosen": 0.9991210699081421,
      "log_odds_ratio": -0.47651368379592896,
      "logits/chosen": -1.0068359375,
      "logits/rejected": -0.9341796636581421,
      "logps/chosen": -0.6558593511581421,
      "logps/rejected": -1.295507788658142,
      "loss": 1.0489,
      "nll_loss": 0.9662109613418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0655517578125,
      "rewards/margins": 0.06397704780101776,
      "rewards/rejected": -0.12954100966453552,
      "step": 1420
    },
    {
      "epoch": 0.10851419031719532,
      "grad_norm": 2.0746542535178603,
      "learning_rate": 2.11554354139178e-06,
      "log_odds_chosen": 0.9710448980331421,
      "log_odds_ratio": -0.5282226800918579,
      "logits/chosen": -0.9888671636581421,
      "logits/rejected": -0.9351562261581421,
      "logps/chosen": -0.8203125,
      "logps/rejected": -1.509765625,
      "loss": 1.0624,
      "nll_loss": 1.026953101158142,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.08211670070886612,
      "rewards/margins": 0.06894836574792862,
      "rewards/rejected": -0.15102538466453552,
      "step": 1430
    },
    {
      "epoch": 0.10927303080892396,
      "grad_norm": 1.592392453133004,
      "learning_rate": 2.1081851067789196e-06,
      "log_odds_chosen": 0.970703125,
      "log_odds_ratio": -0.47978514432907104,
      "logits/chosen": -0.912109375,
      "logits/rejected": -0.851367175579071,
      "logps/chosen": -0.677734375,
      "logps/rejected": -1.3093750476837158,
      "loss": 1.0409,
      "nll_loss": 0.966796875,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0677490234375,
      "rewards/margins": 0.06310882419347763,
      "rewards/rejected": -0.13095703721046448,
      "step": 1440
    },
    {
      "epoch": 0.1100318713006526,
      "grad_norm": 1.622693092846365,
      "learning_rate": 2.1009029257555606e-06,
      "log_odds_chosen": 0.923657238483429,
      "log_odds_ratio": -0.487548828125,
      "logits/chosen": -0.943164050579071,
      "logits/rejected": -0.868359386920929,
      "logps/chosen": -0.6958984136581421,
      "logps/rejected": -1.3054687976837158,
      "loss": 1.0425,
      "nll_loss": 1.00390625,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06960449367761612,
      "rewards/margins": 0.06094970554113388,
      "rewards/rejected": -0.13059082627296448,
      "step": 1450
    },
    {
      "epoch": 0.11079071179238124,
      "grad_norm": 5.009691330274977,
      "learning_rate": 2.0936956903608545e-06,
      "log_odds_chosen": 0.996289074420929,
      "log_odds_ratio": -0.47612303495407104,
      "logits/chosen": -0.931835949420929,
      "logits/rejected": -0.8363281488418579,
      "logps/chosen": -0.7001953125,
      "logps/rejected": -1.349218726158142,
      "loss": 1.0934,
      "nll_loss": 1.058007836341858,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07009277492761612,
      "rewards/margins": 0.06481323391199112,
      "rewards/rejected": -0.13486328721046448,
      "step": 1460
    },
    {
      "epoch": 0.11154955228410988,
      "grad_norm": 1.5133282112647446,
      "learning_rate": 2.0865621238292046e-06,
      "log_odds_chosen": 0.929492175579071,
      "log_odds_ratio": -0.4931640625,
      "logits/chosen": -0.900390625,
      "logits/rejected": -0.8248046636581421,
      "logps/chosen": -0.71484375,
      "logps/rejected": -1.3298828601837158,
      "loss": 1.0446,
      "nll_loss": 1.064062476158142,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.071533203125,
      "rewards/margins": 0.06144409254193306,
      "rewards/rejected": -0.13300780951976776,
      "step": 1470
    },
    {
      "epoch": 0.11230839277583853,
      "grad_norm": 1.6448887742190683,
      "learning_rate": 2.079500979640145e-06,
      "log_odds_chosen": 0.8451293706893921,
      "log_odds_ratio": -0.503125011920929,
      "logits/chosen": -0.9248046875,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.701855480670929,
      "logps/rejected": -1.2658202648162842,
      "loss": 1.0617,
      "nll_loss": 0.9662109613418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0701904296875,
      "rewards/margins": 0.05644073337316513,
      "rewards/rejected": -0.12651367485523224,
      "step": 1480
    },
    {
      "epoch": 0.11306723326756715,
      "grad_norm": 1.6079772952545273,
      "learning_rate": 2.072511040603359e-06,
      "log_odds_chosen": 0.8741699457168579,
      "log_odds_ratio": -0.5518554449081421,
      "logits/chosen": -0.915820300579071,
      "logits/rejected": -0.830859363079071,
      "logps/chosen": -0.749804675579071,
      "logps/rejected": -1.3640625476837158,
      "loss": 1.0472,
      "nll_loss": 0.9970703125,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07498779147863388,
      "rewards/margins": 0.061373136937618256,
      "rewards/rejected": -0.13645020127296448,
      "step": 1490
    },
    {
      "epoch": 0.11382607375929579,
      "grad_norm": 1.5317978336165587,
      "learning_rate": 2.065591117977289e-06,
      "log_odds_chosen": 0.8371948003768921,
      "log_odds_ratio": -0.5206054449081421,
      "logits/chosen": -0.9642578363418579,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.7547851800918579,
      "logps/rejected": -1.3203125,
      "loss": 1.0712,
      "nll_loss": 1.0656249523162842,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07547607272863388,
      "rewards/margins": 0.05657196044921875,
      "rewards/rejected": -0.13210448622703552,
      "step": 1500
    },
    {
      "epoch": 0.11458491425102443,
      "grad_norm": 2.3627141683478374,
      "learning_rate": 2.058740050619915e-06,
      "log_odds_chosen": 1.017968773841858,
      "log_odds_ratio": -0.46826171875,
      "logits/chosen": -1.0009765625,
      "logits/rejected": -0.8857421875,
      "logps/chosen": -0.7376953363418579,
      "logps/rejected": -1.4552733898162842,
      "loss": 1.0386,
      "nll_loss": 0.9716796875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07375488430261612,
      "rewards/margins": 0.07177734375,
      "rewards/rejected": -0.1455078125,
      "step": 1510
    },
    {
      "epoch": 0.11534375474275307,
      "grad_norm": 1.8229312547459466,
      "learning_rate": 2.0519567041703083e-06,
      "log_odds_chosen": 0.9739745855331421,
      "log_odds_ratio": -0.4912109375,
      "logits/chosen": -0.9917968511581421,
      "logits/rejected": -0.9136718511581421,
      "logps/chosen": -0.746874988079071,
      "logps/rejected": -1.441796898841858,
      "loss": 1.0527,
      "nll_loss": 0.9945312738418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07463379204273224,
      "rewards/margins": 0.069488525390625,
      "rewards/rejected": -0.14421387016773224,
      "step": 1520
    },
    {
      "epoch": 0.11610259523448171,
      "grad_norm": 1.781377185920502,
      "learning_rate": 2.0452399702596544e-06,
      "log_odds_chosen": 0.925097644329071,
      "log_odds_ratio": -0.48115235567092896,
      "logits/chosen": -0.938281238079071,
      "logits/rejected": -0.8638671636581421,
      "logps/chosen": -0.7284179925918579,
      "logps/rejected": -1.318750023841858,
      "loss": 1.033,
      "nll_loss": 0.90234375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07282714545726776,
      "rewards/margins": 0.05898742750287056,
      "rewards/rejected": -0.1317138671875,
      "step": 1530
    },
    {
      "epoch": 0.11686143572621036,
      "grad_norm": 1.7487600964578844,
      "learning_rate": 2.0385887657505017e-06,
      "log_odds_chosen": 1.035742163658142,
      "log_odds_ratio": -0.4712890684604645,
      "logits/chosen": -0.9691406488418579,
      "logits/rejected": -0.9072265625,
      "logps/chosen": -0.739453136920929,
      "logps/rejected": -1.449804663658142,
      "loss": 1.0245,
      "nll_loss": 0.939453125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07387695461511612,
      "rewards/margins": 0.071044921875,
      "rewards/rejected": -0.14497070014476776,
      "step": 1540
    },
    {
      "epoch": 0.117620276217939,
      "grad_norm": 1.454469728477322,
      "learning_rate": 2.032002032003048e-06,
      "log_odds_chosen": 0.838940441608429,
      "log_odds_ratio": -0.51806640625,
      "logits/chosen": -0.9722656011581421,
      "logits/rejected": -0.8675781488418579,
      "logps/chosen": -0.70751953125,
      "logps/rejected": -1.2646484375,
      "loss": 1.0354,
      "nll_loss": 1.0078125,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07076416164636612,
      "rewards/margins": 0.05572357028722763,
      "rewards/rejected": -0.12663574516773224,
      "step": 1550
    },
    {
      "epoch": 0.11837911670966762,
      "grad_norm": 1.6474342799970592,
      "learning_rate": 2.025478734167333e-06,
      "log_odds_chosen": 0.848681628704071,
      "log_odds_ratio": -0.5018554925918579,
      "logits/chosen": -0.921875,
      "logits/rejected": -0.810351550579071,
      "logps/chosen": -0.7069336175918579,
      "logps/rejected": -1.231054663658142,
      "loss": 1.0514,
      "nll_loss": 0.983203113079071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07071533054113388,
      "rewards/margins": 0.05249633640050888,
      "rewards/rejected": -0.12314452975988388,
      "step": 1560
    },
    {
      "epoch": 0.11913795720139626,
      "grad_norm": 1.6449397249587745,
      "learning_rate": 2.0190178605002747e-06,
      "log_odds_chosen": 0.708850085735321,
      "log_odds_ratio": -0.52001953125,
      "logits/chosen": -0.9468749761581421,
      "logits/rejected": -0.879101574420929,
      "logps/chosen": -0.728320300579071,
      "logps/rejected": -1.185156226158142,
      "loss": 1.0435,
      "nll_loss": 0.990429699420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07283935695886612,
      "rewards/margins": 0.04575958102941513,
      "rewards/rejected": -0.11857910454273224,
      "step": 1570
    },
    {
      "epoch": 0.1198967976931249,
      "grad_norm": 1.5827948770408402,
      "learning_rate": 2.0126184217065104e-06,
      "log_odds_chosen": 1.0593750476837158,
      "log_odds_ratio": -0.46733397245407104,
      "logits/chosen": -1.004296898841858,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.705078125,
      "logps/rejected": -1.451562523841858,
      "loss": 1.0502,
      "nll_loss": 0.982226550579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07044677436351776,
      "rewards/margins": 0.07470397651195526,
      "rewards/rejected": -0.1451416015625,
      "step": 1580
    },
    {
      "epoch": 0.12065563818485354,
      "grad_norm": 1.6904017282178356,
      "learning_rate": 2.0062794503020765e-06,
      "log_odds_chosen": 1.045166015625,
      "log_odds_ratio": -0.44599610567092896,
      "logits/chosen": -0.9761718511581421,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.6561523675918579,
      "logps/rejected": -1.330078125,
      "loss": 1.0226,
      "nll_loss": 0.9820312261581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06564941257238388,
      "rewards/margins": 0.06737060844898224,
      "rewards/rejected": -0.13310547173023224,
      "step": 1590
    },
    {
      "epoch": 0.12141447867658219,
      "grad_norm": 1.668297709829456,
      "learning_rate": 2e-06,
      "log_odds_chosen": 1.0166747570037842,
      "log_odds_ratio": -0.4783691465854645,
      "logits/chosen": -0.9869140386581421,
      "logits/rejected": -0.8951171636581421,
      "logps/chosen": -0.7251952886581421,
      "logps/rejected": -1.4386718273162842,
      "loss": 1.0305,
      "nll_loss": 1.0388672351837158,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07248535007238388,
      "rewards/margins": 0.07139892876148224,
      "rewards/rejected": -0.14394530653953552,
      "step": 1600
    },
    {
      "epoch": 0.12217331916831083,
      "grad_norm": 1.6682675310046382,
      "learning_rate": 1.993779145116907e-06,
      "log_odds_chosen": 1.00390625,
      "log_odds_ratio": -0.4740234315395355,
      "logits/chosen": -0.9330078363418579,
      "logits/rejected": -0.853710949420929,
      "logps/chosen": -0.6610351800918579,
      "logps/rejected": -1.3292968273162842,
      "loss": 1.0441,
      "nll_loss": 1.013281226158142,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06612548977136612,
      "rewards/margins": 0.0667724609375,
      "rewards/rejected": -0.13295897841453552,
      "step": 1610
    },
    {
      "epoch": 0.12293215966003945,
      "grad_norm": 1.7474719643333534,
      "learning_rate": 1.987615979999813e-06,
      "log_odds_chosen": 0.926440417766571,
      "log_odds_ratio": -0.503369152545929,
      "logits/chosen": -0.912890613079071,
      "logits/rejected": -0.834179699420929,
      "logps/chosen": -0.751953125,
      "logps/rejected": -1.3935546875,
      "loss": 1.0461,
      "nll_loss": 1.0597655773162842,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07512207329273224,
      "rewards/margins": 0.06424407660961151,
      "rewards/rejected": -0.13930663466453552,
      "step": 1620
    },
    {
      "epoch": 0.1236910001517681,
      "grad_norm": 1.6993682853829732,
      "learning_rate": 1.9815096184722797e-06,
      "log_odds_chosen": 0.7530761957168579,
      "log_odds_ratio": -0.5435546636581421,
      "logits/chosen": -0.955273449420929,
      "logits/rejected": -0.853320300579071,
      "logps/chosen": -0.714062511920929,
      "logps/rejected": -1.2175781726837158,
      "loss": 1.0468,
      "nll_loss": 1.0089843273162842,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07138671725988388,
      "rewards/margins": 0.05036468431353569,
      "rewards/rejected": -0.12172851711511612,
      "step": 1630
    },
    {
      "epoch": 0.12444984064349673,
      "grad_norm": 1.6683755597518508,
      "learning_rate": 1.9754591932991793e-06,
      "log_odds_chosen": 1.045678734779358,
      "log_odds_ratio": -0.462646484375,
      "logits/chosen": -1.0167968273162842,
      "logits/rejected": -0.935546875,
      "logps/chosen": -0.717480480670929,
      "logps/rejected": -1.446874976158142,
      "loss": 1.0296,
      "nll_loss": 1.0029296875,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07177734375,
      "rewards/margins": 0.0728912353515625,
      "rewards/rejected": -0.1446533203125,
      "step": 1640
    },
    {
      "epoch": 0.12520868113522537,
      "grad_norm": 1.8852024815139483,
      "learning_rate": 1.9694638556693235e-06,
      "log_odds_chosen": 0.8907715082168579,
      "log_odds_ratio": -0.5005859136581421,
      "logits/chosen": -0.937304675579071,
      "logits/rejected": -0.86328125,
      "logps/chosen": -0.686328113079071,
      "logps/rejected": -1.248437523841858,
      "loss": 1.0571,
      "nll_loss": 0.938671886920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06870117038488388,
      "rewards/margins": 0.056203462183475494,
      "rewards/rejected": -0.12495116889476776,
      "step": 1650
    },
    {
      "epoch": 0.12596752162695402,
      "grad_norm": 1.6508004109020493,
      "learning_rate": 1.963522774695264e-06,
      "log_odds_chosen": 1.0453612804412842,
      "log_odds_ratio": -0.4659667909145355,
      "logits/chosen": -0.986132800579071,
      "logits/rejected": -0.87109375,
      "logps/chosen": -0.696972668170929,
      "logps/rejected": -1.4103515148162842,
      "loss": 1.0341,
      "nll_loss": 0.9345703125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06975097954273224,
      "rewards/margins": 0.0713348388671875,
      "rewards/rejected": -0.14116211235523224,
      "step": 1660
    },
    {
      "epoch": 0.12672636211868266,
      "grad_norm": 1.521239804311641,
      "learning_rate": 1.9576351369295853e-06,
      "log_odds_chosen": 1.26806640625,
      "log_odds_ratio": -0.4283203184604645,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.876757800579071,
      "logps/chosen": -0.6666015386581421,
      "logps/rejected": -1.5470702648162842,
      "loss": 1.0098,
      "nll_loss": 0.956835925579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06671142578125,
      "rewards/margins": 0.08796997368335724,
      "rewards/rejected": -0.15471191704273224,
      "step": 1670
    },
    {
      "epoch": 0.1274852026104113,
      "grad_norm": 1.5577811905592742,
      "learning_rate": 1.951800145897066e-06,
      "log_odds_chosen": 0.88671875,
      "log_odds_ratio": -0.5093749761581421,
      "logits/chosen": -0.9527343511581421,
      "logits/rejected": -0.8958984613418579,
      "logps/chosen": -0.727734386920929,
      "logps/rejected": -1.3302733898162842,
      "loss": 1.0335,
      "nll_loss": 1.004296898841858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07281494140625,
      "rewards/margins": 0.06018371507525444,
      "rewards/rejected": -0.13300780951976776,
      "step": 1680
    },
    {
      "epoch": 0.12824404310213994,
      "grad_norm": 1.7603246411817988,
      "learning_rate": 1.9460170216420797e-06,
      "log_odds_chosen": 0.936328113079071,
      "log_odds_ratio": -0.500732421875,
      "logits/chosen": -0.977343738079071,
      "logits/rejected": -0.8462890386581421,
      "logps/chosen": -0.71875,
      "logps/rejected": -1.3875000476837158,
      "loss": 1.0302,
      "nll_loss": 0.973828136920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07187499850988388,
      "rewards/margins": 0.06686095893383026,
      "rewards/rejected": -0.13859863579273224,
      "step": 1690
    },
    {
      "epoch": 0.12900288359386858,
      "grad_norm": 1.8509322922788338,
      "learning_rate": 1.9402850002906637e-06,
      "log_odds_chosen": 0.94775390625,
      "log_odds_ratio": -0.5142577886581421,
      "logits/chosen": -0.9898437261581421,
      "logits/rejected": -0.9117187261581421,
      "logps/chosen": -0.7020508050918579,
      "logps/rejected": -1.3449218273162842,
      "loss": 1.0089,
      "nll_loss": 0.9169921875,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.0701904296875,
      "rewards/margins": 0.06424560397863388,
      "rewards/rejected": -0.13444824516773224,
      "step": 1700
    },
    {
      "epoch": 0.12976172408559722,
      "grad_norm": 1.6167006582441545,
      "learning_rate": 1.9346033336266974e-06,
      "log_odds_chosen": 0.915332019329071,
      "log_odds_ratio": -0.5011230707168579,
      "logits/chosen": -0.997265636920929,
      "logits/rejected": -0.8988281488418579,
      "logps/chosen": -0.695117175579071,
      "logps/rejected": -1.2619140148162842,
      "loss": 1.0415,
      "nll_loss": 0.981640636920929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06951904296875,
      "rewards/margins": 0.05672607570886612,
      "rewards/rejected": -0.12624511122703552,
      "step": 1710
    },
    {
      "epoch": 0.13052056457732586,
      "grad_norm": 1.6591686774932126,
      "learning_rate": 1.9289712886816486e-06,
      "log_odds_chosen": 0.6905517578125,
      "log_odds_ratio": -0.588671863079071,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.923632800579071,
      "logps/chosen": -0.75,
      "logps/rejected": -1.218359351158142,
      "loss": 1.0406,
      "nll_loss": 0.9466797113418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07501220703125,
      "rewards/margins": 0.04682159423828125,
      "rewards/rejected": -0.12175293266773224,
      "step": 1720
    },
    {
      "epoch": 0.13127940506905447,
      "grad_norm": 1.585308028498714,
      "learning_rate": 1.92338814733738e-06,
      "log_odds_chosen": 1.0014159679412842,
      "log_odds_ratio": -0.4735351502895355,
      "logits/chosen": -0.9554687738418579,
      "logits/rejected": -0.865039050579071,
      "logps/chosen": -0.671679675579071,
      "logps/rejected": -1.2966797351837158,
      "loss": 1.0322,
      "nll_loss": 0.9839843511581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06717529147863388,
      "rewards/margins": 0.06257476657629013,
      "rewards/rejected": -0.12978515028953552,
      "step": 1730
    },
    {
      "epoch": 0.1320382455607831,
      "grad_norm": 1.7984044160498958,
      "learning_rate": 1.9178532059415367e-06,
      "log_odds_chosen": 0.91259765625,
      "log_odds_ratio": -0.5243164300918579,
      "logits/chosen": -0.9722656011581421,
      "logits/rejected": -0.883593738079071,
      "logps/chosen": -0.701953113079071,
      "logps/rejected": -1.354882836341858,
      "loss": 1.0332,
      "nll_loss": 0.9810546636581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0701904296875,
      "rewards/margins": 0.0653533935546875,
      "rewards/rejected": -0.13559570908546448,
      "step": 1740
    },
    {
      "epoch": 0.13279708605251175,
      "grad_norm": 1.6269570980620953,
      "learning_rate": 1.9123657749350298e-06,
      "log_odds_chosen": 0.9396728277206421,
      "log_odds_ratio": -0.529589831829071,
      "logits/chosen": -0.9664062261581421,
      "logits/rejected": -0.855273425579071,
      "logps/chosen": -0.6953125,
      "logps/rejected": -1.3576171398162842,
      "loss": 1.041,
      "nll_loss": 1.004296898841858,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06959228217601776,
      "rewards/margins": 0.0662078857421875,
      "rewards/rejected": -0.13571777939796448,
      "step": 1750
    },
    {
      "epoch": 0.1335559265442404,
      "grad_norm": 1.6259186946124007,
      "learning_rate": 1.9069251784911844e-06,
      "log_odds_chosen": 0.8407348394393921,
      "log_odds_ratio": -0.5379883050918579,
      "logits/chosen": -0.993945300579071,
      "logits/rejected": -0.9056640863418579,
      "logps/chosen": -0.70849609375,
      "logps/rejected": -1.2921874523162842,
      "loss": 1.0583,
      "nll_loss": 0.951953113079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07082519680261612,
      "rewards/margins": 0.05832824856042862,
      "rewards/rejected": -0.12924805283546448,
      "step": 1760
    },
    {
      "epoch": 0.13431476703596903,
      "grad_norm": 1.7328381362199747,
      "learning_rate": 1.9015307541661132e-06,
      "log_odds_chosen": 0.80621337890625,
      "log_odds_ratio": -0.551562488079071,
      "logits/chosen": -0.9144531488418579,
      "logits/rejected": -0.888867199420929,
      "logps/chosen": -0.720703125,
      "logps/rejected": -1.255273461341858,
      "loss": 1.0261,
      "nll_loss": 0.962109386920929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07207031548023224,
      "rewards/margins": 0.053497314453125,
      "rewards/rejected": -0.1256103515625,
      "step": 1770
    },
    {
      "epoch": 0.13507360752769768,
      "grad_norm": 1.6704809725658407,
      "learning_rate": 1.8961818525599089e-06,
      "log_odds_chosen": 0.929638683795929,
      "log_odds_ratio": -0.5340820550918579,
      "logits/chosen": -1.002343773841858,
      "logits/rejected": -0.9302734136581421,
      "logps/chosen": -0.705859363079071,
      "logps/rejected": -1.334375023841858,
      "loss": 1.0152,
      "nll_loss": 0.9947265386581421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07058105617761612,
      "rewards/margins": 0.06288299709558487,
      "rewards/rejected": -0.13349609076976776,
      "step": 1780
    },
    {
      "epoch": 0.13583244801942632,
      "grad_norm": 1.5345342962965025,
      "learning_rate": 1.890877836988262e-06,
      "log_odds_chosen": 0.905834972858429,
      "log_odds_ratio": -0.513378918170929,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.876171886920929,
      "logps/chosen": -0.7416015863418579,
      "logps/rejected": -1.358984351158142,
      "loss": 1.019,
      "nll_loss": 0.966015636920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07421875,
      "rewards/margins": 0.06169281154870987,
      "rewards/rejected": -0.13581542670726776,
      "step": 1790
    },
    {
      "epoch": 0.13659128851115496,
      "grad_norm": 1.6843206698869806,
      "learning_rate": 1.8856180831641269e-06,
      "log_odds_chosen": 1.1140625476837158,
      "log_odds_ratio": -0.4765625,
      "logits/chosen": -0.9654296636581421,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.7650390863418579,
      "logps/rejected": -1.559960961341858,
      "loss": 1.0252,
      "nll_loss": 1.0236327648162842,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07652588188648224,
      "rewards/margins": 0.07954101264476776,
      "rewards/rejected": -0.1561279296875,
      "step": 1800
    },
    {
      "epoch": 0.1373501290028836,
      "grad_norm": 1.5515481390991603,
      "learning_rate": 1.8804019788890737e-06,
      "log_odds_chosen": 1.1181640625,
      "log_odds_ratio": -0.4608398377895355,
      "logits/chosen": -0.9326171875,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.7177734375,
      "logps/rejected": -1.4988281726837158,
      "loss": 1.022,
      "nll_loss": 1.052734375,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07178954780101776,
      "rewards/margins": 0.07809142768383026,
      "rewards/rejected": -0.14985351264476776,
      "step": 1810
    },
    {
      "epoch": 0.13810896949461224,
      "grad_norm": 1.6376595591782022,
      "learning_rate": 1.8752289237539816e-06,
      "log_odds_chosen": 1.2386963367462158,
      "log_odds_ratio": -0.43408203125,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.8837890625,
      "logps/chosen": -0.698925793170929,
      "logps/rejected": -1.574609398841858,
      "loss": 1.0241,
      "nll_loss": 0.9544922113418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06990966945886612,
      "rewards/margins": 0.08756866306066513,
      "rewards/rejected": -0.15737304091453552,
      "step": 1820
    },
    {
      "epoch": 0.13886780998634088,
      "grad_norm": 1.7008481895449235,
      "learning_rate": 1.8700983288487376e-06,
      "log_odds_chosen": 1.112207055091858,
      "log_odds_ratio": -0.4563964903354645,
      "logits/chosen": -1.0037109851837158,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.701953113079071,
      "logps/rejected": -1.4744141101837158,
      "loss": 1.0151,
      "nll_loss": 0.9326171875,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07020263373851776,
      "rewards/margins": 0.0773773193359375,
      "rewards/rejected": -0.14755859971046448,
      "step": 1830
    },
    {
      "epoch": 0.13962665047806952,
      "grad_norm": 1.9175312329784642,
      "learning_rate": 1.8650096164806275e-06,
      "log_odds_chosen": 1.190039038658142,
      "log_odds_ratio": -0.4319824278354645,
      "logits/chosen": -0.9283202886581421,
      "logits/rejected": -0.8541015386581421,
      "logps/chosen": -0.649707019329071,
      "logps/rejected": -1.4619140625,
      "loss": 1.0337,
      "nll_loss": 0.9732421636581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06492920219898224,
      "rewards/margins": 0.08115844428539276,
      "rewards/rejected": -0.14606933295726776,
      "step": 1840
    },
    {
      "epoch": 0.14038549096979816,
      "grad_norm": 2.443144967546165,
      "learning_rate": 1.8599622199011084e-06,
      "log_odds_chosen": 1.020605444908142,
      "log_odds_ratio": -0.4903808534145355,
      "logits/chosen": -0.9642578363418579,
      "logits/rejected": -0.889843761920929,
      "logps/chosen": -0.6878906488418579,
      "logps/rejected": -1.416406273841858,
      "loss": 1.0489,
      "nll_loss": 0.9966796636581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06878662109375,
      "rewards/margins": 0.07291259616613388,
      "rewards/rejected": -0.1417236328125,
      "step": 1850
    },
    {
      "epoch": 0.1411443314615268,
      "grad_norm": 1.663086728989741,
      "learning_rate": 1.854955583040673e-06,
      "log_odds_chosen": 1.0613281726837158,
      "log_odds_ratio": -0.4869140684604645,
      "logits/chosen": -0.9642578363418579,
      "logits/rejected": -0.901171863079071,
      "logps/chosen": -0.7315429449081421,
      "logps/rejected": -1.4523437023162842,
      "loss": 0.9994,
      "nll_loss": 0.9876953363418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07315673679113388,
      "rewards/margins": 0.07206115871667862,
      "rewards/rejected": -0.14519043266773224,
      "step": 1860
    },
    {
      "epoch": 0.1419031719532554,
      "grad_norm": 1.9117980136195871,
      "learning_rate": 1.849989160251521e-06,
      "log_odds_chosen": 0.9891723394393921,
      "log_odds_ratio": -0.5174804925918579,
      "logits/chosen": -0.968554675579071,
      "logits/rejected": -0.898242175579071,
      "logps/chosen": -0.693554699420929,
      "logps/rejected": -1.387304663658142,
      "loss": 1.0329,
      "nll_loss": 0.9984375238418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06934814155101776,
      "rewards/margins": 0.06931991875171661,
      "rewards/rejected": -0.13864746689796448,
      "step": 1870
    },
    {
      "epoch": 0.14266201244498405,
      "grad_norm": 2.0856627792753577,
      "learning_rate": 1.8450624160577701e-06,
      "log_odds_chosen": 1.063867211341858,
      "log_odds_ratio": -0.49394530057907104,
      "logits/chosen": -1.012304663658142,
      "logits/rejected": -0.9169921875,
      "logps/chosen": -0.740234375,
      "logps/rejected": -1.5066406726837158,
      "loss": 1.007,
      "nll_loss": 1.001367211341858,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07404784858226776,
      "rewards/margins": 0.07670287787914276,
      "rewards/rejected": -0.15078124403953552,
      "step": 1880
    },
    {
      "epoch": 0.1434208529367127,
      "grad_norm": 1.576304279008401,
      "learning_rate": 1.8401748249129445e-06,
      "log_odds_chosen": 1.0471680164337158,
      "log_odds_ratio": -0.4671386778354645,
      "logits/chosen": -1.000390648841858,
      "logits/rejected": -0.910351574420929,
      "logps/chosen": -0.6888672113418579,
      "logps/rejected": -1.420507788658142,
      "loss": 1.015,
      "nll_loss": 0.985546886920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06890869140625,
      "rewards/margins": 0.0731201171875,
      "rewards/rejected": -0.14199218153953552,
      "step": 1890
    },
    {
      "epoch": 0.14417969342844134,
      "grad_norm": 2.2943205801406075,
      "learning_rate": 1.8353258709644938e-06,
      "log_odds_chosen": 1.0422852039337158,
      "log_odds_ratio": -0.4927734434604645,
      "logits/chosen": -0.955859363079071,
      "logits/rejected": -0.901562511920929,
      "logps/chosen": -0.7401367425918579,
      "logps/rejected": -1.451562523841858,
      "loss": 1.0349,
      "nll_loss": 1.0314452648162842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07406006008386612,
      "rewards/margins": 0.07120971381664276,
      "rewards/rejected": -0.14523926377296448,
      "step": 1900
    },
    {
      "epoch": 0.14493853392016998,
      "grad_norm": 2.0490705861246323,
      "learning_rate": 1.830515047825102e-06,
      "log_odds_chosen": 0.9310058355331421,
      "log_odds_ratio": -0.512890636920929,
      "logits/chosen": -0.9560546875,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.7147461175918579,
      "logps/rejected": -1.365625023841858,
      "loss": 1.0137,
      "nll_loss": 0.9736328125,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07149658352136612,
      "rewards/margins": 0.06497955322265625,
      "rewards/rejected": -0.13642577826976776,
      "step": 1910
    },
    {
      "epoch": 0.14569737441189862,
      "grad_norm": 1.7749586371627204,
      "learning_rate": 1.8257418583505536e-06,
      "log_odds_chosen": 1.212499976158142,
      "log_odds_ratio": -0.42607420682907104,
      "logits/chosen": -0.954882800579071,
      "logits/rejected": -0.861132800579071,
      "logps/chosen": -0.6689453125,
      "logps/rejected": -1.5017578601837158,
      "loss": 1.0019,
      "nll_loss": 0.922656238079071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06691894680261612,
      "rewards/margins": 0.08331298828125,
      "rewards/rejected": -0.15012207627296448,
      "step": 1920
    },
    {
      "epoch": 0.14645621490362726,
      "grad_norm": 1.6902217952090752,
      "learning_rate": 1.8210058144239416e-06,
      "log_odds_chosen": 0.8382812738418579,
      "log_odds_ratio": -0.5379883050918579,
      "logits/chosen": -0.9527343511581421,
      "logits/rejected": -0.8731445074081421,
      "logps/chosen": -0.73974609375,
      "logps/rejected": -1.356054663658142,
      "loss": 1.0088,
      "nll_loss": 1.009179711341858,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07390137016773224,
      "rewards/margins": 0.06163330003619194,
      "rewards/rejected": -0.13544921576976776,
      "step": 1930
    },
    {
      "epoch": 0.1472150553953559,
      "grad_norm": 1.8073168305465488,
      "learning_rate": 1.816306436745999e-06,
      "log_odds_chosen": 0.97314453125,
      "log_odds_ratio": -0.520800769329071,
      "logits/chosen": -0.963085949420929,
      "logits/rejected": -0.865429699420929,
      "logps/chosen": -0.755078136920929,
      "logps/rejected": -1.433984398841858,
      "loss": 1.0157,
      "nll_loss": 0.952929675579071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07548828423023224,
      "rewards/margins": 0.06783447414636612,
      "rewards/rejected": -0.143310546875,
      "step": 1940
    },
    {
      "epoch": 0.14797389588708454,
      "grad_norm": 2.1030165974817656,
      "learning_rate": 1.8116432546313529e-06,
      "log_odds_chosen": 0.807910144329071,
      "log_odds_ratio": -0.547119140625,
      "logits/chosen": -0.967968761920929,
      "logits/rejected": -0.854296863079071,
      "logps/chosen": -0.7210937738418579,
      "logps/rejected": -1.2625000476837158,
      "loss": 1.0078,
      "nll_loss": 0.974804699420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07205810397863388,
      "rewards/margins": 0.054046630859375,
      "rewards/rejected": -0.12607422471046448,
      "step": 1950
    },
    {
      "epoch": 0.14873273637881318,
      "grad_norm": 1.7734373496612974,
      "learning_rate": 1.8070158058105026e-06,
      "log_odds_chosen": 1.068139672279358,
      "log_odds_ratio": -0.47832030057907104,
      "logits/chosen": -0.9847656488418579,
      "logits/rejected": -0.9007812738418579,
      "logps/chosen": -0.6900390386581421,
      "logps/rejected": -1.4294922351837158,
      "loss": 1.0195,
      "nll_loss": 0.934374988079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06900634616613388,
      "rewards/margins": 0.0738983154296875,
      "rewards/rejected": -0.14304199814796448,
      "step": 1960
    },
    {
      "epoch": 0.14949157687054182,
      "grad_norm": 1.5424244301559844,
      "learning_rate": 1.8024236362373315e-06,
      "log_odds_chosen": 0.9842773675918579,
      "log_odds_ratio": -0.46113282442092896,
      "logits/chosen": -0.9634765386581421,
      "logits/rejected": -0.8519531488418579,
      "logps/chosen": -0.681835949420929,
      "logps/rejected": -1.3474609851837158,
      "loss": 1.0132,
      "nll_loss": 0.969921886920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06818847358226776,
      "rewards/margins": 0.0665283203125,
      "rewards/rejected": -0.13474121689796448,
      "step": 1970
    },
    {
      "epoch": 0.15025041736227046,
      "grad_norm": 4.294685050248751,
      "learning_rate": 1.7978662999019787e-06,
      "log_odds_chosen": 1.1591796875,
      "log_odds_ratio": -0.4561523497104645,
      "logits/chosen": -0.9873046875,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.675000011920929,
      "logps/rejected": -1.481835961341858,
      "loss": 1.018,
      "nll_loss": 0.981640636920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06749267876148224,
      "rewards/margins": 0.080718994140625,
      "rewards/rejected": -0.14836426079273224,
      "step": 1980
    },
    {
      "epoch": 0.1510092578539991,
      "grad_norm": 1.8285917174807669,
      "learning_rate": 1.793343358648881e-06,
      "log_odds_chosen": 1.26025390625,
      "log_odds_ratio": -0.43413084745407104,
      "logits/chosen": -0.942187488079071,
      "logits/rejected": -0.839062511920929,
      "logps/chosen": -0.688281238079071,
      "logps/rejected": -1.578710913658142,
      "loss": 0.9978,
      "nll_loss": 0.955273449420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06879882514476776,
      "rewards/margins": 0.0889892578125,
      "rewards/rejected": -0.15781250596046448,
      "step": 1990
    },
    {
      "epoch": 0.15176809834572771,
      "grad_norm": 1.7246984869527726,
      "learning_rate": 1.7888543819998317e-06,
      "log_odds_chosen": 1.23291015625,
      "log_odds_ratio": -0.46337890625,
      "logits/chosen": -0.979296863079071,
      "logits/rejected": -0.8853515386581421,
      "logps/chosen": -0.7085937261581421,
      "logps/rejected": -1.5851562023162842,
      "loss": 1.0057,
      "nll_loss": 0.959765613079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07087402045726776,
      "rewards/margins": 0.08778686821460724,
      "rewards/rejected": -0.1585693359375,
      "step": 2000
    },
    {
      "epoch": 0.15252693883745636,
      "grad_norm": 1.8078525016188747,
      "learning_rate": 1.7843989469818819e-06,
      "log_odds_chosen": 0.66796875,
      "log_odds_ratio": -0.581250011920929,
      "logits/chosen": -0.9097656011581421,
      "logits/rejected": -0.8248046636581421,
      "logps/chosen": -0.721484363079071,
      "logps/rejected": -1.169531226158142,
      "loss": 1.0303,
      "nll_loss": 0.930859386920929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07215575873851776,
      "rewards/margins": 0.044759370386600494,
      "rewards/rejected": -0.11696777492761612,
      "step": 2010
    },
    {
      "epoch": 0.153285779329185,
      "grad_norm": 1.750955590287212,
      "learning_rate": 1.779976637959939e-06,
      "log_odds_chosen": 1.244970679283142,
      "log_odds_ratio": -0.44819337129592896,
      "logits/chosen": -0.9283202886581421,
      "logits/rejected": -0.814648449420929,
      "logps/chosen": -0.7040039300918579,
      "logps/rejected": -1.57421875,
      "loss": 1.0132,
      "nll_loss": 0.975781261920929,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07039795070886612,
      "rewards/margins": 0.0870361328125,
      "rewards/rejected": -0.15751953423023224,
      "step": 2020
    },
    {
      "epoch": 0.15404461982091364,
      "grad_norm": 1.8459833504202043,
      "learning_rate": 1.7755870464739012e-06,
      "log_odds_chosen": 0.91015625,
      "log_odds_ratio": -0.5287109613418579,
      "logits/chosen": -0.950976550579071,
      "logits/rejected": -0.8560546636581421,
      "logps/chosen": -0.6919921636581421,
      "logps/rejected": -1.326562523841858,
      "loss": 1.0335,
      "nll_loss": 0.981249988079071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06920166313648224,
      "rewards/margins": 0.06353607028722763,
      "rewards/rejected": -0.13271483778953552,
      "step": 2030
    },
    {
      "epoch": 0.15480346031264228,
      "grad_norm": 1.7056677979498798,
      "learning_rate": 1.7712297710801907e-06,
      "log_odds_chosen": 1.076074242591858,
      "log_odds_ratio": -0.47509765625,
      "logits/chosen": -0.9117187261581421,
      "logits/rejected": -0.833789050579071,
      "logps/chosen": -0.6922851800918579,
      "logps/rejected": -1.421484351158142,
      "loss": 1.0171,
      "nll_loss": 0.967968761920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0692138671875,
      "rewards/margins": 0.07306518405675888,
      "rewards/rejected": -0.14218750596046448,
      "step": 2040
    },
    {
      "epoch": 0.15556230080437092,
      "grad_norm": 1.605351728289974,
      "learning_rate": 1.7669044171975444e-06,
      "log_odds_chosen": 1.0661499500274658,
      "log_odds_ratio": -0.4999023377895355,
      "logits/chosen": -0.9091796875,
      "logits/rejected": -0.8255859613418579,
      "logps/chosen": -0.7171875238418579,
      "logps/rejected": -1.4519531726837158,
      "loss": 1.0172,
      "nll_loss": 0.9818359613418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07167968899011612,
      "rewards/margins": 0.07357482612133026,
      "rewards/rejected": -0.14523926377296448,
      "step": 2050
    },
    {
      "epoch": 0.15632114129609956,
      "grad_norm": 1.6960342772381833,
      "learning_rate": 1.7626105969569268e-06,
      "log_odds_chosen": 1.058984398841858,
      "log_odds_ratio": -0.460693359375,
      "logits/chosen": -0.9281250238418579,
      "logits/rejected": -0.8433593511581421,
      "logps/chosen": -0.7040039300918579,
      "logps/rejected": -1.4226562976837158,
      "loss": 1.0291,
      "nll_loss": 1.0041015148162842,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07033691555261612,
      "rewards/margins": 0.07182617485523224,
      "rewards/rejected": -0.14223632216453552,
      "step": 2060
    },
    {
      "epoch": 0.1570799817878282,
      "grad_norm": 2.0842118121138205,
      "learning_rate": 1.758347929055432e-06,
      "log_odds_chosen": 1.00048828125,
      "log_odds_ratio": -0.455810546875,
      "logits/chosen": -0.9208984375,
      "logits/rejected": -0.8349609375,
      "logps/chosen": -0.6654297113418579,
      "logps/rejected": -1.3132812976837158,
      "loss": 0.9886,
      "nll_loss": 0.919140636920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06655273586511612,
      "rewards/margins": 0.06477661430835724,
      "rewards/rejected": -0.13137206435203552,
      "step": 2070
    },
    {
      "epoch": 0.15783882227955684,
      "grad_norm": 2.120569477558702,
      "learning_rate": 1.7541160386140582e-06,
      "log_odds_chosen": 1.3015625476837158,
      "log_odds_ratio": -0.4002929627895355,
      "logits/chosen": -0.9339843988418579,
      "logits/rejected": -0.81640625,
      "logps/chosen": -0.648242175579071,
      "logps/rejected": -1.5167968273162842,
      "loss": 0.9989,
      "nll_loss": 0.984375,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06483153998851776,
      "rewards/margins": 0.08698730170726776,
      "rewards/rejected": -0.15175780653953552,
      "step": 2080
    },
    {
      "epoch": 0.15859766277128548,
      "grad_norm": 1.6539249012178663,
      "learning_rate": 1.7499145570392284e-06,
      "log_odds_chosen": 1.0309326648712158,
      "log_odds_ratio": -0.49125975370407104,
      "logits/chosen": -1.0031249523162842,
      "logits/rejected": -0.8828125,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -1.4093749523162842,
      "loss": 1.002,
      "nll_loss": 0.9527343511581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07062987983226776,
      "rewards/margins": 0.07036437839269638,
      "rewards/rejected": -0.140869140625,
      "step": 2090
    },
    {
      "epoch": 0.15935650326301412,
      "grad_norm": 1.6299919394171718,
      "learning_rate": 1.745743121887939e-06,
      "log_odds_chosen": 0.9908691644668579,
      "log_odds_ratio": -0.49169921875,
      "logits/chosen": -0.9585937261581421,
      "logits/rejected": -0.833984375,
      "logps/chosen": -0.6826171875,
      "logps/rejected": -1.361914038658142,
      "loss": 1.0307,
      "nll_loss": 0.9857422113418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06827392429113388,
      "rewards/margins": 0.06794128566980362,
      "rewards/rejected": -0.13613280653953552,
      "step": 2100
    },
    {
      "epoch": 0.16011534375474276,
      "grad_norm": 1.5610258992867714,
      "learning_rate": 1.7416013767364324e-06,
      "log_odds_chosen": 0.9908202886581421,
      "log_odds_ratio": -0.4573730528354645,
      "logits/chosen": -0.960156261920929,
      "logits/rejected": -0.8716796636581421,
      "logps/chosen": -0.6415039300918579,
      "logps/rejected": -1.2478516101837158,
      "loss": 0.9743,
      "nll_loss": 0.8705078363418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06418456882238388,
      "rewards/margins": 0.06059875339269638,
      "rewards/rejected": -0.12473144382238388,
      "step": 2110
    },
    {
      "epoch": 0.1608741842464714,
      "grad_norm": 1.7782745410513578,
      "learning_rate": 1.7374889710522776e-06,
      "log_odds_chosen": 1.322363257408142,
      "log_odds_ratio": -0.4240478575229645,
      "logits/chosen": -0.943554699420929,
      "logits/rejected": -0.85546875,
      "logps/chosen": -0.656542956829071,
      "logps/rejected": -1.5681641101837158,
      "loss": 0.9845,
      "nll_loss": 0.860156238079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06564941257238388,
      "rewards/margins": 0.09116820991039276,
      "rewards/rejected": -0.15683594346046448,
      "step": 2120
    },
    {
      "epoch": 0.16163302473820004,
      "grad_norm": 1.6843050386022687,
      "learning_rate": 1.7334055600697579e-06,
      "log_odds_chosen": 1.218359351158142,
      "log_odds_ratio": -0.44462889432907104,
      "logits/chosen": -0.924023449420929,
      "logits/rejected": -0.867382824420929,
      "logps/chosen": -0.6788085699081421,
      "logps/rejected": -1.494726538658142,
      "loss": 1.0162,
      "nll_loss": 0.98046875,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06787109375,
      "rewards/margins": 0.08159790188074112,
      "rewards/rejected": -0.14948730170726776,
      "step": 2130
    },
    {
      "epoch": 0.16239186522992866,
      "grad_norm": 1.8290613554640052,
      "learning_rate": 1.7293508046684678e-06,
      "log_odds_chosen": 0.8995605707168579,
      "log_odds_ratio": -0.5249999761581421,
      "logits/chosen": -0.9195312261581421,
      "logits/rejected": -0.853320300579071,
      "logps/chosen": -0.704296886920929,
      "logps/rejected": -1.318359375,
      "loss": 0.9991,
      "nll_loss": 0.9419921636581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07045898586511612,
      "rewards/margins": 0.06148681789636612,
      "rewards/rejected": -0.13188476860523224,
      "step": 2140
    },
    {
      "epoch": 0.1631507057216573,
      "grad_norm": 1.7340261251194247,
      "learning_rate": 1.7253243712550145e-06,
      "log_odds_chosen": 0.962109386920929,
      "log_odds_ratio": -0.5,
      "logits/chosen": -0.900390625,
      "logits/rejected": -0.802441418170929,
      "logps/chosen": -0.6602538824081421,
      "logps/rejected": -1.2755858898162842,
      "loss": 1.0306,
      "nll_loss": 1.052148461341858,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06602783501148224,
      "rewards/margins": 0.06157531589269638,
      "rewards/rejected": -0.12758788466453552,
      "step": 2150
    },
    {
      "epoch": 0.16390954621338594,
      "grad_norm": 1.6987763744425957,
      "learning_rate": 1.7213259316477406e-06,
      "log_odds_chosen": 1.0940430164337158,
      "log_odds_ratio": -0.5050293207168579,
      "logits/chosen": -0.9619140625,
      "logits/rejected": -0.8824218511581421,
      "logps/chosen": -0.757031261920929,
      "logps/rejected": -1.5478515625,
      "loss": 1.0029,
      "nll_loss": 0.9525390863418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07576904445886612,
      "rewards/margins": 0.07902832329273224,
      "rewards/rejected": -0.15480956435203552,
      "step": 2160
    },
    {
      "epoch": 0.16466838670511458,
      "grad_norm": 1.9183213512351749,
      "learning_rate": 1.7173551629643674e-06,
      "log_odds_chosen": 1.0694091320037842,
      "log_odds_ratio": -0.48066407442092896,
      "logits/chosen": -0.904101550579071,
      "logits/rejected": -0.8335937261581421,
      "logps/chosen": -0.738085925579071,
      "logps/rejected": -1.4951171875,
      "loss": 1.0171,
      "nll_loss": 1.003515601158142,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07376708835363388,
      "rewards/margins": 0.07562103122472763,
      "rewards/rejected": -0.14951172471046448,
      "step": 2170
    },
    {
      "epoch": 0.16542722719684322,
      "grad_norm": 1.7591486737480413,
      "learning_rate": 1.713411747512477e-06,
      "log_odds_chosen": 1.2626953125,
      "log_odds_ratio": -0.4422851502895355,
      "logits/chosen": -0.9052734375,
      "logits/rejected": -0.803906261920929,
      "logps/chosen": -0.6700195074081421,
      "logps/rejected": -1.5408203601837158,
      "loss": 1.0147,
      "nll_loss": 0.922656238079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06697998195886612,
      "rewards/margins": 0.08709716796875,
      "rewards/rejected": -0.15410156548023224,
      "step": 2180
    },
    {
      "epoch": 0.16618606768857186,
      "grad_norm": 2.2730392023754424,
      "learning_rate": 1.709495372682753e-06,
      "log_odds_chosen": 1.2048828601837158,
      "log_odds_ratio": -0.47001951932907104,
      "logits/chosen": -0.890429675579071,
      "logits/rejected": -0.804492175579071,
      "logps/chosen": -0.738085925579071,
      "logps/rejected": -1.5867187976837158,
      "loss": 0.9889,
      "nll_loss": 0.924023449420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07381591945886612,
      "rewards/margins": 0.08494262397289276,
      "rewards/rejected": -0.15876464545726776,
      "step": 2190
    },
    {
      "epoch": 0.1669449081803005,
      "grad_norm": 2.037458706640104,
      "learning_rate": 1.7056057308448832e-06,
      "log_odds_chosen": 1.121679663658142,
      "log_odds_ratio": -0.43730467557907104,
      "logits/chosen": -0.916210949420929,
      "logits/rejected": -0.8021484613418579,
      "logps/chosen": -0.645312488079071,
      "logps/rejected": -1.4080078601837158,
      "loss": 0.9718,
      "nll_loss": 0.9302734136581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06459961086511612,
      "rewards/margins": 0.07626952975988388,
      "rewards/rejected": -0.14077147841453552,
      "step": 2200
    },
    {
      "epoch": 0.16770374867202914,
      "grad_norm": 2.046099172625031,
      "learning_rate": 1.701742519246068e-06,
      "log_odds_chosen": 0.951708972454071,
      "log_odds_ratio": -0.4844726622104645,
      "logits/chosen": -0.9341796636581421,
      "logits/rejected": -0.8324218988418579,
      "logps/chosen": -0.7157226800918579,
      "logps/rejected": -1.3728516101837158,
      "loss": 1.0105,
      "nll_loss": 0.9869140386581421,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07159423828125,
      "rewards/margins": 0.065643310546875,
      "rewards/rejected": -0.13715820014476776,
      "step": 2210
    },
    {
      "epoch": 0.16846258916375778,
      "grad_norm": 1.6540454740677957,
      "learning_rate": 1.6979054399120355e-06,
      "log_odds_chosen": 1.15185546875,
      "log_odds_ratio": -0.47382813692092896,
      "logits/chosen": -0.906054675579071,
      "logits/rejected": -0.828320324420929,
      "logps/chosen": -0.669628918170929,
      "logps/rejected": -1.450781226158142,
      "loss": 0.9966,
      "nll_loss": 0.908984363079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06693115085363388,
      "rewards/margins": 0.07802734524011612,
      "rewards/rejected": -0.14506836235523224,
      "step": 2220
    },
    {
      "epoch": 0.16922142965548642,
      "grad_norm": 1.6756053571033922,
      "learning_rate": 1.6940941995505069e-06,
      "log_odds_chosen": 0.881176769733429,
      "log_odds_ratio": -0.536328136920929,
      "logits/chosen": -0.912890613079071,
      "logits/rejected": -0.8296874761581421,
      "logps/chosen": -0.773242175579071,
      "logps/rejected": -1.3869140148162842,
      "loss": 1.0266,
      "nll_loss": 1.0226562023162842,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07731933891773224,
      "rewards/margins": 0.06138915941119194,
      "rewards/rejected": -0.13857421278953552,
      "step": 2230
    },
    {
      "epoch": 0.16998027014721506,
      "grad_norm": 1.555734920343979,
      "learning_rate": 1.6903085094570331e-06,
      "log_odds_chosen": 0.941113293170929,
      "log_odds_ratio": -0.49560546875,
      "logits/chosen": -0.943359375,
      "logits/rejected": -0.8056640625,
      "logps/chosen": -0.69580078125,
      "logps/rejected": -1.3240234851837158,
      "loss": 0.9995,
      "nll_loss": 0.9052734375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06951904296875,
      "rewards/margins": 0.06293945014476776,
      "rewards/rejected": -0.13237304985523224,
      "step": 2240
    },
    {
      "epoch": 0.1707391106389437,
      "grad_norm": 1.7021231359836484,
      "learning_rate": 1.6865480854231356e-06,
      "log_odds_chosen": 0.9696289300918579,
      "log_odds_ratio": -0.51904296875,
      "logits/chosen": -0.891796886920929,
      "logits/rejected": -0.807421863079071,
      "logps/chosen": -0.720898449420929,
      "logps/rejected": -1.3908202648162842,
      "loss": 1.0088,
      "nll_loss": 0.9750000238418579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07203368842601776,
      "rewards/margins": 0.06697998195886612,
      "rewards/rejected": -0.13901367783546448,
      "step": 2250
    },
    {
      "epoch": 0.17149795113067234,
      "grad_norm": 1.657905024680801,
      "learning_rate": 1.682812647646685e-06,
      "log_odds_chosen": 1.2488281726837158,
      "log_odds_ratio": -0.39140623807907104,
      "logits/chosen": -0.9117187261581421,
      "logits/rejected": -0.821484386920929,
      "logps/chosen": -0.67041015625,
      "logps/rejected": -1.491796851158142,
      "loss": 0.9972,
      "nll_loss": 0.909375011920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06705322116613388,
      "rewards/margins": 0.08208008110523224,
      "rewards/rejected": -0.14914551377296448,
      "step": 2260
    },
    {
      "epoch": 0.17225679162240098,
      "grad_norm": 1.6187004410213444,
      "learning_rate": 1.6791019206444541e-06,
      "log_odds_chosen": 0.9906250238418579,
      "log_odds_ratio": -0.4784179627895355,
      "logits/chosen": -0.935351550579071,
      "logits/rejected": -0.831250011920929,
      "logps/chosen": -0.707324206829071,
      "logps/rejected": -1.4052734375,
      "loss": 1.0238,
      "nll_loss": 0.9925781488418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07073974609375,
      "rewards/margins": 0.06994018703699112,
      "rewards/rejected": -0.140625,
      "step": 2270
    },
    {
      "epoch": 0.1730156321141296,
      "grad_norm": 2.3583378920089197,
      "learning_rate": 1.675415633166782e-06,
      "log_odds_chosen": 1.3103516101837158,
      "log_odds_ratio": -0.42724609375,
      "logits/chosen": -0.949023425579071,
      "logits/rejected": -0.8480468988418579,
      "logps/chosen": -0.6993163824081421,
      "logps/rejected": -1.608984351158142,
      "loss": 0.9873,
      "nll_loss": 0.891406238079071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06988525390625,
      "rewards/margins": 0.09085693210363388,
      "rewards/rejected": -0.160888671875,
      "step": 2280
    },
    {
      "epoch": 0.17377447260585824,
      "grad_norm": 1.8684296943890162,
      "learning_rate": 1.6717535181142914e-06,
      "log_odds_chosen": 0.8514159917831421,
      "log_odds_ratio": -0.5181640386581421,
      "logits/chosen": -0.957812488079071,
      "logits/rejected": -0.8353515863418579,
      "logps/chosen": -0.724609375,
      "logps/rejected": -1.302734375,
      "loss": 1.0093,
      "nll_loss": 0.948437511920929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07241211086511612,
      "rewards/margins": 0.05777130275964737,
      "rewards/rejected": -0.13020019233226776,
      "step": 2290
    },
    {
      "epoch": 0.17453331309758688,
      "grad_norm": 1.7443057982305137,
      "learning_rate": 1.668115312456598e-06,
      "log_odds_chosen": 0.9159179925918579,
      "log_odds_ratio": -0.482666015625,
      "logits/chosen": -0.918749988079071,
      "logits/rejected": -0.8291015625,
      "logps/chosen": -0.676953136920929,
      "logps/rejected": -1.273828148841858,
      "loss": 0.9937,
      "nll_loss": 0.9429687261581421,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06765136867761612,
      "rewards/margins": 0.05970611423254013,
      "rewards/rejected": -0.12736816704273224,
      "step": 2300
    },
    {
      "epoch": 0.17529215358931552,
      "grad_norm": 1.7495476536042223,
      "learning_rate": 1.6645007571529578e-06,
      "log_odds_chosen": 1.193359375,
      "log_odds_ratio": -0.4368896484375,
      "logits/chosen": -0.9228515625,
      "logits/rejected": -0.783984363079071,
      "logps/chosen": -0.710156261920929,
      "logps/rejected": -1.5369141101837158,
      "loss": 0.9818,
      "nll_loss": 0.991406261920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07098388671875,
      "rewards/margins": 0.082611083984375,
      "rewards/rejected": -0.15371093153953552,
      "step": 2310
    },
    {
      "epoch": 0.17605099408104416,
      "grad_norm": 2.061978268838713,
      "learning_rate": 1.6609095970747992e-06,
      "log_odds_chosen": 0.933422863483429,
      "log_odds_ratio": -0.5176757574081421,
      "logits/chosen": -0.9085937738418579,
      "logits/rejected": -0.8267577886581421,
      "logps/chosen": -0.770312488079071,
      "logps/rejected": -1.447265625,
      "loss": 1.0063,
      "nll_loss": 1.0353515148162842,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07706298679113388,
      "rewards/margins": 0.06778564304113388,
      "rewards/rejected": -0.14484862983226776,
      "step": 2320
    },
    {
      "epoch": 0.1768098345727728,
      "grad_norm": 1.9864693037076206,
      "learning_rate": 1.6573415809300833e-06,
      "log_odds_chosen": 0.85400390625,
      "log_odds_ratio": -0.526562511920929,
      "logits/chosen": -0.8955078125,
      "logits/rejected": -0.846484363079071,
      "logps/chosen": -0.7152343988418579,
      "logps/rejected": -1.2712891101837158,
      "loss": 0.999,
      "nll_loss": 1.0095703601837158,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07152099907398224,
      "rewards/margins": 0.05556793138384819,
      "rewards/rejected": -0.12712402641773224,
      "step": 2330
    },
    {
      "epoch": 0.17756867506450144,
      "grad_norm": 1.5250036352910814,
      "learning_rate": 1.6537964611894462e-06,
      "log_odds_chosen": 0.8726806640625,
      "log_odds_ratio": -0.4984374940395355,
      "logits/chosen": -0.9457031488418579,
      "logits/rejected": -0.881054699420929,
      "logps/chosen": -0.6864258050918579,
      "logps/rejected": -1.264257788658142,
      "loss": 0.9864,
      "nll_loss": 0.995898425579071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06871338188648224,
      "rewards/margins": 0.05774535983800888,
      "rewards/rejected": -0.12641601264476776,
      "step": 2340
    },
    {
      "epoch": 0.17832751555623008,
      "grad_norm": 1.7843935011591716,
      "learning_rate": 1.6502739940140692e-06,
      "log_odds_chosen": 0.805798351764679,
      "log_odds_ratio": -0.510449230670929,
      "logits/chosen": -0.935546875,
      "logits/rejected": -0.809765636920929,
      "logps/chosen": -0.7056640386581421,
      "logps/rejected": -1.2316405773162842,
      "loss": 1.0055,
      "nll_loss": 1.05859375,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07052002102136612,
      "rewards/margins": 0.05268402025103569,
      "rewards/rejected": -0.12321777641773224,
      "step": 2350
    },
    {
      "epoch": 0.17908635604795872,
      "grad_norm": 1.7329387674163914,
      "learning_rate": 1.6467739391852364e-06,
      "log_odds_chosen": 0.932861328125,
      "log_odds_ratio": -0.492919921875,
      "logits/chosen": -0.903124988079071,
      "logits/rejected": -0.8291015625,
      "logps/chosen": -0.696582019329071,
      "logps/rejected": -1.316015601158142,
      "loss": 0.972,
      "nll_loss": 0.9710937738418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06962890923023224,
      "rewards/margins": 0.06194610521197319,
      "rewards/rejected": -0.13166503608226776,
      "step": 2360
    },
    {
      "epoch": 0.17984519653968736,
      "grad_norm": 1.6461497139628067,
      "learning_rate": 1.6432960600355221e-06,
      "log_odds_chosen": 1.0046875476837158,
      "log_odds_ratio": -0.46044921875,
      "logits/chosen": -0.933398425579071,
      "logits/rejected": -0.8314453363418579,
      "logps/chosen": -0.6783202886581421,
      "logps/rejected": -1.338281273841858,
      "loss": 0.9891,
      "nll_loss": 0.9117187261581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06779785454273224,
      "rewards/margins": 0.0659637451171875,
      "rewards/rejected": -0.13381347060203552,
      "step": 2370
    },
    {
      "epoch": 0.180604037031416,
      "grad_norm": 1.925964568167815,
      "learning_rate": 1.6398401233815756e-06,
      "log_odds_chosen": 1.0750000476837158,
      "log_odds_ratio": -0.47919923067092896,
      "logits/chosen": -0.955078125,
      "logits/rejected": -0.859375,
      "logps/chosen": -0.7408202886581421,
      "logps/rejected": -1.4873046875,
      "loss": 0.9939,
      "nll_loss": 0.900585949420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07413329929113388,
      "rewards/margins": 0.07451782375574112,
      "rewards/rejected": -0.14860840141773224,
      "step": 2380
    },
    {
      "epoch": 0.18136287752314464,
      "grad_norm": 1.8137519236514643,
      "learning_rate": 1.6364058994584524e-06,
      "log_odds_chosen": 0.9222656488418579,
      "log_odds_ratio": -0.5028320550918579,
      "logits/chosen": -0.8755859136581421,
      "logits/rejected": -0.799023449420929,
      "logps/chosen": -0.7115234136581421,
      "logps/rejected": -1.3359375,
      "loss": 0.9888,
      "nll_loss": 0.9359375238418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07113037258386612,
      "rewards/margins": 0.06235351413488388,
      "rewards/rejected": -0.13352051377296448,
      "step": 2390
    },
    {
      "epoch": 0.18212171801487329,
      "grad_norm": 1.688614794458292,
      "learning_rate": 1.6329931618554522e-06,
      "log_odds_chosen": 1.150390625,
      "log_odds_ratio": -0.453125,
      "logits/chosen": -0.930468738079071,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.686230480670929,
      "logps/rejected": -1.476171851158142,
      "loss": 0.9823,
      "nll_loss": 0.946484386920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06862793117761612,
      "rewards/margins": 0.079010009765625,
      "rewards/rejected": -0.14760741591453552,
      "step": 2400
    },
    {
      "epoch": 0.18288055850660193,
      "grad_norm": 1.8014616595320827,
      "learning_rate": 1.6296016874534209e-06,
      "log_odds_chosen": 1.108154296875,
      "log_odds_ratio": -0.458984375,
      "logits/chosen": -0.911328136920929,
      "logits/rejected": -0.850390613079071,
      "logps/chosen": -0.693554699420929,
      "logps/rejected": -1.4207031726837158,
      "loss": 0.9831,
      "nll_loss": 0.981640636920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06934814155101776,
      "rewards/margins": 0.07284469902515411,
      "rewards/rejected": -0.14213867485523224,
      "step": 2410
    },
    {
      "epoch": 0.18363939899833054,
      "grad_norm": 1.7304693115361636,
      "learning_rate": 1.6262312563634835e-06,
      "log_odds_chosen": 1.104882836341858,
      "log_odds_ratio": -0.4971679747104645,
      "logits/chosen": -0.998828113079071,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.7134765386581421,
      "logps/rejected": -1.490820288658142,
      "loss": 0.9875,
      "nll_loss": 0.934765636920929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07139892876148224,
      "rewards/margins": 0.07766113430261612,
      "rewards/rejected": -0.14907225966453552,
      "step": 2420
    },
    {
      "epoch": 0.18439823949005918,
      "grad_norm": 2.032034928793834,
      "learning_rate": 1.6228816518671587e-06,
      "log_odds_chosen": 1.02294921875,
      "log_odds_ratio": -0.4745117127895355,
      "logits/chosen": -0.957812488079071,
      "logits/rejected": -0.842968761920929,
      "logps/chosen": -0.710156261920929,
      "logps/rejected": -1.451171875,
      "loss": 0.9879,
      "nll_loss": 0.9244140386581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07102050632238388,
      "rewards/margins": 0.07415161281824112,
      "rewards/rejected": -0.14511719346046448,
      "step": 2430
    },
    {
      "epoch": 0.18515707998178782,
      "grad_norm": 2.0892961715952696,
      "learning_rate": 1.619552660357832e-06,
      "log_odds_chosen": 1.128198266029358,
      "log_odds_ratio": -0.49091798067092896,
      "logits/chosen": -0.934765636920929,
      "logits/rejected": -0.8541015386581421,
      "logps/chosen": -0.7372070550918579,
      "logps/rejected": -1.5187499523162842,
      "loss": 0.99,
      "nll_loss": 0.9781249761581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07374267280101776,
      "rewards/margins": 0.07811889797449112,
      "rewards/rejected": -0.15183106064796448,
      "step": 2440
    },
    {
      "epoch": 0.18591592047351646,
      "grad_norm": 1.66781243666752,
      "learning_rate": 1.616244071283537e-06,
      "log_odds_chosen": 1.009863257408142,
      "log_odds_ratio": -0.4766601622104645,
      "logits/chosen": -0.9652343988418579,
      "logits/rejected": -0.8421875238418579,
      "logps/chosen": -0.7496093511581421,
      "logps/rejected": -1.4240233898162842,
      "loss": 0.9889,
      "nll_loss": 0.971484363079071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07501220703125,
      "rewards/margins": 0.06739501655101776,
      "rewards/rejected": -0.14240722358226776,
      "step": 2450
    },
    {
      "epoch": 0.1866747609652451,
      "grad_norm": 1.8462614488666282,
      "learning_rate": 1.6129556770910235e-06,
      "log_odds_chosen": 1.1233398914337158,
      "log_odds_ratio": -0.4259277284145355,
      "logits/chosen": -0.985546886920929,
      "logits/rejected": -0.8519531488418579,
      "logps/chosen": -0.646679699420929,
      "logps/rejected": -1.3787109851837158,
      "loss": 0.9742,
      "nll_loss": 0.9052734375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06461181491613388,
      "rewards/margins": 0.07322387397289276,
      "rewards/rejected": -0.13774414360523224,
      "step": 2460
    },
    {
      "epoch": 0.18743360145697374,
      "grad_norm": 2.077714916187083,
      "learning_rate": 1.6096872731710673e-06,
      "log_odds_chosen": 1.1902344226837158,
      "log_odds_ratio": -0.42304688692092896,
      "logits/chosen": -0.9332031011581421,
      "logits/rejected": -0.84375,
      "logps/chosen": -0.743457019329071,
      "logps/rejected": -1.571679711341858,
      "loss": 0.9835,
      "nll_loss": 0.9898437261581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07436523586511612,
      "rewards/margins": 0.0826416015625,
      "rewards/rejected": -0.15708008408546448,
      "step": 2470
    },
    {
      "epoch": 0.18819244194870238,
      "grad_norm": 1.8616129591738162,
      "learning_rate": 1.6064386578049978e-06,
      "log_odds_chosen": 1.16064453125,
      "log_odds_ratio": -0.47441405057907104,
      "logits/chosen": -0.904296875,
      "logits/rejected": -0.820507824420929,
      "logps/chosen": -0.693164050579071,
      "logps/rejected": -1.5470702648162842,
      "loss": 0.9871,
      "nll_loss": 0.9693359136581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06928710639476776,
      "rewards/margins": 0.08541259914636612,
      "rewards/rejected": -0.1546630859375,
      "step": 2480
    },
    {
      "epoch": 0.18895128244043102,
      "grad_norm": 2.1166882079269342,
      "learning_rate": 1.6032096321124046e-06,
      "log_odds_chosen": 0.8385375738143921,
      "log_odds_ratio": -0.5474609136581421,
      "logits/chosen": -0.9267578125,
      "logits/rejected": -0.8587890863418579,
      "logps/chosen": -0.755175769329071,
      "logps/rejected": -1.3542969226837158,
      "loss": 0.9952,
      "nll_loss": 0.971484363079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07552490383386612,
      "rewards/margins": 0.05993194505572319,
      "rewards/rejected": -0.13544921576976776,
      "step": 2490
    },
    {
      "epoch": 0.18971012293215966,
      "grad_norm": 2.110336575040382,
      "learning_rate": 1.6e-06,
      "log_odds_chosen": 1.1823241710662842,
      "log_odds_ratio": -0.46098631620407104,
      "logits/chosen": -0.9212890863418579,
      "logits/rejected": -0.8544921875,
      "logps/chosen": -0.652636706829071,
      "logps/rejected": -1.503320336341858,
      "loss": 0.9806,
      "nll_loss": 0.9033203125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06525878608226776,
      "rewards/margins": 0.0849609375,
      "rewards/rejected": -0.15021972358226776,
      "step": 2500
    },
    {
      "epoch": 0.1904689634238883,
      "grad_norm": 1.6348076610742528,
      "learning_rate": 1.5968095681115984e-06,
      "log_odds_chosen": 1.142578125,
      "log_odds_ratio": -0.486328125,
      "logits/chosen": -0.9554687738418579,
      "logits/rejected": -0.8511718511581421,
      "logps/chosen": -0.7681640386581421,
      "logps/rejected": -1.591796875,
      "loss": 0.9933,
      "nll_loss": 0.9615234136581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07675781100988388,
      "rewards/margins": 0.08242187649011612,
      "rewards/rejected": -0.15913085639476776,
      "step": 2510
    },
    {
      "epoch": 0.19122780391561695,
      "grad_norm": 2.136414388223389,
      "learning_rate": 1.5936381457791914e-06,
      "log_odds_chosen": 1.12158203125,
      "log_odds_ratio": -0.4488281309604645,
      "logits/chosen": -1.003515601158142,
      "logits/rejected": -0.9029296636581421,
      "logps/chosen": -0.730761706829071,
      "logps/rejected": -1.494726538658142,
      "loss": 0.9646,
      "nll_loss": 0.960156261920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07314453274011612,
      "rewards/margins": 0.076385498046875,
      "rewards/rejected": -0.14958496391773224,
      "step": 2520
    },
    {
      "epoch": 0.19198664440734559,
      "grad_norm": 1.709686144800437,
      "learning_rate": 1.590485544975088e-06,
      "log_odds_chosen": 1.1706054210662842,
      "log_odds_ratio": -0.43974608182907104,
      "logits/chosen": -0.9906250238418579,
      "logits/rejected": -0.8759765625,
      "logps/chosen": -0.724804699420929,
      "logps/rejected": -1.545507788658142,
      "loss": 0.9767,
      "nll_loss": 1.0037109851837158,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07244873046875,
      "rewards/margins": 0.08211059868335724,
      "rewards/rejected": -0.15458984673023224,
      "step": 2530
    },
    {
      "epoch": 0.19274548489907423,
      "grad_norm": 1.8255597130329309,
      "learning_rate": 1.5873515802650901e-06,
      "log_odds_chosen": 0.9172118902206421,
      "log_odds_ratio": -0.507275402545929,
      "logits/chosen": -0.990429699420929,
      "logits/rejected": -0.904101550579071,
      "logps/chosen": -0.6786133050918579,
      "logps/rejected": -1.294531226158142,
      "loss": 0.9685,
      "nll_loss": 0.9468749761581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06785888969898224,
      "rewards/margins": 0.06157226487994194,
      "rewards/rejected": -0.12946777045726776,
      "step": 2540
    },
    {
      "epoch": 0.19350432539080287,
      "grad_norm": 1.789326725731532,
      "learning_rate": 1.584236068762679e-06,
      "log_odds_chosen": 1.466210961341858,
      "log_odds_ratio": -0.3741455078125,
      "logits/chosen": -0.9769531488418579,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.6670898199081421,
      "logps/rejected": -1.676367163658142,
      "loss": 1.0203,
      "nll_loss": 0.9556640386581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06667480617761612,
      "rewards/margins": 0.10093994438648224,
      "rewards/rejected": -0.16752929985523224,
      "step": 2550
    },
    {
      "epoch": 0.19426316588253148,
      "grad_norm": 1.8900962347223054,
      "learning_rate": 1.5811388300841894e-06,
      "log_odds_chosen": 1.261132836341858,
      "log_odds_ratio": -0.4341796934604645,
      "logits/chosen": -0.9541015625,
      "logits/rejected": -0.8617187738418579,
      "logps/chosen": -0.699023425579071,
      "logps/rejected": -1.5593750476837158,
      "loss": 0.9719,
      "nll_loss": 1.002539038658142,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06990966945886612,
      "rewards/margins": 0.08603973686695099,
      "rewards/rejected": -0.15598145127296448,
      "step": 2560
    },
    {
      "epoch": 0.19502200637426012,
      "grad_norm": 1.6750647053286092,
      "learning_rate": 1.5780596863049431e-06,
      "log_odds_chosen": 1.4158203601837158,
      "log_odds_ratio": -0.38447266817092896,
      "logits/chosen": -0.9541015625,
      "logits/rejected": -0.8203125,
      "logps/chosen": -0.653613269329071,
      "logps/rejected": -1.607031226158142,
      "loss": 0.9651,
      "nll_loss": 0.8833984136581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06544189155101776,
      "rewards/margins": 0.09538574516773224,
      "rewards/rejected": -0.16069336235523224,
      "step": 2570
    },
    {
      "epoch": 0.19578084686598876,
      "grad_norm": 1.9326493464721346,
      "learning_rate": 1.5749984619163156e-06,
      "log_odds_chosen": 1.1027343273162842,
      "log_odds_ratio": -0.4658203125,
      "logits/chosen": -0.9818359613418579,
      "logits/rejected": -0.8500000238418579,
      "logps/chosen": -0.693066418170929,
      "logps/rejected": -1.4611327648162842,
      "loss": 0.9866,
      "nll_loss": 0.9554687738418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06929931789636612,
      "rewards/margins": 0.07676391303539276,
      "rewards/rejected": -0.14606933295726776,
      "step": 2580
    },
    {
      "epoch": 0.1965396873577174,
      "grad_norm": 1.67777232703704,
      "learning_rate": 1.5719549837837187e-06,
      "log_odds_chosen": 0.979296863079071,
      "log_odds_ratio": -0.48750001192092896,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.916210949420929,
      "logps/chosen": -0.693652331829071,
      "logps/rejected": -1.3572266101837158,
      "loss": 0.9592,
      "nll_loss": 0.8955078125,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06942138820886612,
      "rewards/margins": 0.06621094048023224,
      "rewards/rejected": -0.13564452528953552,
      "step": 2590
    },
    {
      "epoch": 0.19729852784944604,
      "grad_norm": 1.847138545524677,
      "learning_rate": 1.5689290811054722e-06,
      "log_odds_chosen": 1.246191382408142,
      "log_odds_ratio": -0.446044921875,
      "logits/chosen": -0.973437488079071,
      "logits/rejected": -0.8705078363418579,
      "logps/chosen": -0.675000011920929,
      "logps/rejected": -1.5144531726837158,
      "loss": 0.9621,
      "nll_loss": 0.9115234613418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06757812201976776,
      "rewards/margins": 0.08388672024011612,
      "rewards/rejected": -0.1513671875,
      "step": 2600
    },
    {
      "epoch": 0.19805736834117468,
      "grad_norm": 1.8218694874948793,
      "learning_rate": 1.5659205853725426e-06,
      "log_odds_chosen": 1.093359351158142,
      "log_odds_ratio": -0.47246092557907104,
      "logits/chosen": -0.990234375,
      "logits/rejected": -0.8970702886581421,
      "logps/chosen": -0.67236328125,
      "logps/rejected": -1.4619140625,
      "loss": 0.9777,
      "nll_loss": 0.9501953125,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06721191108226776,
      "rewards/margins": 0.07899780571460724,
      "rewards/rejected": -0.14621582627296448,
      "step": 2610
    },
    {
      "epoch": 0.19881620883290332,
      "grad_norm": 1.837412074872585,
      "learning_rate": 1.562929330329127e-06,
      "log_odds_chosen": 0.895214855670929,
      "log_odds_ratio": -0.4990234375,
      "logits/chosen": -0.9296875,
      "logits/rejected": -0.865234375,
      "logps/chosen": -0.706835925579071,
      "logps/rejected": -1.3162109851837158,
      "loss": 0.9985,
      "nll_loss": 1.008398413658142,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07066650688648224,
      "rewards/margins": 0.06090698391199112,
      "rewards/rejected": -0.13166503608226776,
      "step": 2620
    },
    {
      "epoch": 0.19957504932463196,
      "grad_norm": 1.7043281951830767,
      "learning_rate": 1.5599551519340636e-06,
      "log_odds_chosen": 1.339453101158142,
      "log_odds_ratio": -0.38862305879592896,
      "logits/chosen": -1.014257788658142,
      "logits/rejected": -0.9058593511581421,
      "logps/chosen": -0.6318359375,
      "logps/rejected": -1.546289086341858,
      "loss": 0.9407,
      "nll_loss": 0.896679699420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06322021782398224,
      "rewards/margins": 0.091400146484375,
      "rewards/rejected": -0.15456542372703552,
      "step": 2630
    },
    {
      "epoch": 0.2003338898163606,
      "grad_norm": 1.7931003101605318,
      "learning_rate": 1.556997888323046e-06,
      "log_odds_chosen": 1.1256835460662842,
      "log_odds_ratio": -0.44111329317092896,
      "logits/chosen": -0.958984375,
      "logits/rejected": -0.8802734613418579,
      "logps/chosen": -0.689257800579071,
      "logps/rejected": -1.437890648841858,
      "loss": 0.9724,
      "nll_loss": 0.929492175579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06889648735523224,
      "rewards/margins": 0.0749664306640625,
      "rewards/rejected": -0.14387206733226776,
      "step": 2640
    },
    {
      "epoch": 0.20109273030808925,
      "grad_norm": 1.571974698546549,
      "learning_rate": 1.5540573797716226e-06,
      "log_odds_chosen": 1.1706054210662842,
      "log_odds_ratio": -0.45585936307907104,
      "logits/chosen": -0.959765613079071,
      "logits/rejected": -0.8968750238418579,
      "logps/chosen": -0.676953136920929,
      "logps/rejected": -1.4599609375,
      "loss": 0.9565,
      "nll_loss": 0.900585949420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06772460788488388,
      "rewards/margins": 0.078369140625,
      "rewards/rejected": -0.14604492485523224,
      "step": 2650
    },
    {
      "epoch": 0.2018515707998179,
      "grad_norm": 1.8202591383722864,
      "learning_rate": 1.5511334686589623e-06,
      "log_odds_chosen": 1.417089819908142,
      "log_odds_ratio": -0.3907714784145355,
      "logits/chosen": -0.9935547113418579,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -0.65087890625,
      "logps/rejected": -1.615625023841858,
      "loss": 0.9577,
      "nll_loss": 0.876171886920929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06508789211511612,
      "rewards/margins": 0.09650878608226776,
      "rewards/rejected": -0.16157226264476776,
      "step": 2660
    },
    {
      "epoch": 0.20261041129154653,
      "grad_norm": 1.8834458505225085,
      "learning_rate": 1.548225999432367e-06,
      "log_odds_chosen": 1.3351562023162842,
      "log_odds_ratio": -0.41259765625,
      "logits/chosen": -0.982617199420929,
      "logits/rejected": -0.847851574420929,
      "logps/chosen": -0.672167956829071,
      "logps/rejected": -1.5802733898162842,
      "loss": 0.9765,
      "nll_loss": 0.915820300579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06721191108226776,
      "rewards/margins": 0.09076537936925888,
      "rewards/rejected": -0.15788574516773224,
      "step": 2670
    },
    {
      "epoch": 0.20336925178327517,
      "grad_norm": 2.0250133124028173,
      "learning_rate": 1.5453348185725114e-06,
      "log_odds_chosen": 0.9356933832168579,
      "log_odds_ratio": -0.510546863079071,
      "logits/chosen": -0.9546874761581421,
      "logits/rejected": -0.8519531488418579,
      "logps/chosen": -0.7032226324081421,
      "logps/rejected": -1.329687476158142,
      "loss": 1.0076,
      "nll_loss": 0.931445300579071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07032470405101776,
      "rewards/margins": 0.06263580173254013,
      "rewards/rejected": -0.13298340141773224,
      "step": 2680
    },
    {
      "epoch": 0.20412809227500378,
      "grad_norm": 2.386913601696167,
      "learning_rate": 1.542459774559398e-06,
      "log_odds_chosen": 1.2929198741912842,
      "log_odds_ratio": -0.4048828184604645,
      "logits/chosen": -0.96484375,
      "logits/rejected": -0.892578125,
      "logps/chosen": -0.708789050579071,
      "logps/rejected": -1.5808594226837158,
      "loss": 0.9777,
      "nll_loss": 0.9580078125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07087402045726776,
      "rewards/margins": 0.08717040717601776,
      "rewards/rejected": -0.15803222358226776,
      "step": 2690
    },
    {
      "epoch": 0.20488693276673242,
      "grad_norm": 1.8473571912546065,
      "learning_rate": 1.539600717839002e-06,
      "log_odds_chosen": 1.0650513172149658,
      "log_odds_ratio": -0.501953125,
      "logits/chosen": -0.962695300579071,
      "logits/rejected": -0.890429675579071,
      "logps/chosen": -0.742968738079071,
      "logps/rejected": -1.5226562023162842,
      "loss": 0.9632,
      "nll_loss": 0.9105468988418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07431640475988388,
      "rewards/margins": 0.07787322998046875,
      "rewards/rejected": -0.15224608778953552,
      "step": 2700
    },
    {
      "epoch": 0.20564577325846106,
      "grad_norm": 1.872727304303063,
      "learning_rate": 1.536757500790597e-06,
      "log_odds_chosen": 1.247460961341858,
      "log_odds_ratio": -0.4300781190395355,
      "logits/chosen": -0.9798828363418579,
      "logits/rejected": -0.8876953125,
      "logps/chosen": -0.7367187738418579,
      "logps/rejected": -1.5886719226837158,
      "loss": 0.9677,
      "nll_loss": 0.928906261920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07359619438648224,
      "rewards/margins": 0.08521118015050888,
      "rewards/rejected": -0.15895995497703552,
      "step": 2710
    },
    {
      "epoch": 0.2064046137501897,
      "grad_norm": 1.7627286806989997,
      "learning_rate": 1.5339299776947407e-06,
      "log_odds_chosen": 1.3813965320587158,
      "log_odds_ratio": -0.44189453125,
      "logits/chosen": -0.9408203363418579,
      "logits/rejected": -0.8451172113418579,
      "logps/chosen": -0.6982421875,
      "logps/rejected": -1.683203101158142,
      "loss": 0.967,
      "nll_loss": 0.920703113079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06981201469898224,
      "rewards/margins": 0.09839019924402237,
      "rewards/rejected": -0.16828612983226776,
      "step": 2720
    },
    {
      "epoch": 0.20716345424191834,
      "grad_norm": 2.225315838235998,
      "learning_rate": 1.5311180047019054e-06,
      "log_odds_chosen": 0.9217284917831421,
      "log_odds_ratio": -0.5398925542831421,
      "logits/chosen": -0.8974609375,
      "logits/rejected": -0.8558593988418579,
      "logps/chosen": -0.7349609136581421,
      "logps/rejected": -1.3703124523162842,
      "loss": 0.9745,
      "nll_loss": 0.9449218511581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07352294772863388,
      "rewards/margins": 0.06348266452550888,
      "rewards/rejected": -0.136962890625,
      "step": 2730
    },
    {
      "epoch": 0.20792229473364698,
      "grad_norm": 1.7123621724820337,
      "learning_rate": 1.5283214398017402e-06,
      "log_odds_chosen": 1.113378882408142,
      "log_odds_ratio": -0.4542480409145355,
      "logits/chosen": -1.009765625,
      "logits/rejected": -0.9136718511581421,
      "logps/chosen": -0.660937488079071,
      "logps/rejected": -1.4210937023162842,
      "loss": 0.9636,
      "nll_loss": 0.865429699420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0660400390625,
      "rewards/margins": 0.07600097358226776,
      "rewards/rejected": -0.14206543564796448,
      "step": 2740
    },
    {
      "epoch": 0.20868113522537562,
      "grad_norm": 1.8912075942460569,
      "learning_rate": 1.5255401427929477e-06,
      "log_odds_chosen": 1.3098633289337158,
      "log_odds_ratio": -0.4039062559604645,
      "logits/chosen": -0.9701172113418579,
      "logits/rejected": -0.884570300579071,
      "logps/chosen": -0.669628918170929,
      "logps/rejected": -1.5916016101837158,
      "loss": 0.9523,
      "nll_loss": 0.807910144329071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06696777045726776,
      "rewards/margins": 0.09216918796300888,
      "rewards/rejected": -0.15913085639476776,
      "step": 2750
    },
    {
      "epoch": 0.20943997571710427,
      "grad_norm": 1.9111616410727177,
      "learning_rate": 1.5227739752537617e-06,
      "log_odds_chosen": 1.12451171875,
      "log_odds_ratio": -0.4161621034145355,
      "logits/chosen": -0.992382824420929,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.6781250238418579,
      "logps/rejected": -1.424414038658142,
      "loss": 0.9826,
      "nll_loss": 0.9384765625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06783447414636612,
      "rewards/margins": 0.07457275688648224,
      "rewards/rejected": -0.1424560546875,
      "step": 2760
    },
    {
      "epoch": 0.2101988162088329,
      "grad_norm": 2.054694787863413,
      "learning_rate": 1.5200228005130127e-06,
      "log_odds_chosen": 1.0719726085662842,
      "log_odds_ratio": -0.48759764432907104,
      "logits/chosen": -0.9869140386581421,
      "logits/rejected": -0.9039062261581421,
      "logps/chosen": -0.7115234136581421,
      "logps/rejected": -1.458398461341858,
      "loss": 0.9641,
      "nll_loss": 0.932421863079071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07111816108226776,
      "rewards/margins": 0.07462158054113388,
      "rewards/rejected": -0.145751953125,
      "step": 2770
    },
    {
      "epoch": 0.21095765670056155,
      "grad_norm": 1.9682083058062396,
      "learning_rate": 1.5172864836217631e-06,
      "log_odds_chosen": 1.0216796398162842,
      "log_odds_ratio": -0.513427734375,
      "logits/chosen": -0.9623047113418579,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.7049804925918579,
      "logps/rejected": -1.436914086341858,
      "loss": 0.9801,
      "nll_loss": 0.90283203125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07052002102136612,
      "rewards/margins": 0.07314453274011612,
      "rewards/rejected": -0.14362792670726776,
      "step": 2780
    },
    {
      "epoch": 0.2117164971922902,
      "grad_norm": 1.7076823298104469,
      "learning_rate": 1.514564891325506e-06,
      "log_odds_chosen": 1.11669921875,
      "log_odds_ratio": -0.45429688692092896,
      "logits/chosen": -0.9652343988418579,
      "logits/rejected": -0.8560546636581421,
      "logps/chosen": -0.624707043170929,
      "logps/rejected": -1.333984375,
      "loss": 0.9698,
      "nll_loss": 0.918749988079071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06241454929113388,
      "rewards/margins": 0.07102050632238388,
      "rewards/rejected": -0.13339844346046448,
      "step": 2790
    },
    {
      "epoch": 0.21247533768401883,
      "grad_norm": 1.8454933653782706,
      "learning_rate": 1.5118578920369086e-06,
      "log_odds_chosen": 1.041650414466858,
      "log_odds_ratio": -0.47929686307907104,
      "logits/chosen": -0.971484363079071,
      "logits/rejected": -0.8687499761581421,
      "logps/chosen": -0.73779296875,
      "logps/rejected": -1.4578125476837158,
      "loss": 0.9761,
      "nll_loss": 0.9339843988418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07375488430261612,
      "rewards/margins": 0.07195434719324112,
      "rewards/rejected": -0.14582519233226776,
      "step": 2800
    },
    {
      "epoch": 0.21323417817574747,
      "grad_norm": 2.3111648063323242,
      "learning_rate": 1.5091653558090898e-06,
      "log_odds_chosen": 1.108007788658142,
      "log_odds_ratio": -0.448486328125,
      "logits/chosen": -0.935351550579071,
      "logits/rejected": -0.8687499761581421,
      "logps/chosen": -0.7041015625,
      "logps/rejected": -1.473046898841858,
      "loss": 0.9538,
      "nll_loss": 0.876757800579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07039795070886612,
      "rewards/margins": 0.076904296875,
      "rewards/rejected": -0.14731445908546448,
      "step": 2810
    },
    {
      "epoch": 0.2139930186674761,
      "grad_norm": 4.148395754218525,
      "learning_rate": 1.506487154309419e-06,
      "log_odds_chosen": 1.150976538658142,
      "log_odds_ratio": -0.45585936307907104,
      "logits/chosen": -0.928515613079071,
      "logits/rejected": -0.889453113079071,
      "logps/chosen": -0.7118164300918579,
      "logps/rejected": -1.466406226158142,
      "loss": 0.9455,
      "nll_loss": 0.878710925579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07124023139476776,
      "rewards/margins": 0.07537536323070526,
      "rewards/rejected": -0.14663085341453552,
      "step": 2820
    },
    {
      "epoch": 0.21475185915920472,
      "grad_norm": 1.9726707897843883,
      "learning_rate": 1.5038231607938247e-06,
      "log_odds_chosen": 1.0893065929412842,
      "log_odds_ratio": -0.48725587129592896,
      "logits/chosen": -0.9554687738418579,
      "logits/rejected": -0.862109363079071,
      "logps/chosen": -0.7093750238418579,
      "logps/rejected": -1.4777343273162842,
      "loss": 0.9787,
      "nll_loss": 0.935742199420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07087402045726776,
      "rewards/margins": 0.07680968940258026,
      "rewards/rejected": -0.1475830078125,
      "step": 2830
    },
    {
      "epoch": 0.21551069965093336,
      "grad_norm": 1.8387982254575808,
      "learning_rate": 1.501173250081603e-06,
      "log_odds_chosen": 1.326562523841858,
      "log_odds_ratio": -0.43168944120407104,
      "logits/chosen": -0.973828136920929,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.6865234375,
      "logps/rejected": -1.63671875,
      "loss": 0.9605,
      "nll_loss": 0.9320312738418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.068603515625,
      "rewards/margins": 0.09507445991039276,
      "rewards/rejected": -0.16372069716453552,
      "step": 2840
    },
    {
      "epoch": 0.216269540142662,
      "grad_norm": 2.2021903588634215,
      "learning_rate": 1.4985372985307103e-06,
      "log_odds_chosen": 1.35150146484375,
      "log_odds_ratio": -0.40996092557907104,
      "logits/chosen": -0.947265625,
      "logits/rejected": -0.8636718988418579,
      "logps/chosen": -0.686328113079071,
      "logps/rejected": -1.6365234851837158,
      "loss": 0.9485,
      "nll_loss": 0.9105468988418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06868896633386612,
      "rewards/margins": 0.09504852443933487,
      "rewards/rejected": -0.16367188096046448,
      "step": 2850
    },
    {
      "epoch": 0.21702838063439064,
      "grad_norm": 1.7415261017566501,
      "learning_rate": 1.4959151840135313e-06,
      "log_odds_chosen": 1.178320288658142,
      "log_odds_ratio": -0.4439453184604645,
      "logits/chosen": -0.990429699420929,
      "logits/rejected": -0.90234375,
      "logps/chosen": -0.7320312261581421,
      "logps/rejected": -1.5597655773162842,
      "loss": 0.9675,
      "nll_loss": 0.963085949420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07321777194738388,
      "rewards/margins": 0.08275756984949112,
      "rewards/rejected": -0.15598145127296448,
      "step": 2860
    },
    {
      "epoch": 0.21778722112611928,
      "grad_norm": 1.7962173674694326,
      "learning_rate": 1.4933067858931148e-06,
      "log_odds_chosen": 1.2593505382537842,
      "log_odds_ratio": -0.41767579317092896,
      "logits/chosen": -0.9828125238418579,
      "logits/rejected": -0.885546863079071,
      "logps/chosen": -0.66357421875,
      "logps/rejected": -1.4919922351837158,
      "loss": 0.9735,
      "nll_loss": 0.9029296636581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06641845405101776,
      "rewards/margins": 0.08284759521484375,
      "rewards/rejected": -0.14924316108226776,
      "step": 2870
    },
    {
      "epoch": 0.21854606161784793,
      "grad_norm": 1.8444965000372955,
      "learning_rate": 1.4907119849998597e-06,
      "log_odds_chosen": 1.254296898841858,
      "log_odds_ratio": -0.45166015625,
      "logits/chosen": -0.9599609375,
      "logits/rejected": -0.887890636920929,
      "logps/chosen": -0.6917968988418579,
      "logps/rejected": -1.537109375,
      "loss": 0.9678,
      "nll_loss": 0.9164062738418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.069091796875,
      "rewards/margins": 0.08457031100988388,
      "rewards/rejected": -0.15373535454273224,
      "step": 2880
    },
    {
      "epoch": 0.21930490210957657,
      "grad_norm": 1.7432037993290854,
      "learning_rate": 1.488130663608649e-06,
      "log_odds_chosen": 1.1090819835662842,
      "log_odds_ratio": -0.43535155057907104,
      "logits/chosen": -0.935546875,
      "logits/rejected": -0.8447265625,
      "logps/chosen": -0.64599609375,
      "logps/rejected": -1.378515601158142,
      "loss": 0.9707,
      "nll_loss": 0.8880859613418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0645751953125,
      "rewards/margins": 0.07330322265625,
      "rewards/rejected": -0.13786621391773224,
      "step": 2890
    },
    {
      "epoch": 0.2200637426013052,
      "grad_norm": 1.8270299370282952,
      "learning_rate": 1.4855627054164149e-06,
      "log_odds_chosen": 1.2333984375,
      "log_odds_ratio": -0.42988282442092896,
      "logits/chosen": -0.9619140625,
      "logits/rejected": -0.8291015625,
      "logps/chosen": -0.6923828125,
      "logps/rejected": -1.574609398841858,
      "loss": 0.9413,
      "nll_loss": 0.9408203363418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06927490234375,
      "rewards/margins": 0.08817748725414276,
      "rewards/rejected": -0.157470703125,
      "step": 2900
    },
    {
      "epoch": 0.22082258309303385,
      "grad_norm": 1.9292240039959558,
      "learning_rate": 1.4830079955201294e-06,
      "log_odds_chosen": 1.2151367664337158,
      "log_odds_ratio": -0.44414061307907104,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.8570312261581421,
      "logps/chosen": -0.715039074420929,
      "logps/rejected": -1.593164086341858,
      "loss": 0.9494,
      "nll_loss": 0.9208984375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07150878757238388,
      "rewards/margins": 0.08788452297449112,
      "rewards/rejected": -0.15939942002296448,
      "step": 2910
    },
    {
      "epoch": 0.2215814235847625,
      "grad_norm": 2.581943848053353,
      "learning_rate": 1.4804664203952103e-06,
      "log_odds_chosen": 1.3484375476837158,
      "log_odds_ratio": -0.41411131620407104,
      "logits/chosen": -0.9193359613418579,
      "logits/rejected": -0.8314453363418579,
      "logps/chosen": -0.697460949420929,
      "logps/rejected": -1.636328101158142,
      "loss": 0.9562,
      "nll_loss": 0.967968761920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06971435248851776,
      "rewards/margins": 0.09394530951976776,
      "rewards/rejected": -0.16376952826976776,
      "step": 2920
    },
    {
      "epoch": 0.22234026407649113,
      "grad_norm": 1.7988904180171552,
      "learning_rate": 1.4779378678743327e-06,
      "log_odds_chosen": 1.0883057117462158,
      "log_odds_ratio": -0.501660168170929,
      "logits/chosen": -0.9693359136581421,
      "logits/rejected": -0.90234375,
      "logps/chosen": -0.715136706829071,
      "logps/rejected": -1.501953125,
      "loss": 0.9604,
      "nll_loss": 0.933789074420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07152099907398224,
      "rewards/margins": 0.07857055962085724,
      "rewards/rejected": -0.15004882216453552,
      "step": 2930
    },
    {
      "epoch": 0.22309910456821977,
      "grad_norm": 1.9160010373129888,
      "learning_rate": 1.4754222271266348e-06,
      "log_odds_chosen": 1.13330078125,
      "log_odds_ratio": -0.47773438692092896,
      "logits/chosen": -0.9613281488418579,
      "logits/rejected": -0.8382812738418579,
      "logps/chosen": -0.697070300579071,
      "logps/rejected": -1.5056641101837158,
      "loss": 0.9634,
      "nll_loss": 0.8658202886581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06971435248851776,
      "rewards/margins": 0.08093567192554474,
      "rewards/rejected": -0.15053710341453552,
      "step": 2940
    },
    {
      "epoch": 0.2238579450599484,
      "grad_norm": 1.861090283579612,
      "learning_rate": 1.4729193886373175e-06,
      "log_odds_chosen": 1.147375464439392,
      "log_odds_ratio": -0.4544921815395355,
      "logits/chosen": -0.9873046875,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.658203125,
      "logps/rejected": -1.4572265148162842,
      "loss": 0.9554,
      "nll_loss": 0.822460949420929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06580810248851776,
      "rewards/margins": 0.07994689792394638,
      "rewards/rejected": -0.14572754502296448,
      "step": 2950
    },
    {
      "epoch": 0.22461678555167705,
      "grad_norm": 2.8696331367440693,
      "learning_rate": 1.4704292441876156e-06,
      "log_odds_chosen": 1.3240234851837158,
      "log_odds_ratio": -0.398193359375,
      "logits/chosen": -0.9740234613418579,
      "logits/rejected": -0.857226550579071,
      "logps/chosen": -0.65380859375,
      "logps/rejected": -1.5812499523162842,
      "loss": 0.949,
      "nll_loss": 0.836718738079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0654296875,
      "rewards/margins": 0.09274597465991974,
      "rewards/rejected": -0.15810546278953552,
      "step": 2960
    },
    {
      "epoch": 0.22537562604340566,
      "grad_norm": 1.6682423889188396,
      "learning_rate": 1.4679516868351474e-06,
      "log_odds_chosen": 1.2998046875,
      "log_odds_ratio": -0.4522460997104645,
      "logits/chosen": -0.9380859136581421,
      "logits/rejected": -0.828320324420929,
      "logps/chosen": -0.75,
      "logps/rejected": -1.726953148841858,
      "loss": 0.9733,
      "nll_loss": 0.953320324420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07496337592601776,
      "rewards/margins": 0.09773864597082138,
      "rewards/rejected": -0.1727294921875,
      "step": 2970
    },
    {
      "epoch": 0.2261344665351343,
      "grad_norm": 1.8546074916340258,
      "learning_rate": 1.4654866108946234e-06,
      "log_odds_chosen": 0.9668945074081421,
      "log_odds_ratio": -0.508056640625,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.862109363079071,
      "logps/chosen": -0.647753894329071,
      "logps/rejected": -1.292382836341858,
      "loss": 0.9674,
      "nll_loss": 0.953320324420929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06480713188648224,
      "rewards/margins": 0.06438598781824112,
      "rewards/rejected": -0.12919922173023224,
      "step": 2980
    },
    {
      "epoch": 0.22689330702686294,
      "grad_norm": 1.7596453202428828,
      "learning_rate": 1.4630339119189101e-06,
      "log_odds_chosen": 0.931640625,
      "log_odds_ratio": -0.5057617425918579,
      "logits/chosen": -0.935546875,
      "logits/rejected": -0.845898449420929,
      "logps/chosen": -0.7242187261581421,
      "logps/rejected": -1.37890625,
      "loss": 0.9486,
      "nll_loss": 0.899609386920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07239989936351776,
      "rewards/margins": 0.06554565578699112,
      "rewards/rejected": -0.13789062201976776,
      "step": 2990
    },
    {
      "epoch": 0.22765214751859159,
      "grad_norm": 1.847450689305971,
      "learning_rate": 1.4605934866804429e-06,
      "log_odds_chosen": 1.257592797279358,
      "log_odds_ratio": -0.400146484375,
      "logits/chosen": -0.987500011920929,
      "logits/rejected": -0.8763672113418579,
      "logps/chosen": -0.6636718511581421,
      "logps/rejected": -1.506250023841858,
      "loss": 0.9456,
      "nll_loss": 0.8394531011581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06638183444738388,
      "rewards/margins": 0.08422088623046875,
      "rewards/rejected": -0.15065917372703552,
      "step": 3000
    },
    {
      "epoch": 0.22841098801032023,
      "grad_norm": 1.7257030329384566,
      "learning_rate": 1.4581652331529784e-06,
      "log_odds_chosen": 1.3411133289337158,
      "log_odds_ratio": -0.4209960997104645,
      "logits/chosen": -0.9423828125,
      "logits/rejected": -0.8199218511581421,
      "logps/chosen": -0.6571289300918579,
      "logps/rejected": -1.5955078601837158,
      "loss": 0.9403,
      "nll_loss": 0.8746093511581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06572265923023224,
      "rewards/margins": 0.09383697807788849,
      "rewards/rejected": -0.15966796875,
      "step": 3010
    },
    {
      "epoch": 0.22916982850204887,
      "grad_norm": 1.6543337971685594,
      "learning_rate": 1.4557490504936778e-06,
      "log_odds_chosen": 0.958984375,
      "log_odds_ratio": -0.4776367247104645,
      "logits/chosen": -0.913867175579071,
      "logits/rejected": -0.82421875,
      "logps/chosen": -0.692187488079071,
      "logps/rejected": -1.3244140148162842,
      "loss": 0.9727,
      "nll_loss": 0.958203136920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06922607123851776,
      "rewards/margins": 0.06323852390050888,
      "rewards/rejected": -0.132568359375,
      "step": 3020
    },
    {
      "epoch": 0.2299286689937775,
      "grad_norm": 1.9978653383725091,
      "learning_rate": 1.453344839025519e-06,
      "log_odds_chosen": 1.125830054283142,
      "log_odds_ratio": -0.46538084745407104,
      "logits/chosen": -0.9478515386581421,
      "logits/rejected": -0.8636718988418579,
      "logps/chosen": -0.657910168170929,
      "logps/rejected": -1.415624976158142,
      "loss": 0.943,
      "nll_loss": 0.846484363079071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06577148288488388,
      "rewards/margins": 0.07586517184972763,
      "rewards/rejected": -0.1416015625,
      "step": 3030
    },
    {
      "epoch": 0.23068750948550615,
      "grad_norm": 1.9001759526352406,
      "learning_rate": 1.4509525002200234e-06,
      "log_odds_chosen": 1.4861938953399658,
      "log_odds_ratio": -0.40898436307907104,
      "logits/chosen": -0.9076172113418579,
      "logits/rejected": -0.833789050579071,
      "logps/chosen": -0.7176758050918579,
      "logps/rejected": -1.7810547351837158,
      "loss": 0.9498,
      "nll_loss": 0.9462890625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07175292819738388,
      "rewards/margins": 0.10638885200023651,
      "rewards/rejected": -0.17822265625,
      "step": 3040
    },
    {
      "epoch": 0.2314463499772348,
      "grad_norm": 1.812933320589776,
      "learning_rate": 1.4485719366802965e-06,
      "log_odds_chosen": 1.245507836341858,
      "log_odds_ratio": -0.4149414002895355,
      "logits/chosen": -0.9673827886581421,
      "logits/rejected": -0.8486328125,
      "logps/chosen": -0.669140636920929,
      "logps/rejected": -1.4578125476837158,
      "loss": 0.9417,
      "nll_loss": 0.8720703125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06687011569738388,
      "rewards/margins": 0.07879638671875,
      "rewards/rejected": -0.14580078423023224,
      "step": 3050
    },
    {
      "epoch": 0.23220519046896343,
      "grad_norm": 1.787462020163273,
      "learning_rate": 1.4462030521243742e-06,
      "log_odds_chosen": 1.0899536609649658,
      "log_odds_ratio": -0.4447265565395355,
      "logits/chosen": -0.9195312261581421,
      "logits/rejected": -0.7939453125,
      "logps/chosen": -0.6776367425918579,
      "logps/rejected": -1.380859375,
      "loss": 0.9589,
      "nll_loss": 0.9677734375,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06778564304113388,
      "rewards/margins": 0.07031555473804474,
      "rewards/rejected": -0.13803711533546448,
      "step": 3060
    },
    {
      "epoch": 0.23296403096069207,
      "grad_norm": 1.6858354000752622,
      "learning_rate": 1.443845751368867e-06,
      "log_odds_chosen": 1.0482909679412842,
      "log_odds_ratio": -0.48193359375,
      "logits/chosen": -0.9501953125,
      "logits/rejected": -0.8919922113418579,
      "logps/chosen": -0.6900390386581421,
      "logps/rejected": -1.3957030773162842,
      "loss": 0.9766,
      "nll_loss": 0.935546875,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06901855766773224,
      "rewards/margins": 0.07039795070886612,
      "rewards/rejected": -0.13947753608226776,
      "step": 3070
    },
    {
      "epoch": 0.2337228714524207,
      "grad_norm": 1.8339758263485157,
      "learning_rate": 1.4414999403128943e-06,
      "log_odds_chosen": 1.2374999523162842,
      "log_odds_ratio": -0.44316405057907104,
      "logits/chosen": -0.887890636920929,
      "logits/rejected": -0.8041015863418579,
      "logps/chosen": -0.6859375238418579,
      "logps/rejected": -1.5460937023162842,
      "loss": 0.9417,
      "nll_loss": 0.9515625238418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06850586086511612,
      "rewards/margins": 0.08607177436351776,
      "rewards/rejected": -0.15461425483226776,
      "step": 3080
    },
    {
      "epoch": 0.23448171194414935,
      "grad_norm": 2.068493502798679,
      "learning_rate": 1.439165525922309e-06,
      "log_odds_chosen": 1.157812476158142,
      "log_odds_ratio": -0.4925781190395355,
      "logits/chosen": -0.904492199420929,
      "logits/rejected": -0.8500000238418579,
      "logps/chosen": -0.6498047113418579,
      "logps/rejected": -1.4328124523162842,
      "loss": 0.9524,
      "nll_loss": 0.868359386920929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06501464545726776,
      "rewards/margins": 0.07831726223230362,
      "rewards/rejected": -0.1434326171875,
      "step": 3090
    },
    {
      "epoch": 0.235240552435878,
      "grad_norm": 1.8901274723008876,
      "learning_rate": 1.4368424162141992e-06,
      "log_odds_chosen": 1.166015625,
      "log_odds_ratio": -0.482177734375,
      "logits/chosen": -0.915820300579071,
      "logits/rejected": -0.81640625,
      "logps/chosen": -0.7708984613418579,
      "logps/rejected": -1.6150391101837158,
      "loss": 0.948,
      "nll_loss": 0.9623047113418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07711181789636612,
      "rewards/margins": 0.08429565280675888,
      "rewards/rejected": -0.1614990234375,
      "step": 3100
    },
    {
      "epoch": 0.2359993929276066,
      "grad_norm": 1.9627846293262432,
      "learning_rate": 1.434530520241665e-06,
      "log_odds_chosen": 1.162011742591858,
      "log_odds_ratio": -0.47773438692092896,
      "logits/chosen": -0.9302734136581421,
      "logits/rejected": -0.835742175579071,
      "logps/chosen": -0.658496081829071,
      "logps/rejected": -1.464453101158142,
      "loss": 0.9522,
      "nll_loss": 0.9443359375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06583251804113388,
      "rewards/margins": 0.08067016303539276,
      "rewards/rejected": -0.14645996689796448,
      "step": 3110
    },
    {
      "epoch": 0.23675823341933525,
      "grad_norm": 1.8470239258734957,
      "learning_rate": 1.4322297480788657e-06,
      "log_odds_chosen": 1.0634765625,
      "log_odds_ratio": -0.49458009004592896,
      "logits/chosen": -0.911914050579071,
      "logits/rejected": -0.8257812261581421,
      "logps/chosen": -0.753125011920929,
      "logps/rejected": -1.4982421398162842,
      "loss": 0.9471,
      "nll_loss": 0.931835949420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07525634765625,
      "rewards/margins": 0.074554443359375,
      "rewards/rejected": -0.14982910454273224,
      "step": 3120
    },
    {
      "epoch": 0.2375170739110639,
      "grad_norm": 1.9227094808395646,
      "learning_rate": 1.4299400108063247e-06,
      "log_odds_chosen": 1.065527319908142,
      "log_odds_ratio": -0.4964843690395355,
      "logits/chosen": -0.9326171875,
      "logits/rejected": -0.841796875,
      "logps/chosen": -0.6968749761581421,
      "logps/rejected": -1.4296875,
      "loss": 0.92,
      "nll_loss": 0.9214843511581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0697021484375,
      "rewards/margins": 0.07349243015050888,
      "rewards/rejected": -0.14306640625,
      "step": 3130
    },
    {
      "epoch": 0.23827591440279253,
      "grad_norm": 1.8725285434199292,
      "learning_rate": 1.4276612204964992e-06,
      "log_odds_chosen": 1.26513671875,
      "log_odds_ratio": -0.45307618379592896,
      "logits/chosen": -0.918164074420929,
      "logits/rejected": -0.8291015625,
      "logps/chosen": -0.675585925579071,
      "logps/rejected": -1.580078125,
      "loss": 0.9348,
      "nll_loss": 0.960742175579071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06754150241613388,
      "rewards/margins": 0.09055785834789276,
      "rewards/rejected": -0.15803222358226776,
      "step": 3140
    },
    {
      "epoch": 0.23903475489452117,
      "grad_norm": 1.7486311411327649,
      "learning_rate": 1.4253932901995967e-06,
      "log_odds_chosen": 1.062890648841858,
      "log_odds_ratio": -0.4705566465854645,
      "logits/chosen": -0.893359363079071,
      "logits/rejected": -0.814257800579071,
      "logps/chosen": -0.67822265625,
      "logps/rejected": -1.414648413658142,
      "loss": 0.9437,
      "nll_loss": 0.8882812261581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06781005859375,
      "rewards/margins": 0.07365112006664276,
      "rewards/rejected": -0.14145508408546448,
      "step": 3150
    },
    {
      "epoch": 0.2397935953862498,
      "grad_norm": 1.871562780745004,
      "learning_rate": 1.42313613392964e-06,
      "log_odds_chosen": 1.2360718250274658,
      "log_odds_ratio": -0.41899412870407104,
      "logits/chosen": -0.942187488079071,
      "logits/rejected": -0.836718738079071,
      "logps/chosen": -0.6473633050918579,
      "logps/rejected": -1.494140625,
      "loss": 0.927,
      "nll_loss": 0.8384765386581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06468506157398224,
      "rewards/margins": 0.08463592827320099,
      "rewards/rejected": -0.14943847060203552,
      "step": 3160
    },
    {
      "epoch": 0.24055243587797845,
      "grad_norm": 1.9319998881984413,
      "learning_rate": 1.4208896666507756e-06,
      "log_odds_chosen": 1.1755859851837158,
      "log_odds_ratio": -0.47407227754592896,
      "logits/chosen": -0.883984386920929,
      "logits/rejected": -0.807421863079071,
      "logps/chosen": -0.6717773675918579,
      "logps/rejected": -1.48046875,
      "loss": 0.9449,
      "nll_loss": 0.9410156011581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06711425632238388,
      "rewards/margins": 0.08091125637292862,
      "rewards/rejected": -0.1480712890625,
      "step": 3170
    },
    {
      "epoch": 0.2413112763697071,
      "grad_norm": 2.812202784971474,
      "learning_rate": 1.4186538042638173e-06,
      "log_odds_chosen": 1.2048828601837158,
      "log_odds_ratio": -0.439453125,
      "logits/chosen": -0.9654296636581421,
      "logits/rejected": -0.8451172113418579,
      "logps/chosen": -0.736328125,
      "logps/rejected": -1.6017577648162842,
      "loss": 0.9318,
      "nll_loss": 0.9462890625,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07366943359375,
      "rewards/margins": 0.08660583198070526,
      "rewards/rejected": -0.16021728515625,
      "step": 3180
    },
    {
      "epoch": 0.24207011686143573,
      "grad_norm": 1.8365782277502412,
      "learning_rate": 1.416428463593022e-06,
      "log_odds_chosen": 1.343164086341858,
      "log_odds_ratio": -0.4187988340854645,
      "logits/chosen": -0.939648449420929,
      "logits/rejected": -0.803906261920929,
      "logps/chosen": -0.6578124761581421,
      "logps/rejected": -1.591796875,
      "loss": 0.9503,
      "nll_loss": 0.8871093988418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06582031399011612,
      "rewards/margins": 0.09349365532398224,
      "rewards/rejected": -0.15922851860523224,
      "step": 3190
    },
    {
      "epoch": 0.24282895735316437,
      "grad_norm": 1.8323563524467135,
      "learning_rate": 1.414213562373095e-06,
      "log_odds_chosen": 1.247460961341858,
      "log_odds_ratio": -0.44145506620407104,
      "logits/chosen": -0.9507812261581421,
      "logits/rejected": -0.8531249761581421,
      "logps/chosen": -0.6361328363418579,
      "logps/rejected": -1.495507836341858,
      "loss": 0.9085,
      "nll_loss": 0.9072265625,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.063720703125,
      "rewards/margins": 0.0858154296875,
      "rewards/rejected": -0.14948730170726776,
      "step": 3200
    },
    {
      "epoch": 0.243587797844893,
      "grad_norm": 1.9099153823357262,
      "learning_rate": 1.4120090192364154e-06,
      "log_odds_chosen": 1.0369141101837158,
      "log_odds_ratio": -0.4970703125,
      "logits/chosen": -0.969531238079071,
      "logits/rejected": -0.8443359136581421,
      "logps/chosen": -0.691601574420929,
      "logps/rejected": -1.372656226158142,
      "loss": 0.9456,
      "nll_loss": 0.9193359613418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06915283203125,
      "rewards/margins": 0.06818237155675888,
      "rewards/rejected": -0.1373291015625,
      "step": 3210
    },
    {
      "epoch": 0.24434663833662165,
      "grad_norm": 2.0375256450793087,
      "learning_rate": 1.4098147537004828e-06,
      "log_odds_chosen": 1.133935570716858,
      "log_odds_ratio": -0.44086915254592896,
      "logits/chosen": -0.9365234375,
      "logits/rejected": -0.8529297113418579,
      "logps/chosen": -0.646679699420929,
      "logps/rejected": -1.3826172351837158,
      "loss": 0.9461,
      "nll_loss": 0.9072265625,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06463623046875,
      "rewards/margins": 0.07357482612133026,
      "rewards/rejected": -0.13820800185203552,
      "step": 3220
    },
    {
      "epoch": 0.2451054788283503,
      "grad_norm": 1.9912908730465613,
      "learning_rate": 1.4076306861555735e-06,
      "log_odds_chosen": 1.2411620616912842,
      "log_odds_ratio": -0.41997069120407104,
      "logits/chosen": -0.9458984136581421,
      "logits/rejected": -0.8486328125,
      "logps/chosen": -0.67578125,
      "logps/rejected": -1.5251953601837158,
      "loss": 0.9424,
      "nll_loss": 0.9007812738418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06752929836511612,
      "rewards/margins": 0.08501434326171875,
      "rewards/rejected": -0.15249022841453552,
      "step": 3230
    },
    {
      "epoch": 0.2458643193200789,
      "grad_norm": 2.274586805113359,
      "learning_rate": 1.405456737852613e-06,
      "log_odds_chosen": 1.1833984851837158,
      "log_odds_ratio": -0.4684081971645355,
      "logits/chosen": -0.9150390625,
      "logits/rejected": -0.835156261920929,
      "logps/chosen": -0.70703125,
      "logps/rejected": -1.558007836341858,
      "loss": 0.9503,
      "nll_loss": 0.878125011920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0706787109375,
      "rewards/margins": 0.08520660549402237,
      "rewards/rejected": -0.15581054985523224,
      "step": 3240
    },
    {
      "epoch": 0.24662315981180755,
      "grad_norm": 1.931992941128289,
      "learning_rate": 1.4032928308912468e-06,
      "log_odds_chosen": 1.2144043445587158,
      "log_odds_ratio": -0.4669433534145355,
      "logits/chosen": -0.9580078125,
      "logits/rejected": -0.8726562261581421,
      "logps/chosen": -0.6917968988418579,
      "logps/rejected": -1.5554687976837158,
      "loss": 0.9286,
      "nll_loss": 0.8759765625,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06925048679113388,
      "rewards/margins": 0.08652496337890625,
      "rewards/rejected": -0.15571288764476776,
      "step": 3250
    },
    {
      "epoch": 0.2473820003035362,
      "grad_norm": 1.889574173524338,
      "learning_rate": 1.4011388882081175e-06,
      "log_odds_chosen": 1.2414062023162842,
      "log_odds_ratio": -0.44091796875,
      "logits/chosen": -0.9189453125,
      "logits/rejected": -0.818164050579071,
      "logps/chosen": -0.604687511920929,
      "logps/rejected": -1.4462890625,
      "loss": 0.9378,
      "nll_loss": 0.8216797113418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06051025539636612,
      "rewards/margins": 0.08405761420726776,
      "rewards/rejected": -0.14462891221046448,
      "step": 3260
    },
    {
      "epoch": 0.24814084079526483,
      "grad_norm": 1.9404559201119644,
      "learning_rate": 1.3989948335653378e-06,
      "log_odds_chosen": 1.404687523841858,
      "log_odds_ratio": -0.427978515625,
      "logits/chosen": -1.0126953125,
      "logits/rejected": -0.861523449420929,
      "logps/chosen": -0.62841796875,
      "logps/rejected": -1.622460961341858,
      "loss": 0.9268,
      "nll_loss": 0.807910144329071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06280517578125,
      "rewards/margins": 0.09949950873851776,
      "rewards/rejected": -0.1622314453125,
      "step": 3270
    },
    {
      "epoch": 0.24889968128699347,
      "grad_norm": 1.7452378668385211,
      "learning_rate": 1.3968605915391564e-06,
      "log_odds_chosen": 1.380468726158142,
      "log_odds_ratio": -0.4466796815395355,
      "logits/chosen": -0.944531261920929,
      "logits/rejected": -0.848437488079071,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -1.7205078601837158,
      "loss": 0.9524,
      "nll_loss": 0.9287109375,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07064209133386612,
      "rewards/margins": 0.10134277492761612,
      "rewards/rejected": -0.17197266221046448,
      "step": 3280
    },
    {
      "epoch": 0.2496585217787221,
      "grad_norm": 1.9903896388016717,
      "learning_rate": 1.3947360875088132e-06,
      "log_odds_chosen": 1.1735351085662842,
      "log_odds_ratio": -0.46064454317092896,
      "logits/chosen": -0.9371093511581421,
      "logits/rejected": -0.8333984613418579,
      "logps/chosen": -0.70166015625,
      "logps/rejected": -1.510156273841858,
      "loss": 0.9344,
      "nll_loss": 0.8818359375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07014159858226776,
      "rewards/margins": 0.08088989555835724,
      "rewards/rejected": -0.15109863877296448,
      "step": 3290
    },
    {
      "epoch": 0.25041736227045075,
      "grad_norm": 1.825929681479405,
      "learning_rate": 1.3926212476455828e-06,
      "log_odds_chosen": 1.066686987876892,
      "log_odds_ratio": -0.496337890625,
      "logits/chosen": -0.9189453125,
      "logits/rejected": -0.8451172113418579,
      "logps/chosen": -0.681640625,
      "logps/rejected": -1.4054687023162842,
      "loss": 0.9379,
      "nll_loss": 0.9091796875,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06812743842601776,
      "rewards/margins": 0.07235412299633026,
      "rewards/rejected": -0.140625,
      "step": 3300
    },
    {
      "epoch": 0.2511762027621794,
      "grad_norm": 2.1511451858131787,
      "learning_rate": 1.3905159989019964e-06,
      "log_odds_chosen": 1.366601586341858,
      "log_odds_ratio": -0.42021483182907104,
      "logits/chosen": -0.9537109136581421,
      "logits/rejected": -0.8388671875,
      "logps/chosen": -0.6988281011581421,
      "logps/rejected": -1.6652343273162842,
      "loss": 0.9458,
      "nll_loss": 0.897265613079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06990966945886612,
      "rewards/margins": 0.09661865234375,
      "rewards/rejected": -0.16650390625,
      "step": 3310
    },
    {
      "epoch": 0.25193504325390803,
      "grad_norm": 2.4080145026266675,
      "learning_rate": 1.3884202690012465e-06,
      "log_odds_chosen": 1.415136694908142,
      "log_odds_ratio": -0.40556639432907104,
      "logits/chosen": -0.946484386920929,
      "logits/rejected": -0.8427734375,
      "logps/chosen": -0.737011730670929,
      "logps/rejected": -1.7296874523162842,
      "loss": 0.9546,
      "nll_loss": 0.9208984375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07371826469898224,
      "rewards/margins": 0.099334716796875,
      "rewards/rejected": -0.17304687201976776,
      "step": 3320
    },
    {
      "epoch": 0.25269388374563667,
      "grad_norm": 2.136961368070725,
      "learning_rate": 1.3863339864267636e-06,
      "log_odds_chosen": 0.906054675579071,
      "log_odds_ratio": -0.49775391817092896,
      "logits/chosen": -0.9507812261581421,
      "logits/rejected": -0.841601550579071,
      "logps/chosen": -0.6709960699081421,
      "logps/rejected": -1.2195312976837158,
      "loss": 0.9513,
      "nll_loss": 0.88671875,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06708984076976776,
      "rewards/margins": 0.05487060546875,
      "rewards/rejected": -0.12202148139476776,
      "step": 3330
    },
    {
      "epoch": 0.2534527242373653,
      "grad_norm": 1.9884853306075043,
      "learning_rate": 1.3842570804119655e-06,
      "log_odds_chosen": 1.0947265625,
      "log_odds_ratio": -0.44355469942092896,
      "logits/chosen": -0.9673827886581421,
      "logits/rejected": -0.832226574420929,
      "logps/chosen": -0.6509765386581421,
      "logps/rejected": -1.3839843273162842,
      "loss": 0.9472,
      "nll_loss": 0.8648437261581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06513671576976776,
      "rewards/margins": 0.07327880710363388,
      "rewards/rejected": -0.13840332627296448,
      "step": 3340
    },
    {
      "epoch": 0.25421156472909395,
      "grad_norm": 3.7716690402223643,
      "learning_rate": 1.3821894809301763e-06,
      "log_odds_chosen": 1.4836914539337158,
      "log_odds_ratio": -0.3805175721645355,
      "logits/chosen": -0.8980468511581421,
      "logits/rejected": -0.7972656488418579,
      "logps/chosen": -0.64111328125,
      "logps/rejected": -1.6281249523162842,
      "loss": 0.9458,
      "nll_loss": 0.8804687261581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06407471001148224,
      "rewards/margins": 0.09869384765625,
      "rewards/rejected": -0.16264648735523224,
      "step": 3350
    },
    {
      "epoch": 0.2549704052208226,
      "grad_norm": 1.9417352866678919,
      "learning_rate": 1.3801311186847081e-06,
      "log_odds_chosen": 1.279882788658142,
      "log_odds_ratio": -0.42607420682907104,
      "logits/chosen": -0.94921875,
      "logits/rejected": -0.835156261920929,
      "logps/chosen": -0.6883789300918579,
      "logps/rejected": -1.540624976158142,
      "loss": 0.9421,
      "nll_loss": 0.9169921875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06882324069738388,
      "rewards/margins": 0.08518066257238388,
      "rewards/rejected": -0.15407714247703552,
      "step": 3360
    },
    {
      "epoch": 0.25572924571255123,
      "grad_norm": 1.9677525270182894,
      "learning_rate": 1.378081925099109e-06,
      "log_odds_chosen": 1.1848633289337158,
      "log_odds_ratio": -0.41083985567092896,
      "logits/chosen": -0.86328125,
      "logits/rejected": -0.802734375,
      "logps/chosen": -0.6630859375,
      "logps/rejected": -1.443945288658142,
      "loss": 0.9459,
      "nll_loss": 0.881640613079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06630859524011612,
      "rewards/margins": 0.07810668647289276,
      "rewards/rejected": -0.1444091796875,
      "step": 3370
    },
    {
      "epoch": 0.2564880862042799,
      "grad_norm": 1.7058936097763349,
      "learning_rate": 1.376041832307563e-06,
      "log_odds_chosen": 1.240234375,
      "log_odds_ratio": -0.4556640684604645,
      "logits/chosen": -0.948437511920929,
      "logits/rejected": -0.8101562261581421,
      "logps/chosen": -0.694042980670929,
      "logps/rejected": -1.537109375,
      "loss": 0.9528,
      "nll_loss": 0.9375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06947021186351776,
      "rewards/margins": 0.08425293117761612,
      "rewards/rejected": -0.15361328423023224,
      "step": 3380
    },
    {
      "epoch": 0.2572469266960085,
      "grad_norm": 2.2082044702359465,
      "learning_rate": 1.3740107731454524e-06,
      "log_odds_chosen": 1.232031226158142,
      "log_odds_ratio": -0.4321533143520355,
      "logits/chosen": -0.9410156011581421,
      "logits/rejected": -0.8345702886581421,
      "logps/chosen": -0.6962890625,
      "logps/rejected": -1.5615234375,
      "loss": 0.9379,
      "nll_loss": 0.9957031011581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06964111328125,
      "rewards/margins": 0.08657226711511612,
      "rewards/rejected": -0.15615233778953552,
      "step": 3390
    },
    {
      "epoch": 0.25800576718773716,
      "grad_norm": 2.1820637460198142,
      "learning_rate": 1.3719886811400705e-06,
      "log_odds_chosen": 1.3191406726837158,
      "log_odds_ratio": -0.4033203125,
      "logits/chosen": -0.981640636920929,
      "logits/rejected": -0.876953125,
      "logps/chosen": -0.6343749761581421,
      "logps/rejected": -1.5291016101837158,
      "loss": 0.922,
      "nll_loss": 0.8228515386581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06345214694738388,
      "rewards/margins": 0.0894775390625,
      "rewards/rejected": -0.15297850966453552,
      "step": 3400
    },
    {
      "epoch": 0.2587646076794658,
      "grad_norm": 1.8723787448291065,
      "learning_rate": 1.3699754905014834e-06,
      "log_odds_chosen": 1.3383300304412842,
      "log_odds_ratio": -0.427001953125,
      "logits/chosen": -0.9251953363418579,
      "logits/rejected": -0.8294922113418579,
      "logps/chosen": -0.6695312261581421,
      "logps/rejected": -1.616796851158142,
      "loss": 0.9542,
      "nll_loss": 0.871874988079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06691894680261612,
      "rewards/margins": 0.09471359103918076,
      "rewards/rejected": -0.16169433295726776,
      "step": 3410
    },
    {
      "epoch": 0.25952344817119444,
      "grad_norm": 1.909954286522631,
      "learning_rate": 1.3679711361135388e-06,
      "log_odds_chosen": 1.097021460533142,
      "log_odds_ratio": -0.4974609315395355,
      "logits/chosen": -0.9791015386581421,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.7193359136581421,
      "logps/rejected": -1.520898461341858,
      "loss": 0.9201,
      "nll_loss": 0.8841797113418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07192382961511612,
      "rewards/margins": 0.08006133884191513,
      "rewards/rejected": -0.152099609375,
      "step": 3420
    },
    {
      "epoch": 0.2602822886629231,
      "grad_norm": 1.8424699803048412,
      "learning_rate": 1.3659755535250212e-06,
      "log_odds_chosen": 1.4440429210662842,
      "log_odds_ratio": -0.39545899629592896,
      "logits/chosen": -0.941601574420929,
      "logits/rejected": -0.8285156488418579,
      "logps/chosen": -0.63427734375,
      "logps/rejected": -1.6384766101837158,
      "loss": 0.9418,
      "nll_loss": 0.841015636920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06340332329273224,
      "rewards/margins": 0.10048218071460724,
      "rewards/rejected": -0.16384276747703552,
      "step": 3430
    },
    {
      "epoch": 0.2610411291546517,
      "grad_norm": 1.9198743792335182,
      "learning_rate": 1.3639886789409469e-06,
      "log_odds_chosen": 1.4453125,
      "log_odds_ratio": -0.38801270723342896,
      "logits/chosen": -0.9693359136581421,
      "logits/rejected": -0.830859363079071,
      "logps/chosen": -0.6615234613418579,
      "logps/rejected": -1.659765601158142,
      "loss": 0.9337,
      "nll_loss": 0.9193359613418579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06612548977136612,
      "rewards/margins": 0.09989013522863388,
      "rewards/rejected": -0.16608886420726776,
      "step": 3440
    },
    {
      "epoch": 0.2617999696463803,
      "grad_norm": 1.8882359433159133,
      "learning_rate": 1.3620104492139977e-06,
      "log_odds_chosen": 1.552832007408142,
      "log_odds_ratio": -0.38276368379592896,
      "logits/chosen": -0.944531261920929,
      "logits/rejected": -0.8271484375,
      "logps/chosen": -0.650195300579071,
      "logps/rejected": -1.726953148841858,
      "loss": 0.9616,
      "nll_loss": 0.879687488079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06500244140625,
      "rewards/margins": 0.10766907036304474,
      "rewards/rejected": -0.17258301377296448,
      "step": 3450
    },
    {
      "epoch": 0.26255881013810894,
      "grad_norm": 1.9301623087868962,
      "learning_rate": 1.3600408018360918e-06,
      "log_odds_chosen": 1.2575194835662842,
      "log_odds_ratio": -0.4151855409145355,
      "logits/chosen": -0.938671886920929,
      "logits/rejected": -0.852343738079071,
      "logps/chosen": -0.65234375,
      "logps/rejected": -1.4910156726837158,
      "loss": 0.9257,
      "nll_loss": 0.842968761920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06523437798023224,
      "rewards/margins": 0.08380126953125,
      "rewards/rejected": -0.14899902045726776,
      "step": 3460
    },
    {
      "epoch": 0.2633176506298376,
      "grad_norm": 1.9454646415269345,
      "learning_rate": 1.3580796749300878e-06,
      "log_odds_chosen": 1.2312500476837158,
      "log_odds_ratio": -0.4684081971645355,
      "logits/chosen": -0.9351562261581421,
      "logits/rejected": -0.8656250238418579,
      "logps/chosen": -0.7134765386581421,
      "logps/rejected": -1.5994141101837158,
      "loss": 0.9468,
      "nll_loss": 0.8939453363418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07143554836511612,
      "rewards/margins": 0.08856506645679474,
      "rewards/rejected": -0.15996094048023224,
      "step": 3470
    },
    {
      "epoch": 0.2640764911215662,
      "grad_norm": 1.8682090773735338,
      "learning_rate": 1.3561270072416209e-06,
      "log_odds_chosen": 1.100976586341858,
      "log_odds_ratio": -0.4833984375,
      "logits/chosen": -0.9535156488418579,
      "logits/rejected": -0.8607422113418579,
      "logps/chosen": -0.6826171875,
      "logps/rejected": -1.4677734375,
      "loss": 0.9192,
      "nll_loss": 0.895703136920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06829833984375,
      "rewards/margins": 0.07857055962085724,
      "rewards/rejected": -0.14677734673023224,
      "step": 3480
    },
    {
      "epoch": 0.26483533161329487,
      "grad_norm": 2.031528833234933,
      "learning_rate": 1.3541827381310652e-06,
      "log_odds_chosen": 1.175878882408142,
      "log_odds_ratio": -0.46772462129592896,
      "logits/chosen": -0.931445300579071,
      "logits/rejected": -0.8638671636581421,
      "logps/chosen": -0.707324206829071,
      "logps/rejected": -1.5478515625,
      "loss": 0.9382,
      "nll_loss": 0.87890625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07070312649011612,
      "rewards/margins": 0.08412780612707138,
      "rewards/rejected": -0.15488281846046448,
      "step": 3490
    },
    {
      "epoch": 0.2655941721050235,
      "grad_norm": 2.0922319319691103,
      "learning_rate": 1.3522468075656264e-06,
      "log_odds_chosen": 1.394921898841858,
      "log_odds_ratio": -0.44111329317092896,
      "logits/chosen": -0.9380859136581421,
      "logits/rejected": -0.8345702886581421,
      "logps/chosen": -0.717480480670929,
      "logps/rejected": -1.7511718273162842,
      "loss": 0.9151,
      "nll_loss": 0.8675781488418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07181396335363388,
      "rewards/margins": 0.10313721001148224,
      "rewards/rejected": -0.17502442002296448,
      "step": 3500
    },
    {
      "epoch": 0.26635301259675215,
      "grad_norm": 2.038393716254202,
      "learning_rate": 1.3503191561115553e-06,
      "log_odds_chosen": 1.2841796875,
      "log_odds_ratio": -0.4270996153354645,
      "logits/chosen": -0.954882800579071,
      "logits/rejected": -0.860546886920929,
      "logps/chosen": -0.656054675579071,
      "logps/rejected": -1.5242187976837158,
      "loss": 0.9097,
      "nll_loss": 0.872265636920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06560058891773224,
      "rewards/margins": 0.08687744289636612,
      "rewards/rejected": -0.15261229872703552,
      "step": 3510
    },
    {
      "epoch": 0.2671118530884808,
      "grad_norm": 2.113088406087,
      "learning_rate": 1.348399724926484e-06,
      "log_odds_chosen": 1.34375,
      "log_odds_ratio": -0.3807128965854645,
      "logits/chosen": -0.9605468511581421,
      "logits/rejected": -0.8394531011581421,
      "logps/chosen": -0.6573241949081421,
      "logps/rejected": -1.5763671398162842,
      "loss": 0.938,
      "nll_loss": 0.9312499761581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06573486328125,
      "rewards/margins": 0.09195556491613388,
      "rewards/rejected": -0.15773925185203552,
      "step": 3520
    },
    {
      "epoch": 0.26787069358020943,
      "grad_norm": 2.206269408386097,
      "learning_rate": 1.346488455751882e-06,
      "log_odds_chosen": 1.100976586341858,
      "log_odds_ratio": -0.45136719942092896,
      "logits/chosen": -0.981249988079071,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.708300769329071,
      "logps/rejected": -1.443359375,
      "loss": 0.9309,
      "nll_loss": 0.9097656011581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07087402045726776,
      "rewards/margins": 0.07344970852136612,
      "rewards/rejected": -0.14433594048023224,
      "step": 3530
    },
    {
      "epoch": 0.26862953407193807,
      "grad_norm": 1.9530518034425253,
      "learning_rate": 1.3445852909056286e-06,
      "log_odds_chosen": 1.2160155773162842,
      "log_odds_ratio": -0.40668946504592896,
      "logits/chosen": -0.971875011920929,
      "logits/rejected": -0.860546886920929,
      "logps/chosen": -0.6458984613418579,
      "logps/rejected": -1.461328148841858,
      "loss": 0.9214,
      "nll_loss": 0.887890636920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06459961086511612,
      "rewards/margins": 0.08159179985523224,
      "rewards/rejected": -0.1461181640625,
      "step": 3540
    },
    {
      "epoch": 0.2693883745636667,
      "grad_norm": 1.9384344977682093,
      "learning_rate": 1.3426901732747024e-06,
      "log_odds_chosen": 1.2693359851837158,
      "log_odds_ratio": -0.4195312559604645,
      "logits/chosen": -0.9878906011581421,
      "logits/rejected": -0.8871093988418579,
      "logps/chosen": -0.684863269329071,
      "logps/rejected": -1.568945288658142,
      "loss": 0.9331,
      "nll_loss": 0.9654296636581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06842041015625,
      "rewards/margins": 0.0883331298828125,
      "rewards/rejected": -0.15693359076976776,
      "step": 3550
    },
    {
      "epoch": 0.27014721505539535,
      "grad_norm": 2.1127083794668517,
      "learning_rate": 1.3408030463079818e-06,
      "log_odds_chosen": 1.28515625,
      "log_odds_ratio": -0.4278808534145355,
      "logits/chosen": -0.9398437738418579,
      "logits/rejected": -0.8525390625,
      "logps/chosen": -0.697265625,
      "logps/rejected": -1.605078101158142,
      "loss": 0.9248,
      "nll_loss": 0.8998047113418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06967773288488388,
      "rewards/margins": 0.09077148139476776,
      "rewards/rejected": -0.16044922173023224,
      "step": 3560
    },
    {
      "epoch": 0.270906055547124,
      "grad_norm": 2.0473745290079193,
      "learning_rate": 1.3389238540091568e-06,
      "log_odds_chosen": 1.4441406726837158,
      "log_odds_ratio": -0.4049316346645355,
      "logits/chosen": -1.032812476158142,
      "logits/rejected": -0.912109375,
      "logps/chosen": -0.709179699420929,
      "logps/rejected": -1.760156273841858,
      "loss": 0.9217,
      "nll_loss": 0.885546863079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07088623195886612,
      "rewards/margins": 0.10515137016773224,
      "rewards/rejected": -0.176025390625,
      "step": 3570
    },
    {
      "epoch": 0.27166489603885263,
      "grad_norm": 2.0116456378081438,
      "learning_rate": 1.337052540929751e-06,
      "log_odds_chosen": 1.521875023841858,
      "log_odds_ratio": -0.3868164122104645,
      "logits/chosen": -0.9634765386581421,
      "logits/rejected": -0.8515625,
      "logps/chosen": -0.721484363079071,
      "logps/rejected": -1.8093750476837158,
      "loss": 0.914,
      "nll_loss": 0.939257800579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0721435546875,
      "rewards/margins": 0.10887451469898224,
      "rewards/rejected": -0.18093261122703552,
      "step": 3580
    },
    {
      "epoch": 0.2724237365305813,
      "grad_norm": 2.0250387953551834,
      "learning_rate": 1.33518905216225e-06,
      "log_odds_chosen": 1.1780273914337158,
      "log_odds_ratio": -0.4666503965854645,
      "logits/chosen": -0.9242187738418579,
      "logits/rejected": -0.8359375,
      "logps/chosen": -0.71484375,
      "logps/rejected": -1.535546898841858,
      "loss": 0.9198,
      "nll_loss": 0.9498046636581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07145996391773224,
      "rewards/margins": 0.0820465087890625,
      "rewards/rejected": -0.15351562201976776,
      "step": 3590
    },
    {
      "epoch": 0.2731825770223099,
      "grad_norm": 1.8833860642550362,
      "learning_rate": 1.3333333333333332e-06,
      "log_odds_chosen": 1.3043944835662842,
      "log_odds_ratio": -0.4422851502895355,
      "logits/chosen": -0.8955078125,
      "logits/rejected": -0.804492175579071,
      "logps/chosen": -0.66796875,
      "logps/rejected": -1.592382788658142,
      "loss": 0.9095,
      "nll_loss": 0.8248046636581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06680908054113388,
      "rewards/margins": 0.09229431301355362,
      "rewards/rejected": -0.15915527939796448,
      "step": 3600
    },
    {
      "epoch": 0.27394141751403855,
      "grad_norm": 1.860998223645059,
      "learning_rate": 1.3314853305972122e-06,
      "log_odds_chosen": 1.147851586341858,
      "log_odds_ratio": -0.4798339903354645,
      "logits/chosen": -1.007226586341858,
      "logits/rejected": -0.916015625,
      "logps/chosen": -0.6913086175918579,
      "logps/rejected": -1.504296898841858,
      "loss": 0.9262,
      "nll_loss": 0.9146484136581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06912841647863388,
      "rewards/margins": 0.08133240044116974,
      "rewards/rejected": -0.15043945610523224,
      "step": 3610
    },
    {
      "epoch": 0.2747002580057672,
      "grad_norm": 3.411752689796777,
      "learning_rate": 1.3296449906290671e-06,
      "log_odds_chosen": 1.2785155773162842,
      "log_odds_ratio": -0.427490234375,
      "logits/chosen": -0.944531261920929,
      "logits/rejected": -0.8177734613418579,
      "logps/chosen": -0.6968749761581421,
      "logps/rejected": -1.5828125476837158,
      "loss": 0.9241,
      "nll_loss": 0.933789074420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06971435248851776,
      "rewards/margins": 0.08861084282398224,
      "rewards/rejected": -0.15817871689796448,
      "step": 3620
    },
    {
      "epoch": 0.27545909849749584,
      "grad_norm": 1.7878113783437786,
      "learning_rate": 1.3278122606185844e-06,
      "log_odds_chosen": 1.1787598133087158,
      "log_odds_ratio": -0.4697265625,
      "logits/chosen": -0.930859386920929,
      "logits/rejected": -0.851367175579071,
      "logps/chosen": -0.6568359136581421,
      "logps/rejected": -1.4609375,
      "loss": 0.9242,
      "nll_loss": 0.8294922113418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.065673828125,
      "rewards/margins": 0.080413818359375,
      "rewards/rejected": -0.1461181640625,
      "step": 3630
    },
    {
      "epoch": 0.2762179389892245,
      "grad_norm": 2.04196840241614,
      "learning_rate": 1.3259870882635918e-06,
      "log_odds_chosen": 1.085351586341858,
      "log_odds_ratio": -0.43281251192092896,
      "logits/chosen": -0.958203136920929,
      "logits/rejected": -0.855664074420929,
      "logps/chosen": -0.705859363079071,
      "logps/rejected": -1.427343726158142,
      "loss": 0.9117,
      "nll_loss": 0.894726574420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07058105617761612,
      "rewards/margins": 0.07220458984375,
      "rewards/rejected": -0.14274902641773224,
      "step": 3640
    },
    {
      "epoch": 0.2769767794809531,
      "grad_norm": 2.0541773707451956,
      "learning_rate": 1.3241694217637886e-06,
      "log_odds_chosen": 1.3690917491912842,
      "log_odds_ratio": -0.41411131620407104,
      "logits/chosen": -0.9505859613418579,
      "logits/rejected": -0.845898449420929,
      "logps/chosen": -0.6644531488418579,
      "logps/rejected": -1.6238281726837158,
      "loss": 0.8978,
      "nll_loss": 0.8138672113418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06641845405101776,
      "rewards/margins": 0.09598693996667862,
      "rewards/rejected": -0.16249999403953552,
      "step": 3650
    },
    {
      "epoch": 0.27773561997268176,
      "grad_norm": 1.85220670548097,
      "learning_rate": 1.3223592098145723e-06,
      "log_odds_chosen": 1.144677758216858,
      "log_odds_ratio": -0.46337890625,
      "logits/chosen": -0.990039050579071,
      "logits/rejected": -0.8544921875,
      "logps/chosen": -0.693359375,
      "logps/rejected": -1.516210913658142,
      "loss": 0.9182,
      "nll_loss": 0.889453113079071,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0693359375,
      "rewards/margins": 0.08226318657398224,
      "rewards/rejected": -0.15158692002296448,
      "step": 3660
    },
    {
      "epoch": 0.2784944604644104,
      "grad_norm": 2.0232170372823512,
      "learning_rate": 1.3205564016009555e-06,
      "log_odds_chosen": 1.1967284679412842,
      "log_odds_ratio": -0.44624024629592896,
      "logits/chosen": -0.9720703363418579,
      "logits/rejected": -0.866015613079071,
      "logps/chosen": -0.638671875,
      "logps/rejected": -1.4365234375,
      "loss": 0.9047,
      "nll_loss": 0.837695300579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0638427734375,
      "rewards/margins": 0.07980956882238388,
      "rewards/rejected": -0.14372558891773224,
      "step": 3670
    },
    {
      "epoch": 0.27925330095613904,
      "grad_norm": 2.0496707214558834,
      "learning_rate": 1.318760946791574e-06,
      "log_odds_chosen": 1.037841796875,
      "log_odds_ratio": -0.4786132872104645,
      "logits/chosen": -0.9595702886581421,
      "logits/rejected": -0.8451172113418579,
      "logps/chosen": -0.758007824420929,
      "logps/rejected": -1.464453101158142,
      "loss": 0.9316,
      "nll_loss": 0.9769531488418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07581786811351776,
      "rewards/margins": 0.07055969536304474,
      "rewards/rejected": -0.1463623046875,
      "step": 3680
    },
    {
      "epoch": 0.2800121414478677,
      "grad_norm": 1.9833909441957962,
      "learning_rate": 1.316972795532786e-06,
      "log_odds_chosen": 1.102392554283142,
      "log_odds_ratio": -0.44877928495407104,
      "logits/chosen": -0.970898449420929,
      "logits/rejected": -0.8890625238418579,
      "logps/chosen": -0.6924804449081421,
      "logps/rejected": -1.426367163658142,
      "loss": 0.9288,
      "nll_loss": 0.881640613079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06929931789636612,
      "rewards/margins": 0.07329712063074112,
      "rewards/rejected": -0.14252929389476776,
      "step": 3690
    },
    {
      "epoch": 0.2807709819395963,
      "grad_norm": 1.8332235239722916,
      "learning_rate": 1.3151918984428582e-06,
      "log_odds_chosen": 1.079687476158142,
      "log_odds_ratio": -0.474365234375,
      "logits/chosen": -0.947460949420929,
      "logits/rejected": -0.8470703363418579,
      "logps/chosen": -0.6900390386581421,
      "logps/rejected": -1.4333984851837158,
      "loss": 0.9166,
      "nll_loss": 0.875781238079071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06898193061351776,
      "rewards/margins": 0.07448425143957138,
      "rewards/rejected": -0.14338378608226776,
      "step": 3700
    },
    {
      "epoch": 0.28152982243132496,
      "grad_norm": 2.1768807174108207,
      "learning_rate": 1.313418206606237e-06,
      "log_odds_chosen": 1.1287109851837158,
      "log_odds_ratio": -0.4718261659145355,
      "logits/chosen": -0.9134765863418579,
      "logits/rejected": -0.8324218988418579,
      "logps/chosen": -0.703320324420929,
      "logps/rejected": -1.4970703125,
      "loss": 0.9236,
      "nll_loss": 0.8587890863418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07032470405101776,
      "rewards/margins": 0.07926025241613388,
      "rewards/rejected": -0.14970703423023224,
      "step": 3710
    },
    {
      "epoch": 0.2822886629230536,
      "grad_norm": 1.69205488461686,
      "learning_rate": 1.3116516715679057e-06,
      "log_odds_chosen": 1.261816382408142,
      "log_odds_ratio": -0.42744141817092896,
      "logits/chosen": -0.9498046636581421,
      "logits/rejected": -0.8570312261581421,
      "logps/chosen": -0.66796875,
      "logps/rejected": -1.5369141101837158,
      "loss": 0.8861,
      "nll_loss": 0.8794921636581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06682129204273224,
      "rewards/margins": 0.086822509765625,
      "rewards/rejected": -0.15363769233226776,
      "step": 3720
    },
    {
      "epoch": 0.2830475034147822,
      "grad_norm": 2.172458859919438,
      "learning_rate": 1.3098922453278258e-06,
      "log_odds_chosen": 1.377343773841858,
      "log_odds_ratio": -0.38911134004592896,
      "logits/chosen": -0.9644531011581421,
      "logits/rejected": -0.8539062738418579,
      "logps/chosen": -0.676074206829071,
      "logps/rejected": -1.6179687976837158,
      "loss": 0.954,
      "nll_loss": 0.8990234136581421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06757812201976776,
      "rewards/margins": 0.09407959133386612,
      "rewards/rejected": -0.16171875596046448,
      "step": 3730
    },
    {
      "epoch": 0.2838063439065108,
      "grad_norm": 1.993112848217689,
      "learning_rate": 1.3081398803354573e-06,
      "log_odds_chosen": 1.075585961341858,
      "log_odds_ratio": -0.4784179627895355,
      "logits/chosen": -0.939257800579071,
      "logits/rejected": -0.861328125,
      "logps/chosen": -0.7232421636581421,
      "logps/rejected": -1.469140648841858,
      "loss": 0.914,
      "nll_loss": 0.815234363079071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07233886420726776,
      "rewards/margins": 0.074615478515625,
      "rewards/rejected": -0.14699706435203552,
      "step": 3740
    },
    {
      "epoch": 0.28456518439823947,
      "grad_norm": 2.044150243486001,
      "learning_rate": 1.3063945294843617e-06,
      "log_odds_chosen": 1.310449242591858,
      "log_odds_ratio": -0.39531248807907104,
      "logits/chosen": -0.927929699420929,
      "logits/rejected": -0.8052734136581421,
      "logps/chosen": -0.637402355670929,
      "logps/rejected": -1.5,
      "loss": 0.9004,
      "nll_loss": 0.8472656011581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06375732272863388,
      "rewards/margins": 0.08640136569738388,
      "rewards/rejected": -0.15000000596046448,
      "step": 3750
    },
    {
      "epoch": 0.2853240248899681,
      "grad_norm": 2.1374911680271174,
      "learning_rate": 1.3046561461068843e-06,
      "log_odds_chosen": 1.138281226158142,
      "log_odds_ratio": -0.43183594942092896,
      "logits/chosen": -0.947070300579071,
      "logits/rejected": -0.845507800579071,
      "logps/chosen": -0.651171863079071,
      "logps/rejected": -1.3779296875,
      "loss": 0.907,
      "nll_loss": 0.8470703363418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06514892727136612,
      "rewards/margins": 0.07267455756664276,
      "rewards/rejected": -0.13779297471046448,
      "step": 3760
    },
    {
      "epoch": 0.28608286538169675,
      "grad_norm": 2.1722206297390354,
      "learning_rate": 1.3029246839689124e-06,
      "log_odds_chosen": 1.3113524913787842,
      "log_odds_ratio": -0.43989259004592896,
      "logits/chosen": -0.980664074420929,
      "logits/rejected": -0.8580077886581421,
      "logps/chosen": -0.712695300579071,
      "logps/rejected": -1.656835913658142,
      "loss": 0.8966,
      "nll_loss": 0.8304687738418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07132568210363388,
      "rewards/margins": 0.09425659477710724,
      "rewards/rejected": -0.16567382216453552,
      "step": 3770
    },
    {
      "epoch": 0.2868417058734254,
      "grad_norm": 2.1742138914896745,
      "learning_rate": 1.3012000972647109e-06,
      "log_odds_chosen": 1.387109398841858,
      "log_odds_ratio": -0.42460936307907104,
      "logits/chosen": -0.951367199420929,
      "logits/rejected": -0.8472656011581421,
      "logps/chosen": -0.6595703363418579,
      "logps/rejected": -1.5859375,
      "loss": 0.8949,
      "nll_loss": 0.8529297113418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06594238430261612,
      "rewards/margins": 0.09256591647863388,
      "rewards/rejected": -0.15861816704273224,
      "step": 3780
    },
    {
      "epoch": 0.28760054636515403,
      "grad_norm": 2.155127876083863,
      "learning_rate": 1.299482340611832e-06,
      "log_odds_chosen": 1.07763671875,
      "log_odds_ratio": -0.45654296875,
      "logits/chosen": -0.9107421636581421,
      "logits/rejected": -0.841796875,
      "logps/chosen": -0.62109375,
      "logps/rejected": -1.2839844226837158,
      "loss": 0.9269,
      "nll_loss": 0.898632824420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06208496168255806,
      "rewards/margins": 0.0662841796875,
      "rewards/rejected": -0.12841796875,
      "step": 3790
    },
    {
      "epoch": 0.28835938685688267,
      "grad_norm": 1.9850754100896,
      "learning_rate": 1.2977713690461003e-06,
      "log_odds_chosen": 0.9205077886581421,
      "log_odds_ratio": -0.5121093988418579,
      "logits/chosen": -0.9189453125,
      "logits/rejected": -0.8374999761581421,
      "logps/chosen": -0.702343761920929,
      "logps/rejected": -1.354101538658142,
      "loss": 0.9102,
      "nll_loss": 0.8666015863418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07027588039636612,
      "rewards/margins": 0.06508179008960724,
      "rewards/rejected": -0.13547363877296448,
      "step": 3800
    },
    {
      "epoch": 0.2891182273486113,
      "grad_norm": 3.6873578144144328,
      "learning_rate": 1.296067138016669e-06,
      "log_odds_chosen": 1.4038574695587158,
      "log_odds_ratio": -0.42290037870407104,
      "logits/chosen": -0.9017578363418579,
      "logits/rejected": -0.8236328363418579,
      "logps/chosen": -0.65576171875,
      "logps/rejected": -1.642968773841858,
      "loss": 0.9158,
      "nll_loss": 0.9505859613418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06558837741613388,
      "rewards/margins": 0.0986480712890625,
      "rewards/rejected": -0.16423340141773224,
      "step": 3810
    },
    {
      "epoch": 0.28987706784033995,
      "grad_norm": 1.7797642030913543,
      "learning_rate": 1.294369603381147e-06,
      "log_odds_chosen": 1.154687523841858,
      "log_odds_ratio": -0.4600585997104645,
      "logits/chosen": -0.940234363079071,
      "logits/rejected": -0.8539062738418579,
      "logps/chosen": -0.6429687738418579,
      "logps/rejected": -1.458398461341858,
      "loss": 0.9202,
      "nll_loss": 0.8675781488418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06423339992761612,
      "rewards/margins": 0.08149413764476776,
      "rewards/rejected": -0.14577636122703552,
      "step": 3820
    },
    {
      "epoch": 0.2906359083320686,
      "grad_norm": 2.069119773208419,
      "learning_rate": 1.2926787214007981e-06,
      "log_odds_chosen": 1.3681640625,
      "log_odds_ratio": -0.40424805879592896,
      "logits/chosen": -0.929492175579071,
      "logits/rejected": -0.828906238079071,
      "logps/chosen": -0.6714843511581421,
      "logps/rejected": -1.6003906726837158,
      "loss": 0.9061,
      "nll_loss": 0.9058593511581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06715087592601776,
      "rewards/margins": 0.092864990234375,
      "rewards/rejected": -0.16010741889476776,
      "step": 3830
    },
    {
      "epoch": 0.29139474882379723,
      "grad_norm": 2.1430312635959963,
      "learning_rate": 1.2909944487358056e-06,
      "log_odds_chosen": 1.35888671875,
      "log_odds_ratio": -0.42060548067092896,
      "logits/chosen": -0.9111328125,
      "logits/rejected": -0.826367199420929,
      "logps/chosen": -0.666699230670929,
      "logps/rejected": -1.606835961341858,
      "loss": 0.9176,
      "nll_loss": 0.8921874761581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06668701022863388,
      "rewards/margins": 0.09390868991613388,
      "rewards/rejected": -0.16062012314796448,
      "step": 3840
    },
    {
      "epoch": 0.2921535893155259,
      "grad_norm": 1.8334780666623056,
      "learning_rate": 1.2893167424406084e-06,
      "log_odds_chosen": 1.49951171875,
      "log_odds_ratio": -0.3844238221645355,
      "logits/chosen": -0.9283202886581421,
      "logits/rejected": -0.83203125,
      "logps/chosen": -0.6283203363418579,
      "logps/rejected": -1.661523461341858,
      "loss": 0.9123,
      "nll_loss": 0.8617187738418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06282959133386612,
      "rewards/margins": 0.10329589992761612,
      "rewards/rejected": -0.1661376953125,
      "step": 3850
    },
    {
      "epoch": 0.2929124298072545,
      "grad_norm": 3.226465128998707,
      "learning_rate": 1.2876455599593008e-06,
      "log_odds_chosen": 1.295019507408142,
      "log_odds_ratio": -0.4169921875,
      "logits/chosen": -0.9349609613418579,
      "logits/rejected": -0.8218749761581421,
      "logps/chosen": -0.687695324420929,
      "logps/rejected": -1.5730469226837158,
      "loss": 0.9304,
      "nll_loss": 0.884960949420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06887207180261612,
      "rewards/margins": 0.08841247856616974,
      "rewards/rejected": -0.15725097060203552,
      "step": 3860
    },
    {
      "epoch": 0.29367127029898316,
      "grad_norm": 1.8748341791580312,
      "learning_rate": 1.285980859121099e-06,
      "log_odds_chosen": 1.3193359375,
      "log_odds_ratio": -0.3985839784145355,
      "logits/chosen": -0.947460949420929,
      "logits/rejected": -0.849414050579071,
      "logps/chosen": -0.6934570074081421,
      "logps/rejected": -1.58984375,
      "loss": 0.8835,
      "nll_loss": 0.816210925579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06939697265625,
      "rewards/margins": 0.089569091796875,
      "rewards/rejected": -0.15903320908546448,
      "step": 3870
    },
    {
      "epoch": 0.2944301107907118,
      "grad_norm": 2.0849897268039674,
      "learning_rate": 1.2843225981358712e-06,
      "log_odds_chosen": 1.206884741783142,
      "log_odds_ratio": -0.4189453125,
      "logits/chosen": -0.894726574420929,
      "logits/rejected": -0.818359375,
      "logps/chosen": -0.6875,
      "logps/rejected": -1.5177733898162842,
      "loss": 0.8965,
      "nll_loss": 0.895703136920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06877441704273224,
      "rewards/margins": 0.08306427299976349,
      "rewards/rejected": -0.15168456733226776,
      "step": 3880
    },
    {
      "epoch": 0.29518895128244044,
      "grad_norm": 2.096660042103269,
      "learning_rate": 1.2826707355897317e-06,
      "log_odds_chosen": 1.1281249523162842,
      "log_odds_ratio": -0.4625488221645355,
      "logits/chosen": -0.9380859136581421,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.712890625,
      "logps/rejected": -1.5154297351837158,
      "loss": 0.9078,
      "nll_loss": 0.9111328125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0712890625,
      "rewards/margins": 0.08025512844324112,
      "rewards/rejected": -0.15156249701976776,
      "step": 3890
    },
    {
      "epoch": 0.2959477917741691,
      "grad_norm": 2.037013139295946,
      "learning_rate": 1.281025230440697e-06,
      "log_odds_chosen": 1.2263672351837158,
      "log_odds_ratio": -0.4684081971645355,
      "logits/chosen": -0.949023425579071,
      "logits/rejected": -0.8646484613418579,
      "logps/chosen": -0.738964855670929,
      "logps/rejected": -1.6062500476837158,
      "loss": 0.9104,
      "nll_loss": 0.9224609136581421,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07393798977136612,
      "rewards/margins": 0.086822509765625,
      "rewards/rejected": -0.16074219346046448,
      "step": 3900
    },
    {
      "epoch": 0.2967066322658977,
      "grad_norm": 2.027263988966167,
      "learning_rate": 1.2793860420144025e-06,
      "log_odds_chosen": 1.141992211341858,
      "log_odds_ratio": -0.46015626192092896,
      "logits/chosen": -0.947070300579071,
      "logits/rejected": -0.85546875,
      "logps/chosen": -0.693359375,
      "logps/rejected": -1.4386718273162842,
      "loss": 0.9024,
      "nll_loss": 0.832226574420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06937255710363388,
      "rewards/margins": 0.07454834133386612,
      "rewards/rejected": -0.14394530653953552,
      "step": 3910
    },
    {
      "epoch": 0.29746547275762636,
      "grad_norm": 2.401330558412098,
      "learning_rate": 1.2777531299998798e-06,
      "log_odds_chosen": 1.37109375,
      "log_odds_ratio": -0.405517578125,
      "logits/chosen": -1.0207030773162842,
      "logits/rejected": -0.8681640625,
      "logps/chosen": -0.656542956829071,
      "logps/rejected": -1.558203101158142,
      "loss": 0.9375,
      "nll_loss": 0.9273437261581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.065673828125,
      "rewards/margins": 0.09022827446460724,
      "rewards/rejected": -0.15598145127296448,
      "step": 3920
    },
    {
      "epoch": 0.298224313249355,
      "grad_norm": 2.0230385894292295,
      "learning_rate": 1.2761264544453928e-06,
      "log_odds_chosen": 1.116113305091858,
      "log_odds_ratio": -0.4505371153354645,
      "logits/chosen": -0.9798828363418579,
      "logits/rejected": -0.9058593511581421,
      "logps/chosen": -0.734570324420929,
      "logps/rejected": -1.5070312023162842,
      "loss": 0.9004,
      "nll_loss": 0.866992175579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07340087741613388,
      "rewards/margins": 0.07725830376148224,
      "rewards/rejected": -0.15078124403953552,
      "step": 3930
    },
    {
      "epoch": 0.29898315374108364,
      "grad_norm": 2.4045696372396175,
      "learning_rate": 1.2745059757543324e-06,
      "log_odds_chosen": 1.170654296875,
      "log_odds_ratio": -0.45654296875,
      "logits/chosen": -0.9185546636581421,
      "logits/rejected": -0.806640625,
      "logps/chosen": -0.7037109136581421,
      "logps/rejected": -1.4855468273162842,
      "loss": 0.8907,
      "nll_loss": 0.8763672113418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07038573920726776,
      "rewards/margins": 0.07807006686925888,
      "rewards/rejected": -0.1485595703125,
      "step": 3940
    },
    {
      "epoch": 0.2997419942328123,
      "grad_norm": 2.38418116530518,
      "learning_rate": 1.272891654681168e-06,
      "log_odds_chosen": 1.058984398841858,
      "log_odds_ratio": -0.475830078125,
      "logits/chosen": -0.8978515863418579,
      "logits/rejected": -0.828320324420929,
      "logps/chosen": -0.6246093511581421,
      "logps/rejected": -1.282617211341858,
      "loss": 0.9058,
      "nll_loss": 0.878125011920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06243896484375,
      "rewards/margins": 0.06582947075366974,
      "rewards/rejected": -0.12822265923023224,
      "step": 3950
    },
    {
      "epoch": 0.3005008347245409,
      "grad_norm": 2.2719917645534053,
      "learning_rate": 1.2712834523274563e-06,
      "log_odds_chosen": 1.1682617664337158,
      "log_odds_ratio": -0.43828123807907104,
      "logits/chosen": -0.954296886920929,
      "logits/rejected": -0.8714843988418579,
      "logps/chosen": -0.62060546875,
      "logps/rejected": -1.387109398841858,
      "loss": 0.9131,
      "nll_loss": 0.8638671636581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06207275390625,
      "rewards/margins": 0.07665405422449112,
      "rewards/rejected": -0.138671875,
      "step": 3960
    },
    {
      "epoch": 0.30125967521626956,
      "grad_norm": 2.6488593849334277,
      "learning_rate": 1.2696813301379032e-06,
      "log_odds_chosen": 1.3332030773162842,
      "log_odds_ratio": -0.4345703125,
      "logits/chosen": -0.9886718988418579,
      "logits/rejected": -0.884960949420929,
      "logps/chosen": -0.663867175579071,
      "logps/rejected": -1.55859375,
      "loss": 0.9141,
      "nll_loss": 0.8414062261581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06636963039636612,
      "rewards/margins": 0.08956298977136612,
      "rewards/rejected": -0.15590819716453552,
      "step": 3970
    },
    {
      "epoch": 0.3020185157079982,
      "grad_norm": 2.10235152985044,
      "learning_rate": 1.2680852498964829e-06,
      "log_odds_chosen": 1.461523413658142,
      "log_odds_ratio": -0.3689941465854645,
      "logits/chosen": -0.960156261920929,
      "logits/rejected": -0.847460925579071,
      "logps/chosen": -0.628710925579071,
      "logps/rejected": -1.6316406726837158,
      "loss": 0.9188,
      "nll_loss": 0.794921875,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06287841498851776,
      "rewards/margins": 0.10035400092601776,
      "rewards/rejected": -0.16318359971046448,
      "step": 3980
    },
    {
      "epoch": 0.30277735619972684,
      "grad_norm": 2.270931300000564,
      "learning_rate": 1.266495173722607e-06,
      "log_odds_chosen": 1.0371582508087158,
      "log_odds_ratio": -0.4625000059604645,
      "logits/chosen": -0.907031238079071,
      "logits/rejected": -0.8451172113418579,
      "logps/chosen": -0.685351550579071,
      "logps/rejected": -1.3771483898162842,
      "loss": 0.9084,
      "nll_loss": 0.817187488079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06854248046875,
      "rewards/margins": 0.06908264011144638,
      "rewards/rejected": -0.13762207329273224,
      "step": 3990
    },
    {
      "epoch": 0.30353619669145543,
      "grad_norm": 2.1861920101261725,
      "learning_rate": 1.2649110640673517e-06,
      "log_odds_chosen": 1.5475585460662842,
      "log_odds_ratio": -0.3888183534145355,
      "logits/chosen": -0.92578125,
      "logits/rejected": -0.838085949420929,
      "logps/chosen": -0.691113293170929,
      "logps/rejected": -1.7919921875,
      "loss": 0.9114,
      "nll_loss": 0.8662109375,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06910400092601776,
      "rewards/margins": 0.11008910834789276,
      "rewards/rejected": -0.17917481064796448,
      "step": 4000
    },
    {
      "epoch": 0.30429503718318407,
      "grad_norm": 1.8779121675796782,
      "learning_rate": 1.2633328837097308e-06,
      "log_odds_chosen": 1.2890625,
      "log_odds_ratio": -0.422119140625,
      "logits/chosen": -0.9326171875,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.65625,
      "logps/rejected": -1.5244140625,
      "loss": 0.903,
      "nll_loss": 0.890820324420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06556396186351776,
      "rewards/margins": 0.08682861179113388,
      "rewards/rejected": -0.1524658203125,
      "step": 4010
    },
    {
      "epoch": 0.3050538776749127,
      "grad_norm": 2.01813428939538,
      "learning_rate": 1.2617605957530233e-06,
      "log_odds_chosen": 1.365332007408142,
      "log_odds_ratio": -0.3958496153354645,
      "logits/chosen": -0.9375,
      "logits/rejected": -0.833984375,
      "logps/chosen": -0.6661132574081421,
      "logps/rejected": -1.6046874523162842,
      "loss": 0.9001,
      "nll_loss": 0.8521484136581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06668701022863388,
      "rewards/margins": 0.09389648586511612,
      "rewards/rejected": -0.16057129204273224,
      "step": 4020
    },
    {
      "epoch": 0.30581271816664135,
      "grad_norm": 2.4756880333260267,
      "learning_rate": 1.2601941636211516e-06,
      "log_odds_chosen": 1.533593773841858,
      "log_odds_ratio": -0.35151368379592896,
      "logits/chosen": -0.97265625,
      "logits/rejected": -0.8453124761581421,
      "logps/chosen": -0.6441406011581421,
      "logps/rejected": -1.714453101158142,
      "loss": 0.8684,
      "nll_loss": 0.843945324420929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06441650539636612,
      "rewards/margins": 0.10716553032398224,
      "rewards/rejected": -0.17145995795726776,
      "step": 4030
    },
    {
      "epoch": 0.30657155865837,
      "grad_norm": 2.1960269276331688,
      "learning_rate": 1.2586335510551052e-06,
      "log_odds_chosen": 1.3595702648162842,
      "log_odds_ratio": -0.40283203125,
      "logits/chosen": -0.9234374761581421,
      "logits/rejected": -0.826367199420929,
      "logps/chosen": -0.689746081829071,
      "logps/rejected": -1.6552734375,
      "loss": 0.9035,
      "nll_loss": 0.8949218988418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06898193061351776,
      "rewards/margins": 0.09656982123851776,
      "rewards/rejected": -0.16552734375,
      "step": 4040
    },
    {
      "epoch": 0.30733039915009863,
      "grad_norm": 4.778497260855035,
      "learning_rate": 1.2570787221094177e-06,
      "log_odds_chosen": 1.20697021484375,
      "log_odds_ratio": -0.40961915254592896,
      "logits/chosen": -0.929882824420929,
      "logits/rejected": -0.8060547113418579,
      "logps/chosen": -0.642871081829071,
      "logps/rejected": -1.4294922351837158,
      "loss": 0.9063,
      "nll_loss": 0.8509765863418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06430663913488388,
      "rewards/margins": 0.07865753024816513,
      "rewards/rejected": -0.14291992783546448,
      "step": 4050
    },
    {
      "epoch": 0.3080892396418273,
      "grad_norm": 2.007858230707382,
      "learning_rate": 1.255529641148689e-06,
      "log_odds_chosen": 1.3185546398162842,
      "log_odds_ratio": -0.406005859375,
      "logits/chosen": -0.9306640625,
      "logits/rejected": -0.8427734375,
      "logps/chosen": -0.6982421875,
      "logps/rejected": -1.6140625476837158,
      "loss": 0.8963,
      "nll_loss": 0.87109375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06978759914636612,
      "rewards/margins": 0.09173583984375,
      "rewards/rejected": -0.16159668564796448,
      "step": 4060
    },
    {
      "epoch": 0.3088480801335559,
      "grad_norm": 2.0422660259092082,
      "learning_rate": 1.2539862728441536e-06,
      "log_odds_chosen": 1.2222168445587158,
      "log_odds_ratio": -0.46635740995407104,
      "logits/chosen": -0.9429687261581421,
      "logits/rejected": -0.8658202886581421,
      "logps/chosen": -0.699999988079071,
      "logps/rejected": -1.578710913658142,
      "loss": 0.9181,
      "nll_loss": 0.825976550579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0699462890625,
      "rewards/margins": 0.0879364013671875,
      "rewards/rejected": -0.157958984375,
      "step": 4070
    },
    {
      "epoch": 0.30960692062528455,
      "grad_norm": 2.2480501927541967,
      "learning_rate": 1.252448582170299e-06,
      "log_odds_chosen": 1.4699218273162842,
      "log_odds_ratio": -0.43339842557907104,
      "logits/chosen": -0.9517577886581421,
      "logits/rejected": -0.873828113079071,
      "logps/chosen": -0.7090820074081421,
      "logps/rejected": -1.7548828125,
      "loss": 0.9077,
      "nll_loss": 0.899609386920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07089843600988388,
      "rewards/margins": 0.10444335639476776,
      "rewards/rejected": -0.17539063096046448,
      "step": 4080
    },
    {
      "epoch": 0.3103657611170132,
      "grad_norm": 2.2268249219221,
      "learning_rate": 1.2509165344015243e-06,
      "log_odds_chosen": 1.2501952648162842,
      "log_odds_ratio": -0.46025389432907104,
      "logits/chosen": -0.967968761920929,
      "logits/rejected": -0.870312511920929,
      "logps/chosen": -0.673632800579071,
      "logps/rejected": -1.5476562976837158,
      "loss": 0.9388,
      "nll_loss": 0.914843738079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06741943210363388,
      "rewards/margins": 0.08745117485523224,
      "rewards/rejected": -0.15473632514476776,
      "step": 4090
    },
    {
      "epoch": 0.31112460160874184,
      "grad_norm": 2.27005999070546,
      "learning_rate": 1.2493900951088486e-06,
      "log_odds_chosen": 1.28955078125,
      "log_odds_ratio": -0.42329102754592896,
      "logits/chosen": -0.951171875,
      "logits/rejected": -0.8755859136581421,
      "logps/chosen": -0.691601574420929,
      "logps/rejected": -1.557031273841858,
      "loss": 0.9157,
      "nll_loss": 0.8306640386581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06904296576976776,
      "rewards/margins": 0.0865631103515625,
      "rewards/rejected": -0.1556396484375,
      "step": 4100
    },
    {
      "epoch": 0.3118834421004705,
      "grad_norm": 2.1754099073977238,
      "learning_rate": 1.2478692301566601e-06,
      "log_odds_chosen": 1.5134766101837158,
      "log_odds_ratio": -0.4137206971645355,
      "logits/chosen": -0.948046863079071,
      "logits/rejected": -0.8490234613418579,
      "logps/chosen": -0.667773425579071,
      "logps/rejected": -1.7433593273162842,
      "loss": 0.902,
      "nll_loss": 0.8935546875,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06674804538488388,
      "rewards/margins": 0.10740967094898224,
      "rewards/rejected": -0.17424316704273224,
      "step": 4110
    },
    {
      "epoch": 0.3126422825921991,
      "grad_norm": 1.8991756706793939,
      "learning_rate": 1.2463539056995116e-06,
      "log_odds_chosen": 1.2667968273162842,
      "log_odds_ratio": -0.44287109375,
      "logits/chosen": -0.9554687738418579,
      "logits/rejected": -0.882617175579071,
      "logps/chosen": -0.703125,
      "logps/rejected": -1.630859375,
      "loss": 0.9017,
      "nll_loss": 0.8999999761581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07025146484375,
      "rewards/margins": 0.09304885566234589,
      "rewards/rejected": -0.16325683891773224,
      "step": 4120
    },
    {
      "epoch": 0.31340112308392776,
      "grad_norm": 1.960326000266427,
      "learning_rate": 1.2448440881789541e-06,
      "log_odds_chosen": 1.252832055091858,
      "log_odds_ratio": -0.4205078184604645,
      "logits/chosen": -0.9300781488418579,
      "logits/rejected": -0.866406261920929,
      "logps/chosen": -0.6532226800918579,
      "logps/rejected": -1.4568359851837158,
      "loss": 0.8844,
      "nll_loss": 0.8388671875,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06529541313648224,
      "rewards/margins": 0.08049926906824112,
      "rewards/rejected": -0.14572754502296448,
      "step": 4130
    },
    {
      "epoch": 0.3141599635756564,
      "grad_norm": 1.9958829149266735,
      "learning_rate": 1.2433397443204184e-06,
      "log_odds_chosen": 1.1348145008087158,
      "log_odds_ratio": -0.4706054627895355,
      "logits/chosen": -0.9605468511581421,
      "logits/rejected": -0.8822265863418579,
      "logps/chosen": -0.6656249761581421,
      "logps/rejected": -1.4658203125,
      "loss": 0.9019,
      "nll_loss": 0.8388671875,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06654052436351776,
      "rewards/margins": 0.07998809963464737,
      "rewards/rejected": -0.14653320610523224,
      "step": 4140
    },
    {
      "epoch": 0.31491880406738504,
      "grad_norm": 4.091569889843401,
      "learning_rate": 1.2418408411301324e-06,
      "log_odds_chosen": 1.275976538658142,
      "log_odds_ratio": -0.400634765625,
      "logits/chosen": -0.9775390625,
      "logits/rejected": -0.8160156011581421,
      "logps/chosen": -0.6201171875,
      "logps/rejected": -1.4548828601837158,
      "loss": 0.901,
      "nll_loss": 0.78271484375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06202392652630806,
      "rewards/margins": 0.08343505859375,
      "rewards/rejected": -0.14548340439796448,
      "step": 4150
    },
    {
      "epoch": 0.3156776445591137,
      "grad_norm": 2.2497340997960076,
      "learning_rate": 1.2403473458920844e-06,
      "log_odds_chosen": 1.372656226158142,
      "log_odds_ratio": -0.43525391817092896,
      "logits/chosen": -0.9624999761581421,
      "logits/rejected": -0.8412109613418579,
      "logps/chosen": -0.692187488079071,
      "logps/rejected": -1.6554687023162842,
      "loss": 0.8977,
      "nll_loss": 0.8486328125,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06931152194738388,
      "rewards/margins": 0.09630127251148224,
      "rewards/rejected": -0.16550293564796448,
      "step": 4160
    },
    {
      "epoch": 0.3164364850508423,
      "grad_norm": 2.038350353584351,
      "learning_rate": 1.2388592261650217e-06,
      "log_odds_chosen": 1.199072241783142,
      "log_odds_ratio": -0.4593749940395355,
      "logits/chosen": -0.9271484613418579,
      "logits/rejected": -0.8607422113418579,
      "logps/chosen": -0.6734374761581421,
      "logps/rejected": -1.5021483898162842,
      "loss": 0.8952,
      "nll_loss": 0.833984375,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06735839694738388,
      "rewards/margins": 0.08281860500574112,
      "rewards/rejected": -0.15019531548023224,
      "step": 4170
    },
    {
      "epoch": 0.31719532554257096,
      "grad_norm": 2.1135411907978474,
      "learning_rate": 1.2373764497794918e-06,
      "log_odds_chosen": 1.4931640625,
      "log_odds_ratio": -0.38886719942092896,
      "logits/chosen": -0.953320324420929,
      "logits/rejected": -0.8460937738418579,
      "logps/chosen": -0.63427734375,
      "logps/rejected": -1.657812476158142,
      "loss": 0.8894,
      "nll_loss": 0.837890625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06341552734375,
      "rewards/margins": 0.10235595703125,
      "rewards/rejected": -0.16586914658546448,
      "step": 4180
    },
    {
      "epoch": 0.3179541660342996,
      "grad_norm": 2.0657817064350033,
      "learning_rate": 1.2358989848349217e-06,
      "log_odds_chosen": 1.474707007408142,
      "log_odds_ratio": -0.37358397245407104,
      "logits/chosen": -0.8857421875,
      "logits/rejected": -0.806445300579071,
      "logps/chosen": -0.6400390863418579,
      "logps/rejected": -1.6335937976837158,
      "loss": 0.8925,
      "nll_loss": 0.833789050579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06398925930261612,
      "rewards/margins": 0.09937743842601776,
      "rewards/rejected": -0.16340331733226776,
      "step": 4190
    },
    {
      "epoch": 0.31871300652602824,
      "grad_norm": 2.182532950524458,
      "learning_rate": 1.2344267996967353e-06,
      "log_odds_chosen": 1.465917944908142,
      "log_odds_ratio": -0.3947509825229645,
      "logits/chosen": -0.9203125238418579,
      "logits/rejected": -0.852734386920929,
      "logps/chosen": -0.662402331829071,
      "logps/rejected": -1.6550781726837158,
      "loss": 0.908,
      "nll_loss": 0.9375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06618652492761612,
      "rewards/margins": 0.09916381537914276,
      "rewards/rejected": -0.16538086533546448,
      "step": 4200
    },
    {
      "epoch": 0.3194718470177569,
      "grad_norm": 1.9590389783181057,
      "learning_rate": 1.2329598629935076e-06,
      "log_odds_chosen": 1.366308569908142,
      "log_odds_ratio": -0.4139648377895355,
      "logits/chosen": -0.9091796875,
      "logits/rejected": -0.8013671636581421,
      "logps/chosen": -0.6680663824081421,
      "logps/rejected": -1.6267578601837158,
      "loss": 0.8914,
      "nll_loss": 0.8695312738418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06683349609375,
      "rewards/margins": 0.09581909328699112,
      "rewards/rejected": -0.16259765625,
      "step": 4210
    },
    {
      "epoch": 0.3202306875094855,
      "grad_norm": 2.1182170742214326,
      "learning_rate": 1.2314981436141583e-06,
      "log_odds_chosen": 1.3098633289337158,
      "log_odds_ratio": -0.4046386778354645,
      "logits/chosen": -0.9716796875,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.619335949420929,
      "logps/rejected": -1.520117163658142,
      "loss": 0.8848,
      "nll_loss": 0.7918945550918579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06196289137005806,
      "rewards/margins": 0.09002685546875,
      "rewards/rejected": -0.15190429985523224,
      "step": 4220
    },
    {
      "epoch": 0.32098952800121416,
      "grad_norm": 2.1681970591127184,
      "learning_rate": 1.2300416107051802e-06,
      "log_odds_chosen": 1.2394530773162842,
      "log_odds_ratio": -0.45927733182907104,
      "logits/chosen": -0.9613281488418579,
      "logits/rejected": -0.832812488079071,
      "logps/chosen": -0.72607421875,
      "logps/rejected": -1.5896484851837158,
      "loss": 0.9274,
      "nll_loss": 0.852343738079071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07264403998851776,
      "rewards/margins": 0.08648681640625,
      "rewards/rejected": -0.15913085639476776,
      "step": 4230
    },
    {
      "epoch": 0.3217483684929428,
      "grad_norm": 2.046706232589475,
      "learning_rate": 1.2285902336679024e-06,
      "log_odds_chosen": 1.12548828125,
      "log_odds_ratio": -0.4857421815395355,
      "logits/chosen": -0.963085949420929,
      "logits/rejected": -0.8150390386581421,
      "logps/chosen": -0.7162109613418579,
      "logps/rejected": -1.516210913658142,
      "loss": 0.9045,
      "nll_loss": 0.8978515863418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07159423828125,
      "rewards/margins": 0.08000488579273224,
      "rewards/rejected": -0.151611328125,
      "step": 4240
    },
    {
      "epoch": 0.32250720898467145,
      "grad_norm": 2.291442872750335,
      "learning_rate": 1.2271439821557926e-06,
      "log_odds_chosen": 1.22265625,
      "log_odds_ratio": -0.43427735567092896,
      "logits/chosen": -0.926953136920929,
      "logits/rejected": -0.859179675579071,
      "logps/chosen": -0.6561523675918579,
      "logps/rejected": -1.5001952648162842,
      "loss": 0.901,
      "nll_loss": 0.830859363079071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06550292670726776,
      "rewards/margins": 0.08442382514476776,
      "rewards/rejected": -0.14992675185203552,
      "step": 4250
    },
    {
      "epoch": 0.3232660494764001,
      "grad_norm": 2.3934126129829516,
      "learning_rate": 1.225702826071791e-06,
      "log_odds_chosen": 1.6476562023162842,
      "log_odds_ratio": -0.3468261659145355,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.870898425579071,
      "logps/chosen": -0.622851550579071,
      "logps/rejected": -1.7775390148162842,
      "loss": 0.8859,
      "nll_loss": 0.803417980670929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06228027492761612,
      "rewards/margins": 0.11549071967601776,
      "rewards/rejected": -0.17783203721046448,
      "step": 4260
    },
    {
      "epoch": 0.3240248899681287,
      "grad_norm": 2.2978147982794916,
      "learning_rate": 1.2242667355656797e-06,
      "log_odds_chosen": 1.464257836341858,
      "log_odds_ratio": -0.3810058534145355,
      "logits/chosen": -0.9378906488418579,
      "logits/rejected": -0.811328113079071,
      "logps/chosen": -0.6431640386581421,
      "logps/rejected": -1.6599609851837158,
      "loss": 0.862,
      "nll_loss": 0.8465820550918579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06434325873851776,
      "rewards/margins": 0.1016845703125,
      "rewards/rejected": -0.16606445610523224,
      "step": 4270
    },
    {
      "epoch": 0.3247837304598573,
      "grad_norm": 2.2231775385940775,
      "learning_rate": 1.2228356810314862e-06,
      "log_odds_chosen": 1.059661865234375,
      "log_odds_ratio": -0.49833983182907104,
      "logits/chosen": -0.930859386920929,
      "logits/rejected": -0.8203125,
      "logps/chosen": -0.7120116949081421,
      "logps/rejected": -1.4490234851837158,
      "loss": 0.9366,
      "nll_loss": 0.8724609613418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07120361179113388,
      "rewards/margins": 0.07371368259191513,
      "rewards/rejected": -0.14499512314796448,
      "step": 4280
    },
    {
      "epoch": 0.32554257095158595,
      "grad_norm": 2.2345119533491937,
      "learning_rate": 1.2214096331049186e-06,
      "log_odds_chosen": 1.5973632335662842,
      "log_odds_ratio": -0.401611328125,
      "logits/chosen": -0.9457031488418579,
      "logits/rejected": -0.8681640625,
      "logps/chosen": -0.6317383050918579,
      "logps/rejected": -1.779687523841858,
      "loss": 0.8845,
      "nll_loss": 0.7544921636581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06319580227136612,
      "rewards/margins": 0.11481933295726776,
      "rewards/rejected": -0.17795410752296448,
      "step": 4290
    },
    {
      "epoch": 0.3263014114433146,
      "grad_norm": 2.5710473923093504,
      "learning_rate": 1.2199885626608373e-06,
      "log_odds_chosen": 1.3624999523162842,
      "log_odds_ratio": -0.4041503965854645,
      "logits/chosen": -0.9730468988418579,
      "logits/rejected": -0.8931640386581421,
      "logps/chosen": -0.6630859375,
      "logps/rejected": -1.584375023841858,
      "loss": 0.902,
      "nll_loss": 0.8310546875,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06634521484375,
      "rewards/margins": 0.09211425483226776,
      "rewards/rejected": -0.15842285752296448,
      "step": 4300
    },
    {
      "epoch": 0.32706025193504323,
      "grad_norm": 1.9495590217172698,
      "learning_rate": 1.2185724408107546e-06,
      "log_odds_chosen": 1.344335913658142,
      "log_odds_ratio": -0.4015136659145355,
      "logits/chosen": -0.976367175579071,
      "logits/rejected": -0.8623046875,
      "logps/chosen": -0.651171863079071,
      "logps/rejected": -1.5617187023162842,
      "loss": 0.9226,
      "nll_loss": 0.8587890863418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06514892727136612,
      "rewards/margins": 0.09112548828125,
      "rewards/rejected": -0.15617676079273224,
      "step": 4310
    },
    {
      "epoch": 0.3278190924267719,
      "grad_norm": 2.0775732902118866,
      "learning_rate": 1.2171612389003689e-06,
      "log_odds_chosen": 1.409570336341858,
      "log_odds_ratio": -0.37944334745407104,
      "logits/chosen": -0.9593750238418579,
      "logits/rejected": -0.8255859613418579,
      "logps/chosen": -0.671679675579071,
      "logps/rejected": -1.689453125,
      "loss": 0.8989,
      "nll_loss": 0.8636718988418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06711425632238388,
      "rewards/margins": 0.101898193359375,
      "rewards/rejected": -0.16921386122703552,
      "step": 4320
    },
    {
      "epoch": 0.3285779329185005,
      "grad_norm": 2.1901595897704644,
      "learning_rate": 1.2157549285071297e-06,
      "log_odds_chosen": 1.056640625,
      "log_odds_ratio": -0.494873046875,
      "logits/chosen": -0.951367199420929,
      "logits/rejected": -0.8515625,
      "logps/chosen": -0.708789050579071,
      "logps/rejected": -1.460546851158142,
      "loss": 0.9165,
      "nll_loss": 0.8740234375,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07082519680261612,
      "rewards/margins": 0.07513427734375,
      "rewards/rejected": -0.14604492485523224,
      "step": 4330
    },
    {
      "epoch": 0.32933677341022916,
      "grad_norm": 1.9604302213993103,
      "learning_rate": 1.2143534814378327e-06,
      "log_odds_chosen": 1.382421851158142,
      "log_odds_ratio": -0.41289061307907104,
      "logits/chosen": -0.914843738079071,
      "logits/rejected": -0.8330078125,
      "logps/chosen": -0.682910144329071,
      "logps/rejected": -1.63671875,
      "loss": 0.8791,
      "nll_loss": 0.846386730670929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06829833984375,
      "rewards/margins": 0.095367431640625,
      "rewards/rejected": -0.16352538764476776,
      "step": 4340
    },
    {
      "epoch": 0.3300956139019578,
      "grad_norm": 2.156153319128,
      "learning_rate": 1.2129568697262454e-06,
      "log_odds_chosen": 1.302148461341858,
      "log_odds_ratio": -0.41865235567092896,
      "logits/chosen": -0.9525390863418579,
      "logits/rejected": -0.8365234136581421,
      "logps/chosen": -0.6341797113418579,
      "logps/rejected": -1.4474608898162842,
      "loss": 0.8645,
      "nll_loss": 0.8104492425918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06348876655101776,
      "rewards/margins": 0.08118285983800888,
      "rewards/rejected": -0.1446533203125,
      "step": 4350
    },
    {
      "epoch": 0.33085445439368644,
      "grad_norm": 2.5408462766533515,
      "learning_rate": 1.2115650656307653e-06,
      "log_odds_chosen": 1.381445288658142,
      "log_odds_ratio": -0.3592773377895355,
      "logits/chosen": -0.9085937738418579,
      "logits/rejected": -0.8330078125,
      "logps/chosen": -0.6597656011581421,
      "logps/rejected": -1.552343726158142,
      "loss": 0.8761,
      "nll_loss": 0.7962890863418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06594238430261612,
      "rewards/margins": 0.08934326469898224,
      "rewards/rejected": -0.15520019829273224,
      "step": 4360
    },
    {
      "epoch": 0.3316132948854151,
      "grad_norm": 2.755454510311766,
      "learning_rate": 1.210178041632103e-06,
      "log_odds_chosen": 1.542578101158142,
      "log_odds_ratio": -0.3629394471645355,
      "logits/chosen": -0.909375011920929,
      "logits/rejected": -0.854296863079071,
      "logps/chosen": -0.62451171875,
      "logps/rejected": -1.678125023841858,
      "loss": 0.9076,
      "nll_loss": 0.865039050579071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06241454929113388,
      "rewards/margins": 0.10528564453125,
      "rewards/rejected": -0.16774901747703552,
      "step": 4370
    },
    {
      "epoch": 0.3323721353771437,
      "grad_norm": 2.043167141768478,
      "learning_rate": 1.2087957704309988e-06,
      "log_odds_chosen": 1.2589843273162842,
      "log_odds_ratio": -0.42744141817092896,
      "logits/chosen": -0.9208984375,
      "logits/rejected": -0.8177734613418579,
      "logps/chosen": -0.638476550579071,
      "logps/rejected": -1.4943358898162842,
      "loss": 0.8986,
      "nll_loss": 0.804492175579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0638427734375,
      "rewards/margins": 0.08549194037914276,
      "rewards/rejected": -0.14938965439796448,
      "step": 4380
    },
    {
      "epoch": 0.33313097586887236,
      "grad_norm": 2.174469428614346,
      "learning_rate": 1.2074182249459642e-06,
      "log_odds_chosen": 1.512841820716858,
      "log_odds_ratio": -0.40498048067092896,
      "logits/chosen": -0.9583984613418579,
      "logits/rejected": -0.875781238079071,
      "logps/chosen": -0.666210949420929,
      "logps/rejected": -1.726171851158142,
      "loss": 0.8872,
      "nll_loss": 0.817675769329071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06667480617761612,
      "rewards/margins": 0.10597991943359375,
      "rewards/rejected": -0.17250975966453552,
      "step": 4390
    },
    {
      "epoch": 0.333889816360601,
      "grad_norm": 2.220872324929471,
      "learning_rate": 1.2060453783110545e-06,
      "log_odds_chosen": 1.1480224132537842,
      "log_odds_ratio": -0.46357423067092896,
      "logits/chosen": -0.9693359136581421,
      "logits/rejected": -0.8525390625,
      "logps/chosen": -0.6807616949081421,
      "logps/rejected": -1.4871094226837158,
      "loss": 0.8837,
      "nll_loss": 0.805468738079071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06805419921875,
      "rewards/margins": 0.08069610595703125,
      "rewards/rejected": -0.14877930283546448,
      "step": 4400
    },
    {
      "epoch": 0.33464865685232964,
      "grad_norm": 2.2372440653759513,
      "learning_rate": 1.2046772038736682e-06,
      "log_odds_chosen": 1.3009765148162842,
      "log_odds_ratio": -0.434326171875,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.8521484136581421,
      "logps/chosen": -0.6429687738418579,
      "logps/rejected": -1.5300781726837158,
      "loss": 0.9141,
      "nll_loss": 0.8697265386581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06434325873851776,
      "rewards/margins": 0.08879394829273224,
      "rewards/rejected": -0.15317383408546448,
      "step": 4410
    },
    {
      "epoch": 0.3354074973440583,
      "grad_norm": 2.2973654270322967,
      "learning_rate": 1.2033136751923736e-06,
      "log_odds_chosen": 1.101904273033142,
      "log_odds_ratio": -0.45087891817092896,
      "logits/chosen": -0.919726550579071,
      "logits/rejected": -0.7900390625,
      "logps/chosen": -0.700976550579071,
      "logps/rejected": -1.473242163658142,
      "loss": 0.9005,
      "nll_loss": 0.849414050579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07012939453125,
      "rewards/margins": 0.07722167670726776,
      "rewards/rejected": -0.14738769829273224,
      "step": 4420
    },
    {
      "epoch": 0.3361663378357869,
      "grad_norm": 2.247061985239726,
      "learning_rate": 1.201954766034762e-06,
      "log_odds_chosen": 1.4426758289337158,
      "log_odds_ratio": -0.42011719942092896,
      "logits/chosen": -0.9330078363418579,
      "logits/rejected": -0.845507800579071,
      "logps/chosen": -0.661425769329071,
      "logps/rejected": -1.6591796875,
      "loss": 0.8758,
      "nll_loss": 0.8121093511581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.066162109375,
      "rewards/margins": 0.09980468451976776,
      "rewards/rejected": -0.16584472358226776,
      "step": 4430
    },
    {
      "epoch": 0.33692517832751556,
      "grad_norm": 2.2570856208655163,
      "learning_rate": 1.2006004503753285e-06,
      "log_odds_chosen": 1.5849609375,
      "log_odds_ratio": -0.4055419862270355,
      "logits/chosen": -0.9097656011581421,
      "logits/rejected": -0.8033202886581421,
      "logps/chosen": -0.640917956829071,
      "logps/rejected": -1.806054711341858,
      "loss": 0.903,
      "nll_loss": 0.824999988079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06409911811351776,
      "rewards/margins": 0.11650238186120987,
      "rewards/rejected": -0.1805419921875,
      "step": 4440
    },
    {
      "epoch": 0.3376840188192442,
      "grad_norm": 2.2141600181166363,
      "learning_rate": 1.1992507023933782e-06,
      "log_odds_chosen": 1.526513695716858,
      "log_odds_ratio": -0.3704589903354645,
      "logits/chosen": -0.939648449420929,
      "logits/rejected": -0.822265625,
      "logps/chosen": -0.69140625,
      "logps/rejected": -1.748437523841858,
      "loss": 0.8751,
      "nll_loss": 0.888476550579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06917724758386612,
      "rewards/margins": 0.10583877563476562,
      "rewards/rejected": -0.1751708984375,
      "step": 4450
    },
    {
      "epoch": 0.33844285931097284,
      "grad_norm": 2.0103647610137925,
      "learning_rate": 1.1979054964709597e-06,
      "log_odds_chosen": 1.1826171875,
      "log_odds_ratio": -0.4234375059604645,
      "logits/chosen": -0.941210925579071,
      "logits/rejected": -0.809374988079071,
      "logps/chosen": -0.6451171636581421,
      "logps/rejected": -1.4169921875,
      "loss": 0.8857,
      "nll_loss": 0.794140636920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06455077975988388,
      "rewards/margins": 0.07726440578699112,
      "rewards/rejected": -0.14177246391773224,
      "step": 4460
    },
    {
      "epoch": 0.3392016998027015,
      "grad_norm": 2.344873760987262,
      "learning_rate": 1.1965648071908207e-06,
      "log_odds_chosen": 1.375,
      "log_odds_ratio": -0.39765626192092896,
      "logits/chosen": -0.9437500238418579,
      "logits/rejected": -0.8138672113418579,
      "logps/chosen": -0.626953125,
      "logps/rejected": -1.572265625,
      "loss": 0.8756,
      "nll_loss": 0.788769543170929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06269530951976776,
      "rewards/margins": 0.09456787258386612,
      "rewards/rejected": -0.15725097060203552,
      "step": 4470
    },
    {
      "epoch": 0.3399605402944301,
      "grad_norm": 1.9603069157506832,
      "learning_rate": 1.1952286093343935e-06,
      "log_odds_chosen": 1.318945288658142,
      "log_odds_ratio": -0.42138671875,
      "logits/chosen": -0.9644531011581421,
      "logits/rejected": -0.848437488079071,
      "logps/chosen": -0.660449206829071,
      "logps/rejected": -1.5525391101837158,
      "loss": 0.8975,
      "nll_loss": 0.8497070074081421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06605224311351776,
      "rewards/margins": 0.08906249701976776,
      "rewards/rejected": -0.15517577528953552,
      "step": 4480
    },
    {
      "epoch": 0.34071938078615877,
      "grad_norm": 2.3007253003330135,
      "learning_rate": 1.1938968778798005e-06,
      "log_odds_chosen": 1.3662109375,
      "log_odds_ratio": -0.3960937559604645,
      "logits/chosen": -0.921875,
      "logits/rejected": -0.7919921875,
      "logps/chosen": -0.637890636920929,
      "logps/rejected": -1.585546851158142,
      "loss": 0.8729,
      "nll_loss": 0.8238281011581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06375732272863388,
      "rewards/margins": 0.09487304836511612,
      "rewards/rejected": -0.15864257514476776,
      "step": 4490
    },
    {
      "epoch": 0.3414782212778874,
      "grad_norm": 2.0606137302048273,
      "learning_rate": 1.1925695879998878e-06,
      "log_odds_chosen": 1.36328125,
      "log_odds_ratio": -0.41728514432907104,
      "logits/chosen": -0.9349609613418579,
      "logits/rejected": -0.8619140386581421,
      "logps/chosen": -0.6795898675918579,
      "logps/rejected": -1.6300780773162842,
      "loss": 0.8698,
      "nll_loss": 0.849609375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0679931640625,
      "rewards/margins": 0.09499511867761612,
      "rewards/rejected": -0.16298827528953552,
      "step": 4500
    },
    {
      "epoch": 0.34223706176961605,
      "grad_norm": 2.2176914597724995,
      "learning_rate": 1.1912467150602794e-06,
      "log_odds_chosen": 1.429296851158142,
      "log_odds_ratio": -0.3785644471645355,
      "logits/chosen": -0.946484386920929,
      "logits/rejected": -0.8521484136581421,
      "logps/chosen": -0.6634765863418579,
      "logps/rejected": -1.657812476158142,
      "loss": 0.8836,
      "nll_loss": 0.82763671875,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06641845405101776,
      "rewards/margins": 0.09949950873851776,
      "rewards/rejected": -0.16584472358226776,
      "step": 4510
    },
    {
      "epoch": 0.3429959022613447,
      "grad_norm": 2.588685801085802,
      "learning_rate": 1.189928234617459e-06,
      "log_odds_chosen": 1.237280249595642,
      "log_odds_ratio": -0.4388671815395355,
      "logits/chosen": -0.9232422113418579,
      "logits/rejected": -0.801953136920929,
      "logps/chosen": -0.6878906488418579,
      "logps/rejected": -1.5712890625,
      "loss": 0.8666,
      "nll_loss": 0.8218749761581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06878662109375,
      "rewards/margins": 0.08840103447437286,
      "rewards/rejected": -0.15731200575828552,
      "step": 4520
    },
    {
      "epoch": 0.34375474275307333,
      "grad_norm": 2.320935556141627,
      "learning_rate": 1.1886141224168716e-06,
      "log_odds_chosen": 1.211328148841858,
      "log_odds_ratio": -0.43071287870407104,
      "logits/chosen": -0.8705078363418579,
      "logits/rejected": -0.8150390386581421,
      "logps/chosen": -0.641406238079071,
      "logps/rejected": -1.460546851158142,
      "loss": 0.8651,
      "nll_loss": 0.822070300579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06413574516773224,
      "rewards/margins": 0.08183594048023224,
      "rewards/rejected": -0.14589843153953552,
      "step": 4530
    },
    {
      "epoch": 0.34451358324480197,
      "grad_norm": 2.1049123689733786,
      "learning_rate": 1.1873043543910495e-06,
      "log_odds_chosen": 1.21533203125,
      "log_odds_ratio": -0.42182618379592896,
      "logits/chosen": -0.920117199420929,
      "logits/rejected": -0.8255859613418579,
      "logps/chosen": -0.618945300579071,
      "logps/rejected": -1.412500023841858,
      "loss": 0.8874,
      "nll_loss": 0.83203125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06195068359375,
      "rewards/margins": 0.07939453423023224,
      "rewards/rejected": -0.14145508408546448,
      "step": 4540
    },
    {
      "epoch": 0.34527242373653055,
      "grad_norm": 2.1976254676362554,
      "learning_rate": 1.1859989066577617e-06,
      "log_odds_chosen": 1.4050781726837158,
      "log_odds_ratio": -0.388671875,
      "logits/chosen": -0.910351574420929,
      "logits/rejected": -0.80078125,
      "logps/chosen": -0.6504882574081421,
      "logps/rejected": -1.6105468273162842,
      "loss": 0.8843,
      "nll_loss": 0.800976574420929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06500244140625,
      "rewards/margins": 0.0960693359375,
      "rewards/rejected": -0.16103515028953552,
      "step": 4550
    },
    {
      "epoch": 0.3460312642282592,
      "grad_norm": 2.1822551995963404,
      "learning_rate": 1.1846977555181846e-06,
      "log_odds_chosen": 1.103369116783142,
      "log_odds_ratio": -0.4722656309604645,
      "logits/chosen": -0.8988281488418579,
      "logits/rejected": -0.824999988079071,
      "logps/chosen": -0.7012695074081421,
      "logps/rejected": -1.4494140148162842,
      "loss": 0.8828,
      "nll_loss": 0.890625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07011719048023224,
      "rewards/margins": 0.07481994479894638,
      "rewards/rejected": -0.14504393935203552,
      "step": 4560
    },
    {
      "epoch": 0.34679010471998784,
      "grad_norm": 2.3308594142253143,
      "learning_rate": 1.1834008774550946e-06,
      "log_odds_chosen": 1.40966796875,
      "log_odds_ratio": -0.40717774629592896,
      "logits/chosen": -0.936328113079071,
      "logits/rejected": -0.8021484613418579,
      "logps/chosen": -0.637011706829071,
      "logps/rejected": -1.5871093273162842,
      "loss": 0.8749,
      "nll_loss": 0.81494140625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06374511867761612,
      "rewards/margins": 0.09494628757238388,
      "rewards/rejected": -0.15874023735523224,
      "step": 4570
    },
    {
      "epoch": 0.3475489452117165,
      "grad_norm": 2.4823474237824605,
      "learning_rate": 1.1821082491310835e-06,
      "log_odds_chosen": 1.223535180091858,
      "log_odds_ratio": -0.4647460877895355,
      "logits/chosen": -0.913281261920929,
      "logits/rejected": -0.8218749761581421,
      "logps/chosen": -0.6629883050918579,
      "logps/rejected": -1.484375,
      "loss": 0.858,
      "nll_loss": 0.824414074420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06619872897863388,
      "rewards/margins": 0.08223648369312286,
      "rewards/rejected": -0.14836426079273224,
      "step": 4580
    },
    {
      "epoch": 0.3483077857034451,
      "grad_norm": 2.0314177948784358,
      "learning_rate": 1.1808198473867937e-06,
      "log_odds_chosen": 1.179785132408142,
      "log_odds_ratio": -0.46484375,
      "logits/chosen": -1.0146484375,
      "logits/rejected": -0.906445324420929,
      "logps/chosen": -0.69921875,
      "logps/rejected": -1.537500023841858,
      "loss": 0.8742,
      "nll_loss": 0.8388671875,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06987304985523224,
      "rewards/margins": 0.08375243842601776,
      "rewards/rejected": -0.15375976264476776,
      "step": 4590
    },
    {
      "epoch": 0.34906662619517376,
      "grad_norm": 1.9804242538388541,
      "learning_rate": 1.179535649239177e-06,
      "log_odds_chosen": 1.2795898914337158,
      "log_odds_ratio": -0.4310058653354645,
      "logits/chosen": -0.903515636920929,
      "logits/rejected": -0.843554675579071,
      "logps/chosen": -0.655957043170929,
      "logps/rejected": -1.542578101158142,
      "loss": 0.8787,
      "nll_loss": 0.8548828363418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06556396186351776,
      "rewards/margins": 0.08857116848230362,
      "rewards/rejected": -0.15422363579273224,
      "step": 4600
    },
    {
      "epoch": 0.3498254666869024,
      "grad_norm": 2.1676793117363498,
      "learning_rate": 1.178255631879771e-06,
      "log_odds_chosen": 1.33642578125,
      "log_odds_ratio": -0.406982421875,
      "logits/chosen": -0.940234363079071,
      "logits/rejected": -0.830273449420929,
      "logps/chosen": -0.685351550579071,
      "logps/rejected": -1.626953125,
      "loss": 0.886,
      "nll_loss": 0.8482421636581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06854248046875,
      "rewards/margins": 0.09415893256664276,
      "rewards/rejected": -0.16267089545726776,
      "step": 4610
    },
    {
      "epoch": 0.35058430717863104,
      "grad_norm": 2.0364662326117187,
      "learning_rate": 1.1769797726729992e-06,
      "log_odds_chosen": 0.963574230670929,
      "log_odds_ratio": -0.4800781309604645,
      "logits/chosen": -0.9091796875,
      "logits/rejected": -0.803515613079071,
      "logps/chosen": -0.69482421875,
      "logps/rejected": -1.3136718273162842,
      "loss": 0.8694,
      "nll_loss": 0.900195300579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06943359225988388,
      "rewards/margins": 0.06182251125574112,
      "rewards/rejected": -0.13129882514476776,
      "step": 4620
    },
    {
      "epoch": 0.3513431476703597,
      "grad_norm": 2.286515715845322,
      "learning_rate": 1.1757080491544881e-06,
      "log_odds_chosen": 1.3899657726287842,
      "log_odds_ratio": -0.4193359315395355,
      "logits/chosen": -0.9341796636581421,
      "logits/rejected": -0.8255859613418579,
      "logps/chosen": -0.683789074420929,
      "logps/rejected": -1.6462891101837158,
      "loss": 0.8888,
      "nll_loss": 0.846972644329071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06842041015625,
      "rewards/margins": 0.09616546332836151,
      "rewards/rejected": -0.16464844346046448,
      "step": 4630
    },
    {
      "epoch": 0.3521019881620883,
      "grad_norm": 2.1123173473303924,
      "learning_rate": 1.1744404390294068e-06,
      "log_odds_chosen": 1.2043945789337158,
      "log_odds_ratio": -0.45966798067092896,
      "logits/chosen": -0.9322265386581421,
      "logits/rejected": -0.8619140386581421,
      "logps/chosen": -0.688183605670929,
      "logps/rejected": -1.525390625,
      "loss": 0.853,
      "nll_loss": 0.8505859375,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06875000149011612,
      "rewards/margins": 0.08389892429113388,
      "rewards/rejected": -0.15251465141773224,
      "step": 4640
    },
    {
      "epoch": 0.35286082865381696,
      "grad_norm": 2.1002667873030645,
      "learning_rate": 1.1731769201708264e-06,
      "log_odds_chosen": 1.4666016101837158,
      "log_odds_ratio": -0.45659178495407104,
      "logits/chosen": -0.916210949420929,
      "logits/rejected": -0.8314453363418579,
      "logps/chosen": -0.671875,
      "logps/rejected": -1.7628905773162842,
      "loss": 0.8795,
      "nll_loss": 0.8500000238418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06715087592601776,
      "rewards/margins": 0.10904540866613388,
      "rewards/rejected": -0.17622070014476776,
      "step": 4650
    },
    {
      "epoch": 0.3536196691455456,
      "grad_norm": 2.5257160808205312,
      "learning_rate": 1.1719174706180952e-06,
      "log_odds_chosen": 1.6716797351837158,
      "log_odds_ratio": -0.3566650450229645,
      "logits/chosen": -0.924609363079071,
      "logits/rejected": -0.7876952886581421,
      "logps/chosen": -0.6436523199081421,
      "logps/rejected": -1.8214843273162842,
      "loss": 0.8786,
      "nll_loss": 0.822265625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06437988579273224,
      "rewards/margins": 0.11765136569738388,
      "rewards/rejected": -0.18222656846046448,
      "step": 4660
    },
    {
      "epoch": 0.35437850963727424,
      "grad_norm": 2.1474539080309687,
      "learning_rate": 1.1706620685752386e-06,
      "log_odds_chosen": 1.5402343273162842,
      "log_odds_ratio": -0.36308592557907104,
      "logits/chosen": -0.9468749761581421,
      "logits/rejected": -0.8091796636581421,
      "logps/chosen": -0.646679699420929,
      "logps/rejected": -1.7273437976837158,
      "loss": 0.8829,
      "nll_loss": 0.829882800579071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06467285007238388,
      "rewards/margins": 0.10795898735523224,
      "rewards/rejected": -0.17270508408546448,
      "step": 4670
    },
    {
      "epoch": 0.3551373501290029,
      "grad_norm": 2.2943867159959916,
      "learning_rate": 1.1694106924093723e-06,
      "log_odds_chosen": 1.168481469154358,
      "log_odds_ratio": -0.4696289002895355,
      "logits/chosen": -0.9359375238418579,
      "logits/rejected": -0.836718738079071,
      "logps/chosen": -0.728515625,
      "logps/rejected": -1.5451171398162842,
      "loss": 0.8728,
      "nll_loss": 0.8746093511581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07291259616613388,
      "rewards/margins": 0.08176727592945099,
      "rewards/rejected": -0.15458984673023224,
      "step": 4680
    },
    {
      "epoch": 0.3558961906207315,
      "grad_norm": 2.6928939956255524,
      "learning_rate": 1.1681633206491381e-06,
      "log_odds_chosen": 1.176171898841858,
      "log_odds_ratio": -0.4358886778354645,
      "logits/chosen": -0.9214843511581421,
      "logits/rejected": -0.7984374761581421,
      "logps/chosen": -0.6650390625,
      "logps/rejected": -1.473242163658142,
      "loss": 0.9016,
      "nll_loss": 0.853515625,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06654052436351776,
      "rewards/margins": 0.08079834282398224,
      "rewards/rejected": -0.147216796875,
      "step": 4690
    },
    {
      "epoch": 0.35665503111246016,
      "grad_norm": 2.6738779587849146,
      "learning_rate": 1.1669199319831564e-06,
      "log_odds_chosen": 1.326562523841858,
      "log_odds_ratio": -0.416015625,
      "logits/chosen": -0.915820300579071,
      "logits/rejected": -0.813671886920929,
      "logps/chosen": -0.674023449420929,
      "logps/rejected": -1.578515648841858,
      "loss": 0.8946,
      "nll_loss": 0.8443359136581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0673828125,
      "rewards/margins": 0.09053955227136612,
      "rewards/rejected": -0.15786132216453552,
      "step": 4700
    },
    {
      "epoch": 0.3574138716041888,
      "grad_norm": 2.400721974422087,
      "learning_rate": 1.1656805052584958e-06,
      "log_odds_chosen": 1.5525391101837158,
      "log_odds_ratio": -0.41228026151657104,
      "logits/chosen": -0.9332031011581421,
      "logits/rejected": -0.82421875,
      "logps/chosen": -0.6337890625,
      "logps/rejected": -1.6853516101837158,
      "loss": 0.8857,
      "nll_loss": 0.789257824420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06333007663488388,
      "rewards/margins": 0.10518798977136612,
      "rewards/rejected": -0.16850586235523224,
      "step": 4710
    },
    {
      "epoch": 0.35817271209591744,
      "grad_norm": 2.78621158336074,
      "learning_rate": 1.164445019479164e-06,
      "log_odds_chosen": 1.36474609375,
      "log_odds_ratio": -0.4100097715854645,
      "logits/chosen": -0.942578136920929,
      "logits/rejected": -0.8260742425918579,
      "logps/chosen": -0.6576172113418579,
      "logps/rejected": -1.57421875,
      "loss": 0.8657,
      "nll_loss": 0.802929699420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06569824367761612,
      "rewards/margins": 0.09161376953125,
      "rewards/rejected": -0.15742187201976776,
      "step": 4720
    },
    {
      "epoch": 0.3589315525876461,
      "grad_norm": 2.1486906743867236,
      "learning_rate": 1.1632134538046105e-06,
      "log_odds_chosen": 1.153161644935608,
      "log_odds_ratio": -0.4811035096645355,
      "logits/chosen": -0.9345703125,
      "logits/rejected": -0.806835949420929,
      "logps/chosen": -0.6722656488418579,
      "logps/rejected": -1.4718749523162842,
      "loss": 0.8575,
      "nll_loss": 0.855664074420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06721191108226776,
      "rewards/margins": 0.07992400974035263,
      "rewards/rejected": -0.14705809950828552,
      "step": 4730
    },
    {
      "epoch": 0.3596903930793747,
      "grad_norm": 2.365164784115729,
      "learning_rate": 1.1619857875482536e-06,
      "log_odds_chosen": 1.594384789466858,
      "log_odds_ratio": -0.3369140625,
      "logits/chosen": -0.931640625,
      "logits/rejected": -0.832226574420929,
      "logps/chosen": -0.630664050579071,
      "logps/rejected": -1.719140648841858,
      "loss": 0.8798,
      "nll_loss": 0.808300793170929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0631103515625,
      "rewards/margins": 0.10862121731042862,
      "rewards/rejected": -0.17164306342601776,
      "step": 4740
    },
    {
      "epoch": 0.36044923357110337,
      "grad_norm": 2.1740905865131164,
      "learning_rate": 1.1607620001760185e-06,
      "log_odds_chosen": 1.189550757408142,
      "log_odds_ratio": -0.46137696504592896,
      "logits/chosen": -0.9443359375,
      "logits/rejected": -0.8707031011581421,
      "logps/chosen": -0.6524413824081421,
      "logps/rejected": -1.4599609375,
      "loss": 0.8714,
      "nll_loss": 0.7978515625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06529541313648224,
      "rewards/margins": 0.08076782524585724,
      "rewards/rejected": -0.14604492485523224,
      "step": 4750
    },
    {
      "epoch": 0.361208074062832,
      "grad_norm": 2.1663302947053142,
      "learning_rate": 1.1595420713048968e-06,
      "log_odds_chosen": 1.485742211341858,
      "log_odds_ratio": -0.3995117247104645,
      "logits/chosen": -0.9716796875,
      "logits/rejected": -0.8482421636581421,
      "logps/chosen": -0.7230468988418579,
      "logps/rejected": -1.788671851158142,
      "loss": 0.8855,
      "nll_loss": 0.843554675579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07232666015625,
      "rewards/margins": 0.10650940239429474,
      "rewards/rejected": -0.17890624701976776,
      "step": 4760
    },
    {
      "epoch": 0.36196691455456065,
      "grad_norm": 3.308691876993286,
      "learning_rate": 1.1583259807015182e-06,
      "log_odds_chosen": 1.368554711341858,
      "log_odds_ratio": -0.4066406190395355,
      "logits/chosen": -0.9326171875,
      "logits/rejected": -0.822070300579071,
      "logps/chosen": -0.694531261920929,
      "logps/rejected": -1.6535155773162842,
      "loss": 0.88,
      "nll_loss": 0.863085925579071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06947021186351776,
      "rewards/margins": 0.09576416015625,
      "rewards/rejected": -0.16523437201976776,
      "step": 4770
    },
    {
      "epoch": 0.3627257550462893,
      "grad_norm": 2.192361081732721,
      "learning_rate": 1.1571137082807434e-06,
      "log_odds_chosen": 1.4275391101837158,
      "log_odds_ratio": -0.38886719942092896,
      "logits/chosen": -0.965039074420929,
      "logits/rejected": -0.8587890863418579,
      "logps/chosen": -0.6673828363418579,
      "logps/rejected": -1.667578101158142,
      "loss": 0.8549,
      "nll_loss": 0.8187500238418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06673584133386612,
      "rewards/margins": 0.10012207180261612,
      "rewards/rejected": -0.16679687798023224,
      "step": 4780
    },
    {
      "epoch": 0.36348459553801793,
      "grad_norm": 3.3996780477885356,
      "learning_rate": 1.155905234104269e-06,
      "log_odds_chosen": 1.564660668373108,
      "log_odds_ratio": -0.3798828125,
      "logits/chosen": -0.943554699420929,
      "logits/rejected": -0.795703113079071,
      "logps/chosen": -0.6434570550918579,
      "logps/rejected": -1.7576172351837158,
      "loss": 0.8883,
      "nll_loss": 0.8423827886581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06425781548023224,
      "rewards/margins": 0.11158599704504013,
      "rewards/rejected": -0.17592772841453552,
      "step": 4790
    },
    {
      "epoch": 0.36424343602974657,
      "grad_norm": 2.185774065398957,
      "learning_rate": 1.1547005383792516e-06,
      "log_odds_chosen": 1.408593773841858,
      "log_odds_ratio": -0.4390625059604645,
      "logits/chosen": -0.964062511920929,
      "logits/rejected": -0.8746093511581421,
      "logps/chosen": -0.65283203125,
      "logps/rejected": -1.615234375,
      "loss": 0.8545,
      "nll_loss": 0.817187488079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06531982123851776,
      "rewards/margins": 0.09609375149011612,
      "rewards/rejected": -0.16140136122703552,
      "step": 4800
    },
    {
      "epoch": 0.3650022765214752,
      "grad_norm": 2.3827746388722084,
      "learning_rate": 1.1534996014569446e-06,
      "log_odds_chosen": 1.5846679210662842,
      "log_odds_ratio": -0.39116209745407104,
      "logits/chosen": -0.992382824420929,
      "logits/rejected": -0.8853515386581421,
      "logps/chosen": -0.662792980670929,
      "logps/rejected": -1.7734375,
      "loss": 0.8416,
      "nll_loss": 0.82470703125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06625976413488388,
      "rewards/margins": 0.11094971001148224,
      "rewards/rejected": -0.17724609375,
      "step": 4810
    },
    {
      "epoch": 0.36576111701320385,
      "grad_norm": 2.22383281870511,
      "learning_rate": 1.1523024038313547e-06,
      "log_odds_chosen": 1.4586913585662842,
      "log_odds_ratio": -0.3944335877895355,
      "logits/chosen": -0.96875,
      "logits/rejected": -0.8539062738418579,
      "logps/chosen": -0.692675769329071,
      "logps/rejected": -1.717187523841858,
      "loss": 0.8684,
      "nll_loss": 0.8714843988418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06920166313648224,
      "rewards/margins": 0.10234222561120987,
      "rewards/rejected": -0.17170409858226776,
      "step": 4820
    },
    {
      "epoch": 0.36651995750493244,
      "grad_norm": 2.405173511680066,
      "learning_rate": 1.1511089261379083e-06,
      "log_odds_chosen": 1.4882080554962158,
      "log_odds_ratio": -0.40507811307907104,
      "logits/chosen": -0.91796875,
      "logits/rejected": -0.8207031488418579,
      "logps/chosen": -0.664257824420929,
      "logps/rejected": -1.7685546875,
      "loss": 0.874,
      "nll_loss": 0.831250011920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06644286960363388,
      "rewards/margins": 0.11036987602710724,
      "rewards/rejected": -0.17680664360523224,
      "step": 4830
    },
    {
      "epoch": 0.3672787979966611,
      "grad_norm": 2.3117284638928,
      "learning_rate": 1.149919149152138e-06,
      "log_odds_chosen": 1.329003930091858,
      "log_odds_ratio": -0.41816407442092896,
      "logits/chosen": -0.9652343988418579,
      "logits/rejected": -0.866992175579071,
      "logps/chosen": -0.6732422113418579,
      "logps/rejected": -1.618749976158142,
      "loss": 0.8594,
      "nll_loss": 0.854687511920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06730957329273224,
      "rewards/margins": 0.09454345703125,
      "rewards/rejected": -0.16188964247703552,
      "step": 4840
    },
    {
      "epoch": 0.3680376384883897,
      "grad_norm": 2.2721615455443147,
      "learning_rate": 1.148733053788381e-06,
      "log_odds_chosen": 1.3947265148162842,
      "log_odds_ratio": -0.4109863340854645,
      "logits/chosen": -0.950390636920929,
      "logits/rejected": -0.8623046875,
      "logps/chosen": -0.6473633050918579,
      "logps/rejected": -1.59375,
      "loss": 0.8642,
      "nll_loss": 0.8150390386581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.064697265625,
      "rewards/margins": 0.09471587836742401,
      "rewards/rejected": -0.159423828125,
      "step": 4850
    },
    {
      "epoch": 0.36879647898011836,
      "grad_norm": 2.2251074013039718,
      "learning_rate": 1.1475506210984938e-06,
      "log_odds_chosen": 1.3505859375,
      "log_odds_ratio": -0.40800780057907104,
      "logits/chosen": -0.9847656488418579,
      "logits/rejected": -0.8919922113418579,
      "logps/chosen": -0.66650390625,
      "logps/rejected": -1.58984375,
      "loss": 0.835,
      "nll_loss": 0.805468738079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06657715141773224,
      "rewards/margins": 0.09235839545726776,
      "rewards/rejected": -0.15900878608226776,
      "step": 4860
    },
    {
      "epoch": 0.369555319471847,
      "grad_norm": 1.9517990530874827,
      "learning_rate": 1.1463718322705807e-06,
      "log_odds_chosen": 1.147802710533142,
      "log_odds_ratio": -0.43701171875,
      "logits/chosen": -0.955273449420929,
      "logits/rejected": -0.8939453363418579,
      "logps/chosen": -0.6595703363418579,
      "logps/rejected": -1.411718726158142,
      "loss": 0.8602,
      "nll_loss": 0.8057616949081421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06586913764476776,
      "rewards/margins": 0.07521972805261612,
      "rewards/rejected": -0.14113768935203552,
      "step": 4870
    },
    {
      "epoch": 0.37031415996357564,
      "grad_norm": 2.4321003116514985,
      "learning_rate": 1.1451966686277364e-06,
      "log_odds_chosen": 1.424414038658142,
      "log_odds_ratio": -0.38740235567092896,
      "logits/chosen": -0.9224609136581421,
      "logits/rejected": -0.7994140386581421,
      "logps/chosen": -0.6337890625,
      "logps/rejected": -1.6320312023162842,
      "loss": 0.859,
      "nll_loss": 0.802734375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0633544921875,
      "rewards/margins": 0.09980468451976776,
      "rewards/rejected": -0.16313476860523224,
      "step": 4880
    },
    {
      "epoch": 0.3710730004553043,
      "grad_norm": 2.3392296736339198,
      "learning_rate": 1.1440251116268034e-06,
      "log_odds_chosen": 1.458593726158142,
      "log_odds_ratio": -0.4004882872104645,
      "logits/chosen": -0.9595702886581421,
      "logits/rejected": -0.8148437738418579,
      "logps/chosen": -0.670703113079071,
      "logps/rejected": -1.6671874523162842,
      "loss": 0.8698,
      "nll_loss": 0.810742199420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06708984076976776,
      "rewards/margins": 0.09968261420726776,
      "rewards/rejected": -0.166748046875,
      "step": 4890
    },
    {
      "epoch": 0.3718318409470329,
      "grad_norm": 2.0434121903510243,
      "learning_rate": 1.1428571428571428e-06,
      "log_odds_chosen": 1.3869140148162842,
      "log_odds_ratio": -0.425048828125,
      "logits/chosen": -0.9623047113418579,
      "logits/rejected": -0.8697265386581421,
      "logps/chosen": -0.664843738079071,
      "logps/rejected": -1.6328125,
      "loss": 0.8433,
      "nll_loss": 0.8255859613418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06645508110523224,
      "rewards/margins": 0.09688720852136612,
      "rewards/rejected": -0.1634521484375,
      "step": 4900
    },
    {
      "epoch": 0.37259068143876156,
      "grad_norm": 2.1097305955384544,
      "learning_rate": 1.14169274403942e-06,
      "log_odds_chosen": 1.365576148033142,
      "log_odds_ratio": -0.4039550721645355,
      "logits/chosen": -0.9173828363418579,
      "logits/rejected": -0.8062499761581421,
      "logps/chosen": -0.664257824420929,
      "logps/rejected": -1.6240234375,
      "loss": 0.836,
      "nll_loss": 0.849804699420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06641845405101776,
      "rewards/margins": 0.09599609673023224,
      "rewards/rejected": -0.162353515625,
      "step": 4910
    },
    {
      "epoch": 0.3733495219304902,
      "grad_norm": 2.096457299618778,
      "learning_rate": 1.140531897024402e-06,
      "log_odds_chosen": 1.4734375476837158,
      "log_odds_ratio": -0.4014648497104645,
      "logits/chosen": -0.982226550579071,
      "logits/rejected": -0.8636718988418579,
      "logps/chosen": -0.63330078125,
      "logps/rejected": -1.6238281726837158,
      "loss": 0.8636,
      "nll_loss": 0.770703136920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06337890774011612,
      "rewards/margins": 0.09904785454273224,
      "rewards/rejected": -0.16240234673023224,
      "step": 4920
    },
    {
      "epoch": 0.37410836242221884,
      "grad_norm": 2.625565690686019,
      "learning_rate": 1.13937458379177e-06,
      "log_odds_chosen": 1.322167992591858,
      "log_odds_ratio": -0.45234376192092896,
      "logits/chosen": -0.924609363079071,
      "logits/rejected": -0.819140613079071,
      "logps/chosen": -0.68115234375,
      "logps/rejected": -1.6484375,
      "loss": 0.856,
      "nll_loss": 0.820996105670929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06809081882238388,
      "rewards/margins": 0.09653320163488388,
      "rewards/rejected": -0.16477051377296448,
      "step": 4930
    },
    {
      "epoch": 0.3748672029139475,
      "grad_norm": 2.203218780916155,
      "learning_rate": 1.1382207864489444e-06,
      "log_odds_chosen": 1.2156250476837158,
      "log_odds_ratio": -0.43525391817092896,
      "logits/chosen": -0.9537109136581421,
      "logits/rejected": -0.828320324420929,
      "logps/chosen": -0.624804675579071,
      "logps/rejected": -1.423437476158142,
      "loss": 0.8619,
      "nll_loss": 0.8041015863418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06251220405101776,
      "rewards/margins": 0.07990722358226776,
      "rewards/rejected": -0.14243164658546448,
      "step": 4940
    },
    {
      "epoch": 0.3756260434056761,
      "grad_norm": 2.7540164581142594,
      "learning_rate": 1.1370704872299223e-06,
      "log_odds_chosen": 1.2926757335662842,
      "log_odds_ratio": -0.41874998807907104,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.891406238079071,
      "logps/chosen": -0.6659179925918579,
      "logps/rejected": -1.556640625,
      "loss": 0.8565,
      "nll_loss": 0.809765636920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06658935546875,
      "rewards/margins": 0.08905029296875,
      "rewards/rejected": -0.15556640923023224,
      "step": 4950
    },
    {
      "epoch": 0.37638488389740477,
      "grad_norm": 2.025269380602589,
      "learning_rate": 1.1359236684941295e-06,
      "log_odds_chosen": 1.349511742591858,
      "log_odds_ratio": -0.41899412870407104,
      "logits/chosen": -0.9544922113418579,
      "logits/rejected": -0.8314453363418579,
      "logps/chosen": -0.6807616949081421,
      "logps/rejected": -1.603515625,
      "loss": 0.8806,
      "nll_loss": 0.882617175579071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06804199516773224,
      "rewards/margins": 0.09243164211511612,
      "rewards/rejected": -0.16044922173023224,
      "step": 4960
    },
    {
      "epoch": 0.3771437243891334,
      "grad_norm": 2.2485565467037207,
      "learning_rate": 1.1347803127252839e-06,
      "log_odds_chosen": 1.569238305091858,
      "log_odds_ratio": -0.3885253965854645,
      "logits/chosen": -1.0224609375,
      "logits/rejected": -0.893359363079071,
      "logps/chosen": -0.633496105670929,
      "logps/rejected": -1.7433593273162842,
      "loss": 0.8377,
      "nll_loss": 0.8037109375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06339111179113388,
      "rewards/margins": 0.11101684719324112,
      "rewards/rejected": -0.17434081435203552,
      "step": 4970
    },
    {
      "epoch": 0.37790256488086205,
      "grad_norm": 2.165794505306308,
      "learning_rate": 1.1336404025302715e-06,
      "log_odds_chosen": 1.4425780773162842,
      "log_odds_ratio": -0.40278321504592896,
      "logits/chosen": -0.9632812738418579,
      "logits/rejected": -0.84375,
      "logps/chosen": -0.6806640625,
      "logps/rejected": -1.71484375,
      "loss": 0.8718,
      "nll_loss": 0.8779296875,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06804199516773224,
      "rewards/margins": 0.10341186821460724,
      "rewards/rejected": -0.17160645127296448,
      "step": 4980
    },
    {
      "epoch": 0.3786614053725907,
      "grad_norm": 2.164096377102248,
      "learning_rate": 1.1325039206380352e-06,
      "log_odds_chosen": 1.0779297351837158,
      "log_odds_ratio": -0.47822266817092896,
      "logits/chosen": -0.9638671875,
      "logits/rejected": -0.884570300579071,
      "logps/chosen": -0.641894519329071,
      "logps/rejected": -1.3117187023162842,
      "loss": 0.8722,
      "nll_loss": 0.8316406011581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06416015326976776,
      "rewards/margins": 0.0670166015625,
      "rewards/rejected": -0.13124999403953552,
      "step": 4990
    },
    {
      "epoch": 0.3794202458643193,
      "grad_norm": 2.4681602129978293,
      "learning_rate": 1.131370849898476e-06,
      "log_odds_chosen": 1.2675902843475342,
      "log_odds_ratio": -0.4202636778354645,
      "logits/chosen": -0.980273425579071,
      "logits/rejected": -0.873046875,
      "logps/chosen": -0.6693359613418579,
      "logps/rejected": -1.5099608898162842,
      "loss": 0.8537,
      "nll_loss": 0.8560546636581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06689453125,
      "rewards/margins": 0.08423157036304474,
      "rewards/rejected": -0.15097656846046448,
      "step": 5000
    },
    {
      "epoch": 0.3794202458643193,
      "eval_log_odds_chosen": 1.0485382080078125,
      "eval_log_odds_ratio": -0.5006263852119446,
      "eval_logits/chosen": -0.8404242396354675,
      "eval_logits/rejected": -0.757588803768158,
      "eval_logps/chosen": -0.7668778300285339,
      "eval_logps/rejected": -1.5307999849319458,
      "eval_loss": 1.1802747249603271,
      "eval_nll_loss": 1.1388089656829834,
      "eval_rewards/accuracies": 0.7127641439437866,
      "eval_rewards/chosen": -0.07668878138065338,
      "eval_rewards/margins": 0.07640629261732101,
      "eval_rewards/rejected": -0.15308316051959991,
      "eval_runtime": 1322.2772,
      "eval_samples_per_second": 71.001,
      "eval_steps_per_second": 1.109,
      "step": 5000
    },
    {
      "epoch": 0.38017908635604797,
      "grad_norm": 1.9817618134624426,
      "learning_rate": 1.130241173281366e-06,
      "log_odds_chosen": 1.358007788658142,
      "log_odds_ratio": -0.4292968809604645,
      "logits/chosen": -0.988476574420929,
      "logits/rejected": -0.8824218511581421,
      "logps/chosen": -0.626171886920929,
      "logps/rejected": -1.5431640148162842,
      "loss": 0.8706,
      "nll_loss": 0.7916015386581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06263427436351776,
      "rewards/margins": 0.09177245944738388,
      "rewards/rejected": -0.15444335341453552,
      "step": 5010
    },
    {
      "epoch": 0.3809379268477766,
      "grad_norm": 2.0887243367857313,
      "learning_rate": 1.1291148738752732e-06,
      "log_odds_chosen": 1.359375,
      "log_odds_ratio": -0.43115234375,
      "logits/chosen": -0.962890625,
      "logits/rejected": -0.859375,
      "logps/chosen": -0.662304699420929,
      "logps/rejected": -1.6101562976837158,
      "loss": 0.8595,
      "nll_loss": 0.8021484613418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06625976413488388,
      "rewards/margins": 0.09468384087085724,
      "rewards/rejected": -0.16091307997703552,
      "step": 5020
    },
    {
      "epoch": 0.38169676733950525,
      "grad_norm": 2.3221585209409144,
      "learning_rate": 1.1279919348864981e-06,
      "log_odds_chosen": 1.262109398841858,
      "log_odds_ratio": -0.4559570252895355,
      "logits/chosen": -0.986328125,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -0.684863269329071,
      "logps/rejected": -1.5759766101837158,
      "loss": 0.8661,
      "nll_loss": 0.780468761920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06840820610523224,
      "rewards/margins": 0.08903808891773224,
      "rewards/rejected": -0.15749511122703552,
      "step": 5030
    },
    {
      "epoch": 0.3824556078312339,
      "grad_norm": 2.4615333295489923,
      "learning_rate": 1.126872339638022e-06,
      "log_odds_chosen": 1.5412108898162842,
      "log_odds_ratio": -0.3860839903354645,
      "logits/chosen": -0.995898425579071,
      "logits/rejected": -0.8568359613418579,
      "logps/chosen": -0.6590820550918579,
      "logps/rejected": -1.711328148841858,
      "loss": 0.8506,
      "nll_loss": 0.8023437261581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06596679985523224,
      "rewards/margins": 0.10516357421875,
      "rewards/rejected": -0.17094726860523224,
      "step": 5040
    },
    {
      "epoch": 0.38321444832296253,
      "grad_norm": 2.2087176039301113,
      "learning_rate": 1.1257560715684668e-06,
      "log_odds_chosen": 1.3230469226837158,
      "log_odds_ratio": -0.3921875059604645,
      "logits/chosen": -0.953906238079071,
      "logits/rejected": -0.861523449420929,
      "logps/chosen": -0.6551758050918579,
      "logps/rejected": -1.529882788658142,
      "loss": 0.8582,
      "nll_loss": 0.8031250238418579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06558837741613388,
      "rewards/margins": 0.08754272758960724,
      "rewards/rejected": -0.15302734076976776,
      "step": 5050
    },
    {
      "epoch": 0.38397328881469117,
      "grad_norm": 2.57913366088348,
      "learning_rate": 1.1246431142310665e-06,
      "log_odds_chosen": 1.323144555091858,
      "log_odds_ratio": -0.45341795682907104,
      "logits/chosen": -0.992382824420929,
      "logits/rejected": -0.9234374761581421,
      "logps/chosen": -0.674511730670929,
      "logps/rejected": -1.611718773841858,
      "loss": 0.8318,
      "nll_loss": 0.787890613079071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06746826320886612,
      "rewards/margins": 0.09369506686925888,
      "rewards/rejected": -0.16118164360523224,
      "step": 5060
    },
    {
      "epoch": 0.3847321293064198,
      "grad_norm": 2.867178072685699,
      "learning_rate": 1.1235334512926484e-06,
      "log_odds_chosen": 1.4119141101837158,
      "log_odds_ratio": -0.44267576932907104,
      "logits/chosen": -0.979296863079071,
      "logits/rejected": -0.896484375,
      "logps/chosen": -0.694531261920929,
      "logps/rejected": -1.689843773841858,
      "loss": 0.8531,
      "nll_loss": 0.8343750238418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06943359225988388,
      "rewards/margins": 0.09953002631664276,
      "rewards/rejected": -0.16896972060203552,
      "step": 5070
    },
    {
      "epoch": 0.38549096979814845,
      "grad_norm": 2.4584468576457086,
      "learning_rate": 1.1224270665326274e-06,
      "log_odds_chosen": 1.535546898841858,
      "log_odds_ratio": -0.3919433653354645,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.886523425579071,
      "logps/chosen": -0.650390625,
      "logps/rejected": -1.6845703125,
      "loss": 0.8546,
      "nll_loss": 0.7789062261581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.064971923828125,
      "rewards/margins": 0.10341797024011612,
      "rewards/rejected": -0.16840820014476776,
      "step": 5080
    },
    {
      "epoch": 0.3862498102898771,
      "grad_norm": 2.286984065038291,
      "learning_rate": 1.12132394384201e-06,
      "log_odds_chosen": 1.4249999523162842,
      "log_odds_ratio": -0.3865966796875,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.8822265863418579,
      "logps/chosen": -0.658984363079071,
      "logps/rejected": -1.6378905773162842,
      "loss": 0.8675,
      "nll_loss": 0.87841796875,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06590576469898224,
      "rewards/margins": 0.09788818657398224,
      "rewards/rejected": -0.16389159858226776,
      "step": 5090
    },
    {
      "epoch": 0.38700865078160573,
      "grad_norm": 2.296460359770309,
      "learning_rate": 1.1202240672224076e-06,
      "log_odds_chosen": 1.221777319908142,
      "log_odds_ratio": -0.4361816346645355,
      "logits/chosen": -1.0087890625,
      "logits/rejected": -0.908203125,
      "logps/chosen": -0.698046863079071,
      "logps/rejected": -1.5496094226837158,
      "loss": 0.8566,
      "nll_loss": 0.845703125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06977538764476776,
      "rewards/margins": 0.08516845852136612,
      "rewards/rejected": -0.15493163466453552,
      "step": 5100
    },
    {
      "epoch": 0.3877674912733343,
      "grad_norm": 2.539802054931241,
      "learning_rate": 1.1191274207850654e-06,
      "log_odds_chosen": 1.351953148841858,
      "log_odds_ratio": -0.3960937559604645,
      "logits/chosen": -0.970898449420929,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -0.65478515625,
      "logps/rejected": -1.5607421398162842,
      "loss": 0.8639,
      "nll_loss": 0.826367199420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06551513820886612,
      "rewards/margins": 0.09067382663488388,
      "rewards/rejected": -0.15610352158546448,
      "step": 5110
    },
    {
      "epoch": 0.38852633176506296,
      "grad_norm": 1.9322585367595608,
      "learning_rate": 1.1180339887498948e-06,
      "log_odds_chosen": 1.215234398841858,
      "log_odds_ratio": -0.4300781190395355,
      "logits/chosen": -0.966992199420929,
      "logits/rejected": -0.8843749761581421,
      "logps/chosen": -0.6258789300918579,
      "logps/rejected": -1.404687523841858,
      "loss": 0.8496,
      "nll_loss": 0.8500000238418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06259765475988388,
      "rewards/margins": 0.07786254584789276,
      "rewards/rejected": -0.14052733778953552,
      "step": 5120
    },
    {
      "epoch": 0.3892851722567916,
      "grad_norm": 2.1834984733601615,
      "learning_rate": 1.1169437554445213e-06,
      "log_odds_chosen": 1.357666015625,
      "log_odds_ratio": -0.4041503965854645,
      "logits/chosen": -1.0068359375,
      "logits/rejected": -0.9208984375,
      "logps/chosen": -0.665332019329071,
      "logps/rejected": -1.6199219226837158,
      "loss": 0.8493,
      "nll_loss": 0.813281238079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06654052436351776,
      "rewards/margins": 0.09534148871898651,
      "rewards/rejected": -0.16188964247703552,
      "step": 5130
    },
    {
      "epoch": 0.39004401274852024,
      "grad_norm": 2.2641930579324483,
      "learning_rate": 1.1158567053033413e-06,
      "log_odds_chosen": 1.4916503429412842,
      "log_odds_ratio": -0.39594727754592896,
      "logits/chosen": -0.956835925579071,
      "logits/rejected": -0.86328125,
      "logps/chosen": -0.6742187738418579,
      "logps/rejected": -1.7208983898162842,
      "loss": 0.8622,
      "nll_loss": 0.7606445550918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06740722805261612,
      "rewards/margins": 0.10485382378101349,
      "rewards/rejected": -0.17209473252296448,
      "step": 5140
    },
    {
      "epoch": 0.3908028532402489,
      "grad_norm": 2.757681266959225,
      "learning_rate": 1.1147728228665882e-06,
      "log_odds_chosen": 1.619531273841858,
      "log_odds_ratio": -0.3351074159145355,
      "logits/chosen": -1.031835913658142,
      "logits/rejected": -0.901562511920929,
      "logps/chosen": -0.58447265625,
      "logps/rejected": -1.67578125,
      "loss": 0.8401,
      "nll_loss": 0.7567383050918579,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05852050706744194,
      "rewards/margins": 0.10905762016773224,
      "rewards/rejected": -0.16745606064796448,
      "step": 5150
    },
    {
      "epoch": 0.3915616937319775,
      "grad_norm": 2.1808989598699657,
      "learning_rate": 1.1136920927794092e-06,
      "log_odds_chosen": 1.546289086341858,
      "log_odds_ratio": -0.3707519471645355,
      "logits/chosen": -0.956250011920929,
      "logits/rejected": -0.8568359613418579,
      "logps/chosen": -0.635546863079071,
      "logps/rejected": -1.701757788658142,
      "loss": 0.8418,
      "nll_loss": 0.804882824420929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06348876655101776,
      "rewards/margins": 0.10668029636144638,
      "rewards/rejected": -0.17021484673023224,
      "step": 5160
    },
    {
      "epoch": 0.39232053422370616,
      "grad_norm": 3.036900593990881,
      "learning_rate": 1.1126144997909508e-06,
      "log_odds_chosen": 1.5876953601837158,
      "log_odds_ratio": -0.40166014432907104,
      "logits/chosen": -1.02734375,
      "logits/rejected": -0.907421886920929,
      "logps/chosen": -0.640429675579071,
      "logps/rejected": -1.763281226158142,
      "loss": 0.8346,
      "nll_loss": 0.832226574420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06407471001148224,
      "rewards/margins": 0.11224365234375,
      "rewards/rejected": -0.17631836235523224,
      "step": 5170
    },
    {
      "epoch": 0.3930793747154348,
      "grad_norm": 2.437818281436921,
      "learning_rate": 1.1115400287534568e-06,
      "log_odds_chosen": 1.438574194908142,
      "log_odds_ratio": -0.40478515625,
      "logits/chosen": -0.925976574420929,
      "logits/rejected": -0.8587890863418579,
      "logps/chosen": -0.6607421636581421,
      "logps/rejected": -1.7107422351837158,
      "loss": 0.8706,
      "nll_loss": 0.8599609136581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06611327826976776,
      "rewards/margins": 0.10502929985523224,
      "rewards/rejected": -0.171142578125,
      "step": 5180
    },
    {
      "epoch": 0.39383821520716344,
      "grad_norm": 2.3360733001189313,
      "learning_rate": 1.110468664621372e-06,
      "log_odds_chosen": 1.1808593273162842,
      "log_odds_ratio": -0.41166990995407104,
      "logits/chosen": -0.9156249761581421,
      "logits/rejected": -0.8248046636581421,
      "logps/chosen": -0.662890613079071,
      "logps/rejected": -1.4318358898162842,
      "loss": 0.8526,
      "nll_loss": 0.8492187261581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06633301079273224,
      "rewards/margins": 0.0768638625741005,
      "rewards/rejected": -0.1431884765625,
      "step": 5190
    },
    {
      "epoch": 0.3945970556988921,
      "grad_norm": 2.241729585825133,
      "learning_rate": 1.1094003924504583e-06,
      "log_odds_chosen": 1.510156273841858,
      "log_odds_ratio": -0.3543945252895355,
      "logits/chosen": -0.962695300579071,
      "logits/rejected": -0.832812488079071,
      "logps/chosen": -0.6650390625,
      "logps/rejected": -1.6951172351837158,
      "loss": 0.8439,
      "nll_loss": 0.7823241949081421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.0665283203125,
      "rewards/margins": 0.10301513969898224,
      "rewards/rejected": -0.16958007216453552,
      "step": 5200
    },
    {
      "epoch": 0.3953558961906207,
      "grad_norm": 2.351667347631347,
      "learning_rate": 1.1083351973969191e-06,
      "log_odds_chosen": 1.4570801258087158,
      "log_odds_ratio": -0.397216796875,
      "logits/chosen": -0.972460925579071,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.64404296875,
      "logps/rejected": -1.672265648841858,
      "loss": 0.826,
      "nll_loss": 0.791015625,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06439208984375,
      "rewards/margins": 0.10289306938648224,
      "rewards/rejected": -0.1673583984375,
      "step": 5210
    },
    {
      "epoch": 0.39611473668234937,
      "grad_norm": 2.390270358393584,
      "learning_rate": 1.107273064716533e-06,
      "log_odds_chosen": 1.48388671875,
      "log_odds_ratio": -0.3770507872104645,
      "logits/chosen": -0.9349609613418579,
      "logits/rejected": -0.855273425579071,
      "logps/chosen": -0.67138671875,
      "logps/rejected": -1.67578125,
      "loss": 0.8374,
      "nll_loss": 0.7816406488418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06712646782398224,
      "rewards/margins": 0.10045166313648224,
      "rewards/rejected": -0.1676025390625,
      "step": 5220
    },
    {
      "epoch": 0.396873577174078,
      "grad_norm": 2.554077173558235,
      "learning_rate": 1.1062139797637962e-06,
      "log_odds_chosen": 1.140527367591858,
      "log_odds_ratio": -0.48124998807907104,
      "logits/chosen": -0.966015636920929,
      "logits/rejected": -0.837890625,
      "logps/chosen": -0.660839855670929,
      "logps/rejected": -1.4445312023162842,
      "loss": 0.8497,
      "nll_loss": 0.812695324420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06608887016773224,
      "rewards/margins": 0.07825317233800888,
      "rewards/rejected": -0.14423827826976776,
      "step": 5230
    },
    {
      "epoch": 0.39763241766580665,
      "grad_norm": 2.455294602481025,
      "learning_rate": 1.1051579279910751e-06,
      "log_odds_chosen": 1.237548828125,
      "log_odds_ratio": -0.4476074278354645,
      "logits/chosen": -0.9859374761581421,
      "logits/rejected": -0.86328125,
      "logps/chosen": -0.6650390625,
      "logps/rejected": -1.5402343273162842,
      "loss": 0.8415,
      "nll_loss": 0.80126953125,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0665283203125,
      "rewards/margins": 0.08743591606616974,
      "rewards/rejected": -0.15388183295726776,
      "step": 5240
    },
    {
      "epoch": 0.3983912581575353,
      "grad_norm": 2.0142702638341405,
      "learning_rate": 1.1041048949477667e-06,
      "log_odds_chosen": 1.4738280773162842,
      "log_odds_ratio": -0.421630859375,
      "logits/chosen": -0.9613281488418579,
      "logits/rejected": -0.848925769329071,
      "logps/chosen": -0.6913086175918579,
      "logps/rejected": -1.7062499523162842,
      "loss": 0.8602,
      "nll_loss": 0.864453136920929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06912841647863388,
      "rewards/margins": 0.10150146484375,
      "rewards/rejected": -0.17070312798023224,
      "step": 5250
    },
    {
      "epoch": 0.39915009864926393,
      "grad_norm": 2.5825414735616294,
      "learning_rate": 1.1030548662794673e-06,
      "log_odds_chosen": 1.609765648841858,
      "log_odds_ratio": -0.3705078065395355,
      "logits/chosen": -0.9417968988418579,
      "logits/rejected": -0.849609375,
      "logps/chosen": -0.632617175579071,
      "logps/rejected": -1.749609351158142,
      "loss": 0.8321,
      "nll_loss": 0.8251953125,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06322021782398224,
      "rewards/margins": 0.11190185695886612,
      "rewards/rejected": -0.17499999701976776,
      "step": 5260
    },
    {
      "epoch": 0.39990893914099257,
      "grad_norm": 2.3958073804549107,
      "learning_rate": 1.102007827727152e-06,
      "log_odds_chosen": 1.192968726158142,
      "log_odds_ratio": -0.461181640625,
      "logits/chosen": -0.9130859375,
      "logits/rejected": -0.821484386920929,
      "logps/chosen": -0.652148425579071,
      "logps/rejected": -1.4617187976837158,
      "loss": 0.8474,
      "nll_loss": 0.787304699420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06522216647863388,
      "rewards/margins": 0.08099365234375,
      "rewards/rejected": -0.14616699516773224,
      "step": 5270
    },
    {
      "epoch": 0.4006677796327212,
      "grad_norm": 2.2048879056759705,
      "learning_rate": 1.1009637651263607e-06,
      "log_odds_chosen": 1.418554663658142,
      "log_odds_ratio": -0.4303222596645355,
      "logits/chosen": -0.9595702886581421,
      "logits/rejected": -0.8666015863418579,
      "logps/chosen": -0.721972644329071,
      "logps/rejected": -1.7548828125,
      "loss": 0.8406,
      "nll_loss": 0.835742175579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07216797024011612,
      "rewards/margins": 0.103271484375,
      "rewards/rejected": -0.17561034858226776,
      "step": 5280
    },
    {
      "epoch": 0.40142662012444985,
      "grad_norm": 2.3839196143077186,
      "learning_rate": 1.0999226644063927e-06,
      "log_odds_chosen": 1.5810546875,
      "log_odds_ratio": -0.41425782442092896,
      "logits/chosen": -0.9673827886581421,
      "logits/rejected": -0.876757800579071,
      "logps/chosen": -0.678417980670929,
      "logps/rejected": -1.767187476158142,
      "loss": 0.8503,
      "nll_loss": 0.7894531488418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06787109375,
      "rewards/margins": 0.10891113430261612,
      "rewards/rejected": -0.17683105170726776,
      "step": 5290
    },
    {
      "epoch": 0.4021854606161785,
      "grad_norm": 2.5262186182515385,
      "learning_rate": 1.0988845115895123e-06,
      "log_odds_chosen": 1.275976538658142,
      "log_odds_ratio": -0.41874998807907104,
      "logits/chosen": -0.9677734375,
      "logits/rejected": -0.8544921875,
      "logps/chosen": -0.6668945550918579,
      "logps/rejected": -1.554101586341858,
      "loss": 0.8674,
      "nll_loss": 0.8570312261581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06662597507238388,
      "rewards/margins": 0.08864746242761612,
      "rewards/rejected": -0.15534667670726776,
      "step": 5300
    },
    {
      "epoch": 0.40294430110790713,
      "grad_norm": 2.451035118002672,
      "learning_rate": 1.0978492927901574e-06,
      "log_odds_chosen": 1.3566405773162842,
      "log_odds_ratio": -0.39653319120407104,
      "logits/chosen": -0.9673827886581421,
      "logits/rejected": -0.87109375,
      "logps/chosen": -0.642382800579071,
      "logps/rejected": -1.5675780773162842,
      "loss": 0.8693,
      "nll_loss": 0.810742199420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06419678032398224,
      "rewards/margins": 0.09238281100988388,
      "rewards/rejected": -0.15671387314796448,
      "step": 5310
    },
    {
      "epoch": 0.4037031415996358,
      "grad_norm": 2.236444996769135,
      "learning_rate": 1.0968169942141634e-06,
      "log_odds_chosen": 1.2775390148162842,
      "log_odds_ratio": -0.4251953065395355,
      "logits/chosen": -0.96484375,
      "logits/rejected": -0.8701171875,
      "logps/chosen": -0.684765636920929,
      "logps/rejected": -1.572656273841858,
      "loss": 0.8418,
      "nll_loss": 0.7955077886581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06851806491613388,
      "rewards/margins": 0.08876953274011612,
      "rewards/rejected": -0.15729980170726776,
      "step": 5320
    },
    {
      "epoch": 0.4044619820913644,
      "grad_norm": 2.0767066319513634,
      "learning_rate": 1.0957876021579874e-06,
      "log_odds_chosen": 1.3289062976837158,
      "log_odds_ratio": -0.42060548067092896,
      "logits/chosen": -0.9751952886581421,
      "logits/rejected": -0.880078136920929,
      "logps/chosen": -0.6249023675918579,
      "logps/rejected": -1.520117163658142,
      "loss": 0.8516,
      "nll_loss": 0.802929699420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06256103515625,
      "rewards/margins": 0.08948364108800888,
      "rewards/rejected": -0.15207520127296448,
      "step": 5330
    },
    {
      "epoch": 0.40522082258309305,
      "grad_norm": 1.9338751696332936,
      "learning_rate": 1.0947611030079466e-06,
      "log_odds_chosen": 1.228759765625,
      "log_odds_ratio": -0.4403320252895355,
      "logits/chosen": -0.993359386920929,
      "logits/rejected": -0.8968750238418579,
      "logps/chosen": -0.6595703363418579,
      "logps/rejected": -1.5138671398162842,
      "loss": 0.8461,
      "nll_loss": 0.8099609613418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06594238430261612,
      "rewards/margins": 0.08539581298828125,
      "rewards/rejected": -0.15126952528953552,
      "step": 5340
    },
    {
      "epoch": 0.4059796630748217,
      "grad_norm": 2.567395908968091,
      "learning_rate": 1.0937374832394612e-06,
      "log_odds_chosen": 1.610937476158142,
      "log_odds_ratio": -0.37236326932907104,
      "logits/chosen": -0.942187488079071,
      "logits/rejected": -0.8658202886581421,
      "logps/chosen": -0.7108398675918579,
      "logps/rejected": -1.8488280773162842,
      "loss": 0.848,
      "nll_loss": 0.817675769329071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07108154147863388,
      "rewards/margins": 0.11353759467601776,
      "rewards/rejected": -0.18464355170726776,
      "step": 5350
    },
    {
      "epoch": 0.40673850356655034,
      "grad_norm": 2.95176322615593,
      "learning_rate": 1.092716729416306e-06,
      "log_odds_chosen": 1.4933593273162842,
      "log_odds_ratio": -0.399169921875,
      "logits/chosen": -0.9750000238418579,
      "logits/rejected": -0.8755859136581421,
      "logps/chosen": -0.6742187738418579,
      "logps/rejected": -1.7248046398162842,
      "loss": 0.8302,
      "nll_loss": 0.746289074420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06743164360523224,
      "rewards/margins": 0.10502319037914276,
      "rewards/rejected": -0.17238768935203552,
      "step": 5360
    },
    {
      "epoch": 0.407497344058279,
      "grad_norm": 2.219558950331525,
      "learning_rate": 1.0916988281898703e-06,
      "log_odds_chosen": 1.5137939453125,
      "log_odds_ratio": -0.43994140625,
      "logits/chosen": -0.9740234613418579,
      "logits/rejected": -0.837109386920929,
      "logps/chosen": -0.635449230670929,
      "logps/rejected": -1.719335913658142,
      "loss": 0.8543,
      "nll_loss": 0.811328113079071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06352539360523224,
      "rewards/margins": 0.10838699340820312,
      "rewards/rejected": -0.17202147841453552,
      "step": 5370
    },
    {
      "epoch": 0.40825618455000756,
      "grad_norm": 2.3407798939658866,
      "learning_rate": 1.0906837662984237e-06,
      "log_odds_chosen": 1.3464844226837158,
      "log_odds_ratio": -0.41777342557907104,
      "logits/chosen": -1.0304687023162842,
      "logits/rejected": -0.9146484136581421,
      "logps/chosen": -0.683886706829071,
      "logps/rejected": -1.640234351158142,
      "loss": 0.8303,
      "nll_loss": 0.7933593988418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06838379055261612,
      "rewards/margins": 0.09565429389476776,
      "rewards/rejected": -0.16396483778953552,
      "step": 5380
    },
    {
      "epoch": 0.4090150250417362,
      "grad_norm": 2.3276245562723905,
      "learning_rate": 1.089671530566391e-06,
      "log_odds_chosen": 1.0404052734375,
      "log_odds_ratio": -0.5146484375,
      "logits/chosen": -0.977343738079071,
      "logits/rejected": -0.8521484136581421,
      "logps/chosen": -0.726269543170929,
      "logps/rejected": -1.465234398841858,
      "loss": 0.8452,
      "nll_loss": 0.826953113079071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07255859673023224,
      "rewards/margins": 0.07400665432214737,
      "rewards/rejected": -0.14645996689796448,
      "step": 5390
    },
    {
      "epoch": 0.40977386553346484,
      "grad_norm": 2.0981773535508603,
      "learning_rate": 1.0886621079036346e-06,
      "log_odds_chosen": 1.3523437976837158,
      "log_odds_ratio": -0.4051757752895355,
      "logits/chosen": -0.9755859375,
      "logits/rejected": -0.8773437738418579,
      "logps/chosen": -0.68310546875,
      "logps/rejected": -1.6330077648162842,
      "loss": 0.8439,
      "nll_loss": 0.788281261920929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06827392429113388,
      "rewards/margins": 0.09505005180835724,
      "rewards/rejected": -0.1632080078125,
      "step": 5400
    },
    {
      "epoch": 0.4105327060251935,
      "grad_norm": 2.450305687307705,
      "learning_rate": 1.0876554853047417e-06,
      "log_odds_chosen": 1.370703101158142,
      "log_odds_ratio": -0.39628905057907104,
      "logits/chosen": -0.9892578125,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.6451171636581421,
      "logps/rejected": -1.584570288658142,
      "loss": 0.8483,
      "nll_loss": 0.826953113079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06447754055261612,
      "rewards/margins": 0.09417724609375,
      "rewards/rejected": -0.15861816704273224,
      "step": 5410
    },
    {
      "epoch": 0.4112915465169221,
      "grad_norm": 2.2184697279127996,
      "learning_rate": 1.0866516498483225e-06,
      "log_odds_chosen": 1.385351538658142,
      "log_odds_ratio": -0.39497071504592896,
      "logits/chosen": -0.948046863079071,
      "logits/rejected": -0.892382800579071,
      "logps/chosen": -0.658984363079071,
      "logps/rejected": -1.6228516101837158,
      "loss": 0.8518,
      "nll_loss": 0.7818359136581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06588134914636612,
      "rewards/margins": 0.09637451171875,
      "rewards/rejected": -0.1622314453125,
      "step": 5420
    },
    {
      "epoch": 0.41205038700865076,
      "grad_norm": 2.229851033600332,
      "learning_rate": 1.0856505886963116e-06,
      "log_odds_chosen": 1.2664062976837158,
      "log_odds_ratio": -0.45341795682907104,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.886523425579071,
      "logps/chosen": -0.6996093988418579,
      "logps/rejected": -1.5519530773162842,
      "loss": 0.8465,
      "nll_loss": 0.783398449420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07000732421875,
      "rewards/margins": 0.08521118015050888,
      "rewards/rejected": -0.15524902939796448,
      "step": 5430
    },
    {
      "epoch": 0.4128092275003794,
      "grad_norm": 2.713777376309095,
      "learning_rate": 1.0846522890932808e-06,
      "log_odds_chosen": 1.4894530773162842,
      "log_odds_ratio": -0.3945556581020355,
      "logits/chosen": -0.9527343511581421,
      "logits/rejected": -0.816601574420929,
      "logps/chosen": -0.6708984375,
      "logps/rejected": -1.730859398841858,
      "loss": 0.847,
      "nll_loss": 0.8365234136581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06711425632238388,
      "rewards/margins": 0.10590209811925888,
      "rewards/rejected": -0.1729736328125,
      "step": 5440
    },
    {
      "epoch": 0.41356806799210805,
      "grad_norm": 2.323029731035624,
      "learning_rate": 1.0836567383657542e-06,
      "log_odds_chosen": 1.615234375,
      "log_odds_ratio": -0.34257811307907104,
      "logits/chosen": -0.948046863079071,
      "logits/rejected": -0.833984375,
      "logps/chosen": -0.602832019329071,
      "logps/rejected": -1.6828124523162842,
      "loss": 0.8152,
      "nll_loss": 0.7875000238418579,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06026611477136612,
      "rewards/margins": 0.10798339545726776,
      "rewards/rejected": -0.16828612983226776,
      "step": 5450
    },
    {
      "epoch": 0.4143269084838367,
      "grad_norm": 2.65671515741868,
      "learning_rate": 1.0826639239215334e-06,
      "log_odds_chosen": 1.4382812976837158,
      "log_odds_ratio": -0.39111328125,
      "logits/chosen": -0.9517577886581421,
      "logits/rejected": -0.8427734375,
      "logps/chosen": -0.6689453125,
      "logps/rejected": -1.661523461341858,
      "loss": 0.8571,
      "nll_loss": 0.7779296636581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06689453125,
      "rewards/margins": 0.0992431640625,
      "rewards/rejected": -0.16616210341453552,
      "step": 5460
    },
    {
      "epoch": 0.4150857489755653,
      "grad_norm": 2.232804110723734,
      "learning_rate": 1.0816738332490292e-06,
      "log_odds_chosen": 1.450585961341858,
      "log_odds_ratio": -0.37749022245407104,
      "logits/chosen": -0.953906238079071,
      "logits/rejected": -0.822070300579071,
      "logps/chosen": -0.6556640863418579,
      "logps/rejected": -1.6687500476837158,
      "loss": 0.8243,
      "nll_loss": 0.7876952886581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06551513820886612,
      "rewards/margins": 0.101318359375,
      "rewards/rejected": -0.16689452528953552,
      "step": 5470
    },
    {
      "epoch": 0.41584458946729397,
      "grad_norm": 8.267218952899983,
      "learning_rate": 1.0806864539165982e-06,
      "log_odds_chosen": 1.243749976158142,
      "log_odds_ratio": -0.47314453125,
      "logits/chosen": -0.961132824420929,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.652050793170929,
      "logps/rejected": -1.510156273841858,
      "loss": 0.87,
      "nll_loss": 0.815136730670929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06517334282398224,
      "rewards/margins": 0.0858154296875,
      "rewards/rejected": -0.15097656846046448,
      "step": 5480
    },
    {
      "epoch": 0.4166034299590226,
      "grad_norm": 2.291167751112026,
      "learning_rate": 1.0797017735718878e-06,
      "log_odds_chosen": 1.391455054283142,
      "log_odds_ratio": -0.42216795682907104,
      "logits/chosen": -0.925000011920929,
      "logits/rejected": -0.825390636920929,
      "logps/chosen": -0.68359375,
      "logps/rejected": -1.667382836341858,
      "loss": 0.8574,
      "nll_loss": 0.798828125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06829833984375,
      "rewards/margins": 0.09837951511144638,
      "rewards/rejected": -0.166748046875,
      "step": 5490
    },
    {
      "epoch": 0.41736227045075125,
      "grad_norm": 2.827968129506355,
      "learning_rate": 1.0787197799411874e-06,
      "log_odds_chosen": 1.484716773033142,
      "log_odds_ratio": -0.376708984375,
      "logits/chosen": -0.966992199420929,
      "logits/rejected": -0.884960949420929,
      "logps/chosen": -0.6605468988418579,
      "logps/rejected": -1.694921851158142,
      "loss": 0.8561,
      "nll_loss": 0.7572265863418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06602783501148224,
      "rewards/margins": 0.10356750339269638,
      "rewards/rejected": -0.169677734375,
      "step": 5500
    },
    {
      "epoch": 0.4181211109424799,
      "grad_norm": 3.141675362699091,
      "learning_rate": 1.0777404608287846e-06,
      "log_odds_chosen": 1.377343773841858,
      "log_odds_ratio": -0.38818359375,
      "logits/chosen": -0.9957031011581421,
      "logits/rejected": -0.888476550579071,
      "logps/chosen": -0.611621081829071,
      "logps/rejected": -1.5291016101837158,
      "loss": 0.8614,
      "nll_loss": 0.754199206829071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0611572265625,
      "rewards/margins": 0.09177245944738388,
      "rewards/rejected": -0.15290527045726776,
      "step": 5510
    },
    {
      "epoch": 0.41887995143420853,
      "grad_norm": 2.2349534850143846,
      "learning_rate": 1.0767638041163309e-06,
      "log_odds_chosen": 1.5626952648162842,
      "log_odds_ratio": -0.3944335877895355,
      "logits/chosen": -0.927929699420929,
      "logits/rejected": -0.852734386920929,
      "logps/chosen": -0.6341797113418579,
      "logps/rejected": -1.6746094226837158,
      "loss": 0.8157,
      "nll_loss": 0.7822265625,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06340332329273224,
      "rewards/margins": 0.10406494140625,
      "rewards/rejected": -0.16755370795726776,
      "step": 5520
    },
    {
      "epoch": 0.41963879192593717,
      "grad_norm": 2.493059537614275,
      "learning_rate": 1.0757897977622107e-06,
      "log_odds_chosen": 1.316992163658142,
      "log_odds_ratio": -0.4212402403354645,
      "logits/chosen": -0.9488281011581421,
      "logits/rejected": -0.837109386920929,
      "logps/chosen": -0.635546863079071,
      "logps/rejected": -1.512109398841858,
      "loss": 0.8426,
      "nll_loss": 0.796875,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06357421725988388,
      "rewards/margins": 0.08762817084789276,
      "rewards/rejected": -0.15117187798023224,
      "step": 5530
    },
    {
      "epoch": 0.4203976324176658,
      "grad_norm": 2.32022031502086,
      "learning_rate": 1.074818429800918e-06,
      "log_odds_chosen": 1.42578125,
      "log_odds_ratio": -0.41650390625,
      "logits/chosen": -0.9419921636581421,
      "logits/rejected": -0.835156261920929,
      "logps/chosen": -0.667285144329071,
      "logps/rejected": -1.6515624523162842,
      "loss": 0.8429,
      "nll_loss": 0.8384765386581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06676025688648224,
      "rewards/margins": 0.09844970703125,
      "rewards/rejected": -0.1651611328125,
      "step": 5540
    },
    {
      "epoch": 0.42115647290939445,
      "grad_norm": 2.4153737572601583,
      "learning_rate": 1.073849688342439e-06,
      "log_odds_chosen": 1.49853515625,
      "log_odds_ratio": -0.369140625,
      "logits/chosen": -0.978515625,
      "logits/rejected": -0.8662109375,
      "logps/chosen": -0.6898437738418579,
      "logps/rejected": -1.750390648841858,
      "loss": 0.8571,
      "nll_loss": 0.773632824420929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06903076171875,
      "rewards/margins": 0.1060791015625,
      "rewards/rejected": -0.17514649033546448,
      "step": 5550
    },
    {
      "epoch": 0.4219153134011231,
      "grad_norm": 2.1402853638188075,
      "learning_rate": 1.07288356157164e-06,
      "log_odds_chosen": 1.2134277820587158,
      "log_odds_ratio": -0.4410644471645355,
      "logits/chosen": -0.933398425579071,
      "logits/rejected": -0.8648437261581421,
      "logps/chosen": -0.661328136920929,
      "logps/rejected": -1.4675781726837158,
      "loss": 0.8474,
      "nll_loss": 0.796679675579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06613769382238388,
      "rewards/margins": 0.08057556301355362,
      "rewards/rejected": -0.14677734673023224,
      "step": 5560
    },
    {
      "epoch": 0.42267415389285173,
      "grad_norm": 2.1941589650936417,
      "learning_rate": 1.0719200377476648e-06,
      "log_odds_chosen": 1.100195288658142,
      "log_odds_ratio": -0.4546875059604645,
      "logits/chosen": -0.9546874761581421,
      "logits/rejected": -0.8228515386581421,
      "logps/chosen": -0.67333984375,
      "logps/rejected": -1.409570336341858,
      "loss": 0.852,
      "nll_loss": 0.7875000238418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06734619289636612,
      "rewards/margins": 0.07369232177734375,
      "rewards/rejected": -0.14104004204273224,
      "step": 5570
    },
    {
      "epoch": 0.4234329943845804,
      "grad_norm": 2.284806569576881,
      "learning_rate": 1.0709591052033317e-06,
      "log_odds_chosen": 1.2415039539337158,
      "log_odds_ratio": -0.4207519590854645,
      "logits/chosen": -0.925000011920929,
      "logits/rejected": -0.7876952886581421,
      "logps/chosen": -0.6590820550918579,
      "logps/rejected": -1.4812500476837158,
      "loss": 0.8272,
      "nll_loss": 0.8017578125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06593017280101776,
      "rewards/margins": 0.082275390625,
      "rewards/rejected": -0.14814452826976776,
      "step": 5580
    },
    {
      "epoch": 0.424191834876309,
      "grad_norm": 2.6557510914633222,
      "learning_rate": 1.0700007523445435e-06,
      "log_odds_chosen": 1.594335913658142,
      "log_odds_ratio": -0.37128907442092896,
      "logits/chosen": -0.970703125,
      "logits/rejected": -0.8941406011581421,
      "logps/chosen": -0.6465820074081421,
      "logps/rejected": -1.769921898841858,
      "loss": 0.8534,
      "nll_loss": 0.8013671636581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06468506157398224,
      "rewards/margins": 0.11247558891773224,
      "rewards/rejected": -0.17707519233226776,
      "step": 5590
    },
    {
      "epoch": 0.42495067536803766,
      "grad_norm": 2.291232417510915,
      "learning_rate": 1.0690449676496976e-06,
      "log_odds_chosen": 1.4367187023162842,
      "log_odds_ratio": -0.3921875059604645,
      "logits/chosen": -0.951367199420929,
      "logits/rejected": -0.8363281488418579,
      "logps/chosen": -0.637988269329071,
      "logps/rejected": -1.618749976158142,
      "loss": 0.823,
      "nll_loss": 0.746777355670929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0638427734375,
      "rewards/margins": 0.097991943359375,
      "rewards/rejected": -0.1617431640625,
      "step": 5600
    },
    {
      "epoch": 0.4257095158597663,
      "grad_norm": 3.146253786769614,
      "learning_rate": 1.0680917396691054e-06,
      "log_odds_chosen": 1.478784203529358,
      "log_odds_ratio": -0.40986329317092896,
      "logits/chosen": -0.9580078125,
      "logits/rejected": -0.8285156488418579,
      "logps/chosen": -0.6039062738418579,
      "logps/rejected": -1.641210913658142,
      "loss": 0.8523,
      "nll_loss": 0.8119140863418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06040038913488388,
      "rewards/margins": 0.10360107570886612,
      "rewards/rejected": -0.16403809189796448,
      "step": 5610
    },
    {
      "epoch": 0.42646835635149494,
      "grad_norm": 2.4839220165258062,
      "learning_rate": 1.0671410570244164e-06,
      "log_odds_chosen": 1.4324219226837158,
      "log_odds_ratio": -0.40996092557907104,
      "logits/chosen": -0.8941406011581421,
      "logits/rejected": -0.8042968511581421,
      "logps/chosen": -0.6836913824081421,
      "logps/rejected": -1.702734351158142,
      "loss": 0.8397,
      "nll_loss": 0.8467773199081421,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06838379055261612,
      "rewards/margins": 0.10189209133386612,
      "rewards/rejected": -0.17045898735523224,
      "step": 5620
    },
    {
      "epoch": 0.4272271968432236,
      "grad_norm": 2.3854633770220968,
      "learning_rate": 1.0661929084080466e-06,
      "log_odds_chosen": 1.4597656726837158,
      "log_odds_ratio": -0.40644532442092896,
      "logits/chosen": -0.9105468988418579,
      "logits/rejected": -0.852734386920929,
      "logps/chosen": -0.6836913824081421,
      "logps/rejected": -1.7257812023162842,
      "loss": 0.8363,
      "nll_loss": 0.798828125,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06840820610523224,
      "rewards/margins": 0.10405273735523224,
      "rewards/rejected": -0.17253418266773224,
      "step": 5630
    },
    {
      "epoch": 0.4279860373349522,
      "grad_norm": 2.618874726406942,
      "learning_rate": 1.0652472825826149e-06,
      "log_odds_chosen": 1.5724608898162842,
      "log_odds_ratio": -0.3803955018520355,
      "logits/chosen": -0.9599609375,
      "logits/rejected": -0.854687511920929,
      "logps/chosen": -0.616503894329071,
      "logps/rejected": -1.720703125,
      "loss": 0.8354,
      "nll_loss": 0.762499988079071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06162109225988388,
      "rewards/margins": 0.11040039360523224,
      "rewards/rejected": -0.17207030951976776,
      "step": 5640
    },
    {
      "epoch": 0.42874487782668086,
      "grad_norm": 2.9664040671175362,
      "learning_rate": 1.0643041683803828e-06,
      "log_odds_chosen": 1.3095703125,
      "log_odds_ratio": -0.4390625059604645,
      "logits/chosen": -0.9208984375,
      "logits/rejected": -0.861328125,
      "logps/chosen": -0.63720703125,
      "logps/rejected": -1.5322265625,
      "loss": 0.8451,
      "nll_loss": 0.792773425579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06373290717601776,
      "rewards/margins": 0.08945312350988388,
      "rewards/rejected": -0.15314941108226776,
      "step": 5650
    },
    {
      "epoch": 0.42950371831840944,
      "grad_norm": 2.192033753033841,
      "learning_rate": 1.063363554702701e-06,
      "log_odds_chosen": 1.289282202720642,
      "log_odds_ratio": -0.4020019471645355,
      "logits/chosen": -0.9697265625,
      "logits/rejected": -0.859570324420929,
      "logps/chosen": -0.639941394329071,
      "logps/rejected": -1.49609375,
      "loss": 0.8336,
      "nll_loss": 0.766894519329071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06397704780101776,
      "rewards/margins": 0.08570556342601776,
      "rewards/rejected": -0.149658203125,
      "step": 5660
    },
    {
      "epoch": 0.4302625588101381,
      "grad_norm": 2.3086066108515944,
      "learning_rate": 1.0624254305194609e-06,
      "log_odds_chosen": 1.379296898841858,
      "log_odds_ratio": -0.4307617247104645,
      "logits/chosen": -0.9359375238418579,
      "logits/rejected": -0.810546875,
      "logps/chosen": -0.6513671875,
      "logps/rejected": -1.6160156726837158,
      "loss": 0.8531,
      "nll_loss": 0.800585925579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06517334282398224,
      "rewards/margins": 0.09659423679113388,
      "rewards/rejected": -0.16169433295726776,
      "step": 5670
    },
    {
      "epoch": 0.4310213993018667,
      "grad_norm": 2.064949707230211,
      "learning_rate": 1.0614897848685505e-06,
      "log_odds_chosen": 1.7146484851837158,
      "log_odds_ratio": -0.36894530057907104,
      "logits/chosen": -0.953320324420929,
      "logits/rejected": -0.887890636920929,
      "logps/chosen": -0.686816394329071,
      "logps/rejected": -1.9425780773162842,
      "loss": 0.8205,
      "nll_loss": 0.8172851800918579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06867675483226776,
      "rewards/margins": 0.12564697861671448,
      "rewards/rejected": -0.19418945908546448,
      "step": 5680
    },
    {
      "epoch": 0.43178023979359537,
      "grad_norm": 2.651212937316763,
      "learning_rate": 1.0605566068553173e-06,
      "log_odds_chosen": 1.5587890148162842,
      "log_odds_ratio": -0.38862305879592896,
      "logits/chosen": -0.927734375,
      "logits/rejected": -0.8412109613418579,
      "logps/chosen": -0.6478515863418579,
      "logps/rejected": -1.733984351158142,
      "loss": 0.8239,
      "nll_loss": 0.7591797113418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06478271633386612,
      "rewards/margins": 0.10858154296875,
      "rewards/rejected": -0.17329101264476776,
      "step": 5690
    },
    {
      "epoch": 0.432539080285324,
      "grad_norm": 2.743960824799714,
      "learning_rate": 1.0596258856520351e-06,
      "log_odds_chosen": 1.4599609375,
      "log_odds_ratio": -0.414306640625,
      "logits/chosen": -0.9634765386581421,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.645214855670929,
      "logps/rejected": -1.668359398841858,
      "loss": 0.804,
      "nll_loss": 0.743359386920929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06452636420726776,
      "rewards/margins": 0.10244140774011612,
      "rewards/rejected": -0.1669921875,
      "step": 5700
    },
    {
      "epoch": 0.43329792077705265,
      "grad_norm": 2.460484764762945,
      "learning_rate": 1.0586976104973764e-06,
      "log_odds_chosen": 1.5458984375,
      "log_odds_ratio": -0.373291015625,
      "logits/chosen": -0.994335949420929,
      "logits/rejected": -0.8560546636581421,
      "logps/chosen": -0.619335949420929,
      "logps/rejected": -1.717382788658142,
      "loss": 0.823,
      "nll_loss": 0.7523437738418579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06191406399011612,
      "rewards/margins": 0.10981445014476776,
      "rewards/rejected": -0.17172852158546448,
      "step": 5710
    },
    {
      "epoch": 0.4340567612687813,
      "grad_norm": 2.194192369645987,
      "learning_rate": 1.05777177069589e-06,
      "log_odds_chosen": 1.3621094226837158,
      "log_odds_ratio": -0.41132813692092896,
      "logits/chosen": -0.971484363079071,
      "logits/rejected": -0.8695312738418579,
      "logps/chosen": -0.650195300579071,
      "logps/rejected": -1.5964844226837158,
      "loss": 0.8224,
      "nll_loss": 0.8091796636581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06502685695886612,
      "rewards/margins": 0.09459228813648224,
      "rewards/rejected": -0.15961913764476776,
      "step": 5720
    },
    {
      "epoch": 0.43481560176050993,
      "grad_norm": 2.3534569396435163,
      "learning_rate": 1.0568483556174834e-06,
      "log_odds_chosen": 1.36181640625,
      "log_odds_ratio": -0.37382811307907104,
      "logits/chosen": -0.916796863079071,
      "logits/rejected": -0.820117175579071,
      "logps/chosen": -0.611035168170929,
      "logps/rejected": -1.4787108898162842,
      "loss": 0.8343,
      "nll_loss": 0.780566394329071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06107177585363388,
      "rewards/margins": 0.08672485500574112,
      "rewards/rejected": -0.14780274033546448,
      "step": 5730
    },
    {
      "epoch": 0.43557444225223857,
      "grad_norm": 2.5341264324790886,
      "learning_rate": 1.055927354696909e-06,
      "log_odds_chosen": 1.527734398841858,
      "log_odds_ratio": -0.40424805879592896,
      "logits/chosen": -0.9203125238418579,
      "logits/rejected": -0.8306640386581421,
      "logps/chosen": -0.6845703125,
      "logps/rejected": -1.749609351158142,
      "loss": 0.8199,
      "nll_loss": 0.8037109375,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06845702975988388,
      "rewards/margins": 0.10635986179113388,
      "rewards/rejected": -0.17485351860523224,
      "step": 5740
    },
    {
      "epoch": 0.4363332827439672,
      "grad_norm": 2.5400347892221515,
      "learning_rate": 1.0550087574332592e-06,
      "log_odds_chosen": 1.577734351158142,
      "log_odds_ratio": -0.3924804627895355,
      "logits/chosen": -0.9095703363418579,
      "logits/rejected": -0.828125,
      "logps/chosen": -0.6397460699081421,
      "logps/rejected": -1.7228515148162842,
      "loss": 0.837,
      "nll_loss": 0.8037109375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06395263969898224,
      "rewards/margins": 0.10823974758386612,
      "rewards/rejected": -0.17226561903953552,
      "step": 5750
    },
    {
      "epoch": 0.43709212323569585,
      "grad_norm": 2.591562426503811,
      "learning_rate": 1.0540925533894598e-06,
      "log_odds_chosen": 1.265649437904358,
      "log_odds_ratio": -0.42963868379592896,
      "logits/chosen": -0.980273425579071,
      "logits/rejected": -0.8765624761581421,
      "logps/chosen": -0.668164074420929,
      "logps/rejected": -1.508203148841858,
      "loss": 0.8458,
      "nll_loss": 0.8193359375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0667724609375,
      "rewards/margins": 0.08410415798425674,
      "rewards/rejected": -0.15097656846046448,
      "step": 5760
    },
    {
      "epoch": 0.4378509637274245,
      "grad_norm": 2.451814486576127,
      "learning_rate": 1.053178732191775e-06,
      "log_odds_chosen": 1.4832031726837158,
      "log_odds_ratio": -0.3822265565395355,
      "logits/chosen": -0.9476562738418579,
      "logits/rejected": -0.836132824420929,
      "logps/chosen": -0.662109375,
      "logps/rejected": -1.67578125,
      "loss": 0.8261,
      "nll_loss": 0.8193359375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.066162109375,
      "rewards/margins": 0.10148315131664276,
      "rewards/rejected": -0.1676025390625,
      "step": 5770
    },
    {
      "epoch": 0.43860980421915313,
      "grad_norm": 2.3419467810243377,
      "learning_rate": 1.0522672835293127e-06,
      "log_odds_chosen": 1.386328101158142,
      "log_odds_ratio": -0.41874998807907104,
      "logits/chosen": -0.9398437738418579,
      "logits/rejected": -0.8355468511581421,
      "logps/chosen": -0.6703125238418579,
      "logps/rejected": -1.632226586341858,
      "loss": 0.8353,
      "nll_loss": 0.8271484375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06697998195886612,
      "rewards/margins": 0.09612426906824112,
      "rewards/rejected": -0.16318359971046448,
      "step": 5780
    },
    {
      "epoch": 0.4393686447108818,
      "grad_norm": 2.515846614172588,
      "learning_rate": 1.0513581971535365e-06,
      "log_odds_chosen": 1.548193335533142,
      "log_odds_ratio": -0.3951171934604645,
      "logits/chosen": -0.9283202886581421,
      "logits/rejected": -0.8158203363418579,
      "logps/chosen": -0.65869140625,
      "logps/rejected": -1.749609351158142,
      "loss": 0.8329,
      "nll_loss": 0.794921875,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06594238430261612,
      "rewards/margins": 0.10897521674633026,
      "rewards/rejected": -0.175048828125,
      "step": 5790
    },
    {
      "epoch": 0.4401274852026104,
      "grad_norm": 2.156252240229296,
      "learning_rate": 1.0504514628777803e-06,
      "log_odds_chosen": 1.318872094154358,
      "log_odds_ratio": -0.42827147245407104,
      "logits/chosen": -0.949414074420929,
      "logits/rejected": -0.7906249761581421,
      "logps/chosen": -0.649707019329071,
      "logps/rejected": -1.53515625,
      "loss": 0.8335,
      "nll_loss": 0.8270508050918579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06499023735523224,
      "rewards/margins": 0.088653564453125,
      "rewards/rejected": -0.15366211533546448,
      "step": 5800
    },
    {
      "epoch": 0.44088632569433905,
      "grad_norm": 2.4786714654919746,
      "learning_rate": 1.0495470705767713e-06,
      "log_odds_chosen": 1.6251952648162842,
      "log_odds_ratio": -0.3487792909145355,
      "logits/chosen": -0.997851550579071,
      "logits/rejected": -0.8681640625,
      "logps/chosen": -0.653515636920929,
      "logps/rejected": -1.7648437023162842,
      "loss": 0.8313,
      "nll_loss": 0.7705078125,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06534423679113388,
      "rewards/margins": 0.11101074516773224,
      "rewards/rejected": -0.17646484076976776,
      "step": 5810
    },
    {
      "epoch": 0.4416451661860677,
      "grad_norm": 2.4398295811022597,
      "learning_rate": 1.0486450101861527e-06,
      "log_odds_chosen": 1.4000976085662842,
      "log_odds_ratio": -0.426513671875,
      "logits/chosen": -0.969531238079071,
      "logits/rejected": -0.8570312261581421,
      "logps/chosen": -0.6625000238418579,
      "logps/rejected": -1.6564452648162842,
      "loss": 0.8354,
      "nll_loss": 0.797070324420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06633301079273224,
      "rewards/margins": 0.09940185397863388,
      "rewards/rejected": -0.16567382216453552,
      "step": 5820
    },
    {
      "epoch": 0.44240400667779634,
      "grad_norm": 2.282514156983795,
      "learning_rate": 1.0477452717020143e-06,
      "log_odds_chosen": 1.3572266101837158,
      "log_odds_ratio": -0.44023436307907104,
      "logits/chosen": -0.9673827886581421,
      "logits/rejected": -0.871289074420929,
      "logps/chosen": -0.682324230670929,
      "logps/rejected": -1.592187523841858,
      "loss": 0.8384,
      "nll_loss": 0.8333984613418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06824950873851776,
      "rewards/margins": 0.09095458686351776,
      "rewards/rejected": -0.1591796875,
      "step": 5830
    },
    {
      "epoch": 0.443162847169525,
      "grad_norm": 7.328556810853576,
      "learning_rate": 1.0468478451804272e-06,
      "log_odds_chosen": 1.4638671875,
      "log_odds_ratio": -0.39252930879592896,
      "logits/chosen": -0.9339843988418579,
      "logits/rejected": -0.7919921875,
      "logps/chosen": -0.6328125,
      "logps/rejected": -1.668554663658142,
      "loss": 0.8096,
      "nll_loss": 0.7823241949081421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06330566108226776,
      "rewards/margins": 0.1036376953125,
      "rewards/rejected": -0.16689452528953552,
      "step": 5840
    },
    {
      "epoch": 0.4439216876612536,
      "grad_norm": 2.336676814883782,
      "learning_rate": 1.0459527207369814e-06,
      "log_odds_chosen": 1.488867163658142,
      "log_odds_ratio": -0.40791016817092896,
      "logits/chosen": -0.8779296875,
      "logits/rejected": -0.8070312738418579,
      "logps/chosen": -0.623242199420929,
      "logps/rejected": -1.6306641101837158,
      "loss": 0.8108,
      "nll_loss": 0.71484375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06234131008386612,
      "rewards/margins": 0.10065917670726776,
      "rewards/rejected": -0.16298827528953552,
      "step": 5850
    },
    {
      "epoch": 0.44468052815298226,
      "grad_norm": 2.724913929783163,
      "learning_rate": 1.0450598885463281e-06,
      "log_odds_chosen": 1.1145508289337158,
      "log_odds_ratio": -0.46943360567092896,
      "logits/chosen": -0.929492175579071,
      "logits/rejected": -0.821093738079071,
      "logps/chosen": -0.66357421875,
      "logps/rejected": -1.383203148841858,
      "loss": 0.8195,
      "nll_loss": 0.754101574420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06639404594898224,
      "rewards/margins": 0.07197265326976776,
      "rewards/rejected": -0.13837890326976776,
      "step": 5860
    },
    {
      "epoch": 0.4454393686447109,
      "grad_norm": 2.3791532169873104,
      "learning_rate": 1.0441693388417282e-06,
      "log_odds_chosen": 1.5329101085662842,
      "log_odds_ratio": -0.41850584745407104,
      "logits/chosen": -0.930468738079071,
      "logits/rejected": -0.8050781488418579,
      "logps/chosen": -0.6905273199081421,
      "logps/rejected": -1.765234351158142,
      "loss": 0.8159,
      "nll_loss": 0.76953125,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06901855766773224,
      "rewards/margins": 0.10758056491613388,
      "rewards/rejected": -0.17673340439796448,
      "step": 5870
    },
    {
      "epoch": 0.44619820913643954,
      "grad_norm": 2.5555748035487507,
      "learning_rate": 1.0432810619146023e-06,
      "log_odds_chosen": 1.203222632408142,
      "log_odds_ratio": -0.46269530057907104,
      "logits/chosen": -0.9208984375,
      "logits/rejected": -0.8460937738418579,
      "logps/chosen": -0.67431640625,
      "logps/rejected": -1.519921898841858,
      "loss": 0.8314,
      "nll_loss": 0.8060547113418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06743164360523224,
      "rewards/margins": 0.08454589545726776,
      "rewards/rejected": -0.15202637016773224,
      "step": 5880
    },
    {
      "epoch": 0.4469570496281682,
      "grad_norm": 2.1543997326364472,
      "learning_rate": 1.042395048114086e-06,
      "log_odds_chosen": 1.38671875,
      "log_odds_ratio": -0.39311522245407104,
      "logits/chosen": -0.891796886920929,
      "logits/rejected": -0.8037109375,
      "logps/chosen": -0.644238293170929,
      "logps/rejected": -1.60546875,
      "loss": 0.8316,
      "nll_loss": 0.776562511920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06442870944738388,
      "rewards/margins": 0.09611816704273224,
      "rewards/rejected": -0.16044922173023224,
      "step": 5890
    },
    {
      "epoch": 0.4477158901198968,
      "grad_norm": 2.609080080996203,
      "learning_rate": 1.041511287846591e-06,
      "log_odds_chosen": 1.197656273841858,
      "log_odds_ratio": -0.4347167909145355,
      "logits/chosen": -0.9453125,
      "logits/rejected": -0.857617199420929,
      "logps/chosen": -0.6744140386581421,
      "logps/rejected": -1.483984351158142,
      "loss": 0.8346,
      "nll_loss": 0.81640625,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06744384765625,
      "rewards/margins": 0.08093567192554474,
      "rewards/rejected": -0.14841309189796448,
      "step": 5900
    },
    {
      "epoch": 0.44847473061162546,
      "grad_norm": 2.4744863652963613,
      "learning_rate": 1.0406297715753674e-06,
      "log_odds_chosen": 1.3464844226837158,
      "log_odds_ratio": -0.40361326932907104,
      "logits/chosen": -0.946093738079071,
      "logits/rejected": -0.834765613079071,
      "logps/chosen": -0.664843738079071,
      "logps/rejected": -1.5847656726837158,
      "loss": 0.8265,
      "nll_loss": 0.7710937261581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06647948920726776,
      "rewards/margins": 0.092041015625,
      "rewards/rejected": -0.1585693359375,
      "step": 5910
    },
    {
      "epoch": 0.4492335711033541,
      "grad_norm": 2.6354067523710953,
      "learning_rate": 1.0397504898200726e-06,
      "log_odds_chosen": 1.32861328125,
      "log_odds_ratio": -0.43876951932907104,
      "logits/chosen": -0.9593750238418579,
      "logits/rejected": -0.8824218511581421,
      "logps/chosen": -0.685351550579071,
      "logps/rejected": -1.609765648841858,
      "loss": 0.8354,
      "nll_loss": 0.803515613079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06851806491613388,
      "rewards/margins": 0.09244994819164276,
      "rewards/rejected": -0.1611328125,
      "step": 5920
    },
    {
      "epoch": 0.4499924115950827,
      "grad_norm": 2.749019246142902,
      "learning_rate": 1.0388734331563415e-06,
      "log_odds_chosen": 1.34130859375,
      "log_odds_ratio": -0.43291014432907104,
      "logits/chosen": -0.8763672113418579,
      "logits/rejected": -0.7671874761581421,
      "logps/chosen": -0.6939452886581421,
      "logps/rejected": -1.6042969226837158,
      "loss": 0.8246,
      "nll_loss": 0.7939453125,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06938476860523224,
      "rewards/margins": 0.09111938625574112,
      "rewards/rejected": -0.16062012314796448,
      "step": 5930
    },
    {
      "epoch": 0.4507512520868113,
      "grad_norm": 2.3947291009117047,
      "learning_rate": 1.037998592215364e-06,
      "log_odds_chosen": 1.4220702648162842,
      "log_odds_ratio": -0.4090820252895355,
      "logits/chosen": -0.920703113079071,
      "logits/rejected": -0.822070300579071,
      "logps/chosen": -0.6778320074081421,
      "logps/rejected": -1.656640648841858,
      "loss": 0.8298,
      "nll_loss": 0.80859375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06779785454273224,
      "rewards/margins": 0.09788818657398224,
      "rewards/rejected": -0.16569824516773224,
      "step": 5940
    },
    {
      "epoch": 0.45151009257853997,
      "grad_norm": 2.619990350526729,
      "learning_rate": 1.037125957683463e-06,
      "log_odds_chosen": 1.7410156726837158,
      "log_odds_ratio": -0.347900390625,
      "logits/chosen": -0.9263671636581421,
      "logits/rejected": -0.7876952886581421,
      "logps/chosen": -0.65771484375,
      "logps/rejected": -1.9148437976837158,
      "loss": 0.8214,
      "nll_loss": 0.7865234613418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06575927883386612,
      "rewards/margins": 0.12595215439796448,
      "rewards/rejected": -0.191650390625,
      "step": 5950
    },
    {
      "epoch": 0.4522689330702686,
      "grad_norm": 2.257524214102477,
      "learning_rate": 1.0362555203016794e-06,
      "log_odds_chosen": 1.3131835460662842,
      "log_odds_ratio": -0.448974609375,
      "logits/chosen": -0.959179699420929,
      "logits/rejected": -0.878125011920929,
      "logps/chosen": -0.685351550579071,
      "logps/rejected": -1.6082031726837158,
      "loss": 0.8043,
      "nll_loss": 0.728320300579071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06856689602136612,
      "rewards/margins": 0.09229431301355362,
      "rewards/rejected": -0.1607666015625,
      "step": 5960
    },
    {
      "epoch": 0.45302777356199725,
      "grad_norm": 2.4508968305713505,
      "learning_rate": 1.035387270865359e-06,
      "log_odds_chosen": 1.4972655773162842,
      "log_odds_ratio": -0.3984375,
      "logits/chosen": -0.9361327886581421,
      "logits/rejected": -0.811328113079071,
      "logps/chosen": -0.6631835699081421,
      "logps/rejected": -1.7414062023162842,
      "loss": 0.8044,
      "nll_loss": 0.7499023675918579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06630859524011612,
      "rewards/margins": 0.10785522311925888,
      "rewards/rejected": -0.17412109673023224,
      "step": 5970
    },
    {
      "epoch": 0.4537866140537259,
      "grad_norm": 2.478417935322954,
      "learning_rate": 1.0345212002237434e-06,
      "log_odds_chosen": 1.457421898841858,
      "log_odds_ratio": -0.3685058653354645,
      "logits/chosen": -0.9369140863418579,
      "logits/rejected": -0.799609363079071,
      "logps/chosen": -0.6485351324081421,
      "logps/rejected": -1.6316406726837158,
      "loss": 0.8093,
      "nll_loss": 0.786425769329071,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06480713188648224,
      "rewards/margins": 0.09834595024585724,
      "rewards/rejected": -0.16323241591453552,
      "step": 5980
    },
    {
      "epoch": 0.45454545454545453,
      "grad_norm": 3.1981452692493337,
      "learning_rate": 1.0336572992795644e-06,
      "log_odds_chosen": 1.35888671875,
      "log_odds_ratio": -0.4383789002895355,
      "logits/chosen": -0.9242187738418579,
      "logits/rejected": -0.800976574420929,
      "logps/chosen": -0.684765636920929,
      "logps/rejected": -1.6574218273162842,
      "loss": 0.8072,
      "nll_loss": 0.788281261920929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06853027641773224,
      "rewards/margins": 0.0973663330078125,
      "rewards/rejected": -0.1658935546875,
      "step": 5990
    },
    {
      "epoch": 0.45530429503718317,
      "grad_norm": 2.532856792619757,
      "learning_rate": 1.0327955589886444e-06,
      "log_odds_chosen": 1.615625023841858,
      "log_odds_ratio": -0.37158203125,
      "logits/chosen": -0.939453125,
      "logits/rejected": -0.814453125,
      "logps/chosen": -0.6898437738418579,
      "logps/rejected": -1.8449218273162842,
      "loss": 0.8198,
      "nll_loss": 0.8472656011581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0689697265625,
      "rewards/margins": 0.11558838188648224,
      "rewards/rejected": -0.18459472060203552,
      "step": 6000
    },
    {
      "epoch": 0.4560631355289118,
      "grad_norm": 2.3624301181871474,
      "learning_rate": 1.0319359703594971e-06,
      "log_odds_chosen": 1.311914086341858,
      "log_odds_ratio": -0.4254394471645355,
      "logits/chosen": -0.9566406011581421,
      "logits/rejected": -0.818164050579071,
      "logps/chosen": -0.666015625,
      "logps/rejected": -1.562109351158142,
      "loss": 0.8213,
      "nll_loss": 0.7979491949081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06658935546875,
      "rewards/margins": 0.08953857421875,
      "rewards/rejected": -0.15605469048023224,
      "step": 6010
    },
    {
      "epoch": 0.45682197602064045,
      "grad_norm": 2.4822771588709673,
      "learning_rate": 1.0310785244529341e-06,
      "log_odds_chosen": 1.3820312023162842,
      "log_odds_ratio": -0.4217773377895355,
      "logits/chosen": -0.934765636920929,
      "logits/rejected": -0.8529297113418579,
      "logps/chosen": -0.6429687738418579,
      "logps/rejected": -1.639062523841858,
      "loss": 0.8556,
      "nll_loss": 0.7791992425918579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06430663913488388,
      "rewards/margins": 0.09968872368335724,
      "rewards/rejected": -0.1640625,
      "step": 6020
    },
    {
      "epoch": 0.4575808165123691,
      "grad_norm": 2.3007797386812587,
      "learning_rate": 1.0302232123816746e-06,
      "log_odds_chosen": 1.421289086341858,
      "log_odds_ratio": -0.4197753965854645,
      "logits/chosen": -0.9466797113418579,
      "logits/rejected": -0.802539050579071,
      "logps/chosen": -0.6351562738418579,
      "logps/rejected": -1.6257812976837158,
      "loss": 0.8092,
      "nll_loss": 0.7437499761581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06351318210363388,
      "rewards/margins": 0.09913329780101776,
      "rewards/rejected": -0.16262206435203552,
      "step": 6030
    },
    {
      "epoch": 0.45833965700409773,
      "grad_norm": 3.105742996146195,
      "learning_rate": 1.0293700253099576e-06,
      "log_odds_chosen": 1.6677734851837158,
      "log_odds_ratio": -0.410400390625,
      "logits/chosen": -0.9009765386581421,
      "logits/rejected": -0.808789074420929,
      "logps/chosen": -0.6617187261581421,
      "logps/rejected": -1.882421851158142,
      "loss": 0.823,
      "nll_loss": 0.7767578363418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06622314453125,
      "rewards/margins": 0.12197265774011612,
      "rewards/rejected": -0.18820801377296448,
      "step": 6040
    },
    {
      "epoch": 0.4590984974958264,
      "grad_norm": 2.1351879873106387,
      "learning_rate": 1.02851895445316e-06,
      "log_odds_chosen": 1.170751929283142,
      "log_odds_ratio": -0.4510742127895355,
      "logits/chosen": -0.943359375,
      "logits/rejected": -0.823437511920929,
      "logps/chosen": -0.6552734375,
      "logps/rejected": -1.4640624523162842,
      "loss": 0.8282,
      "nll_loss": 0.7867187261581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06549072265625,
      "rewards/margins": 0.0809326171875,
      "rewards/rejected": -0.14643554389476776,
      "step": 6050
    },
    {
      "epoch": 0.459857337987555,
      "grad_norm": 2.3780810708738698,
      "learning_rate": 1.0276699910774159e-06,
      "log_odds_chosen": 1.5333983898162842,
      "log_odds_ratio": -0.37885743379592896,
      "logits/chosen": -0.939648449420929,
      "logits/rejected": -0.862500011920929,
      "logps/chosen": -0.6524413824081421,
      "logps/rejected": -1.6730468273162842,
      "loss": 0.8051,
      "nll_loss": 0.7705078125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06528320163488388,
      "rewards/margins": 0.10205078125,
      "rewards/rejected": -0.16730956733226776,
      "step": 6060
    },
    {
      "epoch": 0.46061617847928366,
      "grad_norm": 2.687672189857859,
      "learning_rate": 1.0268231264992398e-06,
      "log_odds_chosen": 1.437744140625,
      "log_odds_ratio": -0.3941406309604645,
      "logits/chosen": -0.9205077886581421,
      "logits/rejected": -0.822070300579071,
      "logps/chosen": -0.6304687261581421,
      "logps/rejected": -1.582617163658142,
      "loss": 0.8198,
      "nll_loss": 0.772656261920929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06304931640625,
      "rewards/margins": 0.09518585354089737,
      "rewards/rejected": -0.15826416015625,
      "step": 6070
    },
    {
      "epoch": 0.4613750189710123,
      "grad_norm": 2.170795848214354,
      "learning_rate": 1.0259783520851542e-06,
      "log_odds_chosen": 1.331640601158142,
      "log_odds_ratio": -0.42583006620407104,
      "logits/chosen": -0.970507800579071,
      "logits/rejected": -0.886914074420929,
      "logps/chosen": -0.59814453125,
      "logps/rejected": -1.482421875,
      "loss": 0.816,
      "nll_loss": 0.7505859136581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.05979003757238388,
      "rewards/margins": 0.08842773735523224,
      "rewards/rejected": -0.14826659858226776,
      "step": 6080
    },
    {
      "epoch": 0.46213385946274094,
      "grad_norm": 2.665480296823221,
      "learning_rate": 1.0251356592513193e-06,
      "log_odds_chosen": 1.791015625,
      "log_odds_ratio": -0.35053712129592896,
      "logits/chosen": -0.967578113079071,
      "logits/rejected": -0.824023425579071,
      "logps/chosen": -0.6451171636581421,
      "logps/rejected": -1.951562523841858,
      "loss": 0.7965,
      "nll_loss": 0.797656238079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06446532905101776,
      "rewards/margins": 0.13068847358226776,
      "rewards/rejected": -0.195068359375,
      "step": 6090
    },
    {
      "epoch": 0.4628926999544696,
      "grad_norm": 2.5087426355227778,
      "learning_rate": 1.0242950394631678e-06,
      "log_odds_chosen": 1.780664086341858,
      "log_odds_ratio": -0.3534179627895355,
      "logits/chosen": -0.8794921636581421,
      "logits/rejected": -0.821093738079071,
      "logps/chosen": -0.711132824420929,
      "logps/rejected": -1.9728515148162842,
      "loss": 0.8093,
      "nll_loss": 0.7499023675918579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.07117919623851776,
      "rewards/margins": 0.12614135444164276,
      "rewards/rejected": -0.19716796278953552,
      "step": 6100
    },
    {
      "epoch": 0.4636515404461982,
      "grad_norm": 2.4831413934098805,
      "learning_rate": 1.0234564842350404e-06,
      "log_odds_chosen": 1.2590820789337158,
      "log_odds_ratio": -0.440185546875,
      "logits/chosen": -0.947070300579071,
      "logits/rejected": -0.8208984136581421,
      "logps/chosen": -0.673828125,
      "logps/rejected": -1.556249976158142,
      "loss": 0.8492,
      "nll_loss": 0.8207031488418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06745605170726776,
      "rewards/margins": 0.08823852241039276,
      "rewards/rejected": -0.15573731064796448,
      "step": 6110
    },
    {
      "epoch": 0.46441038093792686,
      "grad_norm": 2.653542006002146,
      "learning_rate": 1.0226199851298272e-06,
      "log_odds_chosen": 1.540502905845642,
      "log_odds_ratio": -0.43144530057907104,
      "logits/chosen": -0.9339843988418579,
      "logits/rejected": -0.827929675579071,
      "logps/chosen": -0.706250011920929,
      "logps/rejected": -1.783789038658142,
      "loss": 0.7819,
      "nll_loss": 0.736132800579071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07059326022863388,
      "rewards/margins": 0.10786743462085724,
      "rewards/rejected": -0.17832031846046448,
      "step": 6120
    },
    {
      "epoch": 0.4651692214296555,
      "grad_norm": 2.1548415042723716,
      "learning_rate": 1.0217855337586106e-06,
      "log_odds_chosen": 1.2821533679962158,
      "log_odds_ratio": -0.42268067598342896,
      "logits/chosen": -0.9935547113418579,
      "logits/rejected": -0.866992175579071,
      "logps/chosen": -0.6748046875,
      "logps/rejected": -1.5068359375,
      "loss": 0.8128,
      "nll_loss": 0.7559570074081421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06744384765625,
      "rewards/margins": 0.08310546725988388,
      "rewards/rejected": -0.15068359673023224,
      "step": 6130
    },
    {
      "epoch": 0.46592806192138414,
      "grad_norm": 2.307825449488109,
      "learning_rate": 1.0209531217803119e-06,
      "log_odds_chosen": 1.13671875,
      "log_odds_ratio": -0.4474121034145355,
      "logits/chosen": -0.973437488079071,
      "logits/rejected": -0.8666015863418579,
      "logps/chosen": -0.6343749761581421,
      "logps/rejected": -1.3761718273162842,
      "loss": 0.8027,
      "nll_loss": 0.7076171636581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06343994289636612,
      "rewards/margins": 0.07416381686925888,
      "rewards/rejected": -0.1376953125,
      "step": 6140
    },
    {
      "epoch": 0.4666869024131128,
      "grad_norm": 2.8040635992428307,
      "learning_rate": 1.0201227409013412e-06,
      "log_odds_chosen": 1.5504882335662842,
      "log_odds_ratio": -0.40693360567092896,
      "logits/chosen": -0.8951171636581421,
      "logits/rejected": -0.7806640863418579,
      "logps/chosen": -0.7017577886581421,
      "logps/rejected": -1.7919921875,
      "loss": 0.8257,
      "nll_loss": 0.786328136920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07017822563648224,
      "rewards/margins": 0.10901489108800888,
      "rewards/rejected": -0.17917481064796448,
      "step": 6150
    },
    {
      "epoch": 0.4674457429048414,
      "grad_norm": 2.2831418752658346,
      "learning_rate": 1.0192943828752509e-06,
      "log_odds_chosen": 1.371679663658142,
      "log_odds_ratio": -0.4322753846645355,
      "logits/chosen": -0.917187511920929,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.641308605670929,
      "logps/rejected": -1.593164086341858,
      "loss": 0.8173,
      "nll_loss": 0.727246105670929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06419678032398224,
      "rewards/margins": 0.09511718899011612,
      "rewards/rejected": -0.15927734971046448,
      "step": 6160
    },
    {
      "epoch": 0.46820458339657006,
      "grad_norm": 2.2630773718540613,
      "learning_rate": 1.0184680395023912e-06,
      "log_odds_chosen": 1.4542968273162842,
      "log_odds_ratio": -0.3931640684604645,
      "logits/chosen": -0.974609375,
      "logits/rejected": -0.87890625,
      "logps/chosen": -0.6356445550918579,
      "logps/rejected": -1.6287109851837158,
      "loss": 0.8154,
      "nll_loss": 0.709765613079071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06364746391773224,
      "rewards/margins": 0.09923706203699112,
      "rewards/rejected": -0.16269531846046448,
      "step": 6170
    },
    {
      "epoch": 0.4689634238882987,
      "grad_norm": 2.6098171460108888,
      "learning_rate": 1.0176437026295688e-06,
      "log_odds_chosen": 1.3972656726837158,
      "log_odds_ratio": -0.4120117127895355,
      "logits/chosen": -0.857421875,
      "logits/rejected": -0.7744140625,
      "logps/chosen": -0.6465820074081421,
      "logps/rejected": -1.575781226158142,
      "loss": 0.8127,
      "nll_loss": 0.8021484613418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06464843451976776,
      "rewards/margins": 0.09293518215417862,
      "rewards/rejected": -0.1575927734375,
      "step": 6180
    },
    {
      "epoch": 0.46972226438002734,
      "grad_norm": 2.6375229216028724,
      "learning_rate": 1.0168213641497094e-06,
      "log_odds_chosen": 1.298437476158142,
      "log_odds_ratio": -0.4339355528354645,
      "logits/chosen": -0.9283202886581421,
      "logits/rejected": -0.8431640863418579,
      "logps/chosen": -0.6709960699081421,
      "logps/rejected": -1.575781226158142,
      "loss": 0.7993,
      "nll_loss": 0.732226550579071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06699218600988388,
      "rewards/margins": 0.09050293266773224,
      "rewards/rejected": -0.15751953423023224,
      "step": 6190
    },
    {
      "epoch": 0.470481104871756,
      "grad_norm": 3.3544143519561898,
      "learning_rate": 1.016001016001524e-06,
      "log_odds_chosen": 1.2083008289337158,
      "log_odds_ratio": -0.4463867247104645,
      "logits/chosen": -0.9027343988418579,
      "logits/rejected": -0.794726550579071,
      "logps/chosen": -0.6932617425918579,
      "logps/rejected": -1.521875023841858,
      "loss": 0.8244,
      "nll_loss": 0.7562500238418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06927490234375,
      "rewards/margins": 0.08277587592601776,
      "rewards/rejected": -0.15219727158546448,
      "step": 6200
    },
    {
      "epoch": 0.47123994536348457,
      "grad_norm": 2.487557177113271,
      "learning_rate": 1.0151826501691747e-06,
      "log_odds_chosen": 1.4021484851837158,
      "log_odds_ratio": -0.3809570372104645,
      "logits/chosen": -0.879687488079071,
      "logits/rejected": -0.802734375,
      "logps/chosen": -0.625683605670929,
      "logps/rejected": -1.536523461341858,
      "loss": 0.7946,
      "nll_loss": 0.7621093988418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06257323920726776,
      "rewards/margins": 0.09113769233226776,
      "rewards/rejected": -0.15361328423023224,
      "step": 6210
    },
    {
      "epoch": 0.4719987858552132,
      "grad_norm": 2.418866186404193,
      "learning_rate": 1.0143662586819475e-06,
      "log_odds_chosen": 1.589453101158142,
      "log_odds_ratio": -0.3534912168979645,
      "logits/chosen": -0.966601550579071,
      "logits/rejected": -0.80078125,
      "logps/chosen": -0.6705077886581421,
      "logps/rejected": -1.813085913658142,
      "loss": 0.8171,
      "nll_loss": 0.7392578125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06708984076976776,
      "rewards/margins": 0.11421509087085724,
      "rewards/rejected": -0.18132324516773224,
      "step": 6220
    },
    {
      "epoch": 0.47275762634694185,
      "grad_norm": 2.381482919906414,
      "learning_rate": 1.0135518336139257e-06,
      "log_odds_chosen": 1.4799315929412842,
      "log_odds_ratio": -0.4188476502895355,
      "logits/chosen": -0.9701172113418579,
      "logits/rejected": -0.8316406011581421,
      "logps/chosen": -0.659375011920929,
      "logps/rejected": -1.708398461341858,
      "loss": 0.8183,
      "nll_loss": 0.7705078125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06590576469898224,
      "rewards/margins": 0.10494537651538849,
      "rewards/rejected": -0.17062988877296448,
      "step": 6230
    },
    {
      "epoch": 0.4735164668386705,
      "grad_norm": 2.3309362231959567,
      "learning_rate": 1.0127393670836666e-06,
      "log_odds_chosen": 1.4430663585662842,
      "log_odds_ratio": -0.40898436307907104,
      "logits/chosen": -0.9566406011581421,
      "logits/rejected": -0.8369140625,
      "logps/chosen": -0.6279296875,
      "logps/rejected": -1.6201171875,
      "loss": 0.8014,
      "nll_loss": 0.7728515863418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06281737983226776,
      "rewards/margins": 0.09914550930261612,
      "rewards/rejected": -0.16201171278953552,
      "step": 6240
    },
    {
      "epoch": 0.47427530733039913,
      "grad_norm": 2.5804457389263047,
      "learning_rate": 1.0119288512538813e-06,
      "log_odds_chosen": 1.963281273841858,
      "log_odds_ratio": -0.35100096464157104,
      "logits/chosen": -0.9498046636581421,
      "logits/rejected": -0.8374999761581421,
      "logps/chosen": -0.628222644329071,
      "logps/rejected": -1.9855468273162842,
      "loss": 0.8272,
      "nll_loss": 0.7630859613418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06281737983226776,
      "rewards/margins": 0.13579101860523224,
      "rewards/rejected": -0.19858399033546448,
      "step": 6250
    },
    {
      "epoch": 0.4750341478221278,
      "grad_norm": 2.3869552633023035,
      "learning_rate": 1.0111202783311173e-06,
      "log_odds_chosen": 1.5701172351837158,
      "log_odds_ratio": -0.3877929747104645,
      "logits/chosen": -0.9560546875,
      "logits/rejected": -0.8099609613418579,
      "logps/chosen": -0.6529296636581421,
      "logps/rejected": -1.7900390625,
      "loss": 0.816,
      "nll_loss": 0.8255859613418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0653076171875,
      "rewards/margins": 0.11373291164636612,
      "rewards/rejected": -0.17902831733226776,
      "step": 6260
    },
    {
      "epoch": 0.4757929883138564,
      "grad_norm": 2.3005688557187263,
      "learning_rate": 1.010313640565443e-06,
      "log_odds_chosen": 1.3795166015625,
      "log_odds_ratio": -0.40571290254592896,
      "logits/chosen": -0.9964843988418579,
      "logits/rejected": -0.866406261920929,
      "logps/chosen": -0.664355456829071,
      "logps/rejected": -1.5773437023162842,
      "loss": 0.8188,
      "nll_loss": 0.745800793170929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06641845405101776,
      "rewards/margins": 0.09126739203929901,
      "rewards/rejected": -0.15764160454273224,
      "step": 6270
    },
    {
      "epoch": 0.47655182880558505,
      "grad_norm": 2.5197130483717154,
      "learning_rate": 1.0095089302501373e-06,
      "log_odds_chosen": 1.5861327648162842,
      "log_odds_ratio": -0.35136717557907104,
      "logits/chosen": -0.9964843988418579,
      "logits/rejected": -0.834765613079071,
      "logps/chosen": -0.6175781488418579,
      "logps/rejected": -1.708593726158142,
      "loss": 0.8025,
      "nll_loss": 0.7518554925918579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06170654296875,
      "rewards/margins": 0.10915527492761612,
      "rewards/rejected": -0.17084960639476776,
      "step": 6280
    },
    {
      "epoch": 0.4773106692973137,
      "grad_norm": 2.078318456287781,
      "learning_rate": 1.0087061397213787e-06,
      "log_odds_chosen": 1.7483398914337158,
      "log_odds_ratio": -0.34101563692092896,
      "logits/chosen": -1.021875023841858,
      "logits/rejected": -0.9097656011581421,
      "logps/chosen": -0.6278320550918579,
      "logps/rejected": -1.861328125,
      "loss": 0.7998,
      "nll_loss": 0.7367187738418579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06281737983226776,
      "rewards/margins": 0.12324218451976776,
      "rewards/rejected": -0.18601074814796448,
      "step": 6290
    },
    {
      "epoch": 0.47806950978904234,
      "grad_norm": 2.5485572768603566,
      "learning_rate": 1.007905261357939e-06,
      "log_odds_chosen": 1.2209961414337158,
      "log_odds_ratio": -0.473388671875,
      "logits/chosen": -0.9798828363418579,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.6318359375,
      "logps/rejected": -1.4533202648162842,
      "loss": 0.8182,
      "nll_loss": 0.750781238079071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06322021782398224,
      "rewards/margins": 0.08211059868335724,
      "rewards/rejected": -0.14533691108226776,
      "step": 6300
    },
    {
      "epoch": 0.478828350280771,
      "grad_norm": 3.300544617659391,
      "learning_rate": 1.0071062875808811e-06,
      "log_odds_chosen": 1.6111328601837158,
      "log_odds_ratio": -0.3555664122104645,
      "logits/chosen": -0.974804699420929,
      "logits/rejected": -0.8792968988418579,
      "logps/chosen": -0.622851550579071,
      "logps/rejected": -1.7429687976837158,
      "loss": 0.8169,
      "nll_loss": 0.763671875,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.062255859375,
      "rewards/margins": 0.11188964545726776,
      "rewards/rejected": -0.17421874403953552,
      "step": 6310
    },
    {
      "epoch": 0.4795871907724996,
      "grad_norm": 2.406231745232268,
      "learning_rate": 1.0063092108532552e-06,
      "log_odds_chosen": 1.4111328125,
      "log_odds_ratio": -0.42919921875,
      "logits/chosen": -0.931445300579071,
      "logits/rejected": -0.8218749761581421,
      "logps/chosen": -0.6539062261581421,
      "logps/rejected": -1.661523461341858,
      "loss": 0.8042,
      "nll_loss": 0.8306640386581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06540527194738388,
      "rewards/margins": 0.10084228217601776,
      "rewards/rejected": -0.16633300483226776,
      "step": 6320
    },
    {
      "epoch": 0.48034603126422826,
      "grad_norm": 2.4878751333253684,
      "learning_rate": 1.005514023679802e-06,
      "log_odds_chosen": 1.4918944835662842,
      "log_odds_ratio": -0.4136718809604645,
      "logits/chosen": -0.966796875,
      "logits/rejected": -0.849414050579071,
      "logps/chosen": -0.675097644329071,
      "logps/rejected": -1.759765625,
      "loss": 0.8069,
      "nll_loss": 0.761523425579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06749267876148224,
      "rewards/margins": 0.10855712741613388,
      "rewards/rejected": -0.17607422173023224,
      "step": 6330
    },
    {
      "epoch": 0.4811048717559569,
      "grad_norm": 2.7882353969131612,
      "learning_rate": 1.0047207186066567e-06,
      "log_odds_chosen": 1.155615210533142,
      "log_odds_ratio": -0.4690918028354645,
      "logits/chosen": -0.9164062738418579,
      "logits/rejected": -0.8017578125,
      "logps/chosen": -0.670214831829071,
      "logps/rejected": -1.4445312023162842,
      "loss": 0.8218,
      "nll_loss": 0.829296886920929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0670166015625,
      "rewards/margins": 0.0775909423828125,
      "rewards/rejected": -0.14462891221046448,
      "step": 6340
    },
    {
      "epoch": 0.48186371224768554,
      "grad_norm": 2.58489950967846,
      "learning_rate": 1.0039292882210538e-06,
      "log_odds_chosen": 1.283203125,
      "log_odds_ratio": -0.40375977754592896,
      "logits/chosen": -0.9554687738418579,
      "logits/rejected": -0.861523449420929,
      "logps/chosen": -0.64013671875,
      "logps/rejected": -1.497460961341858,
      "loss": 0.8111,
      "nll_loss": 0.7305663824081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06403808295726776,
      "rewards/margins": 0.08568725734949112,
      "rewards/rejected": -0.14975586533546448,
      "step": 6350
    },
    {
      "epoch": 0.4826225527394142,
      "grad_norm": 4.037771382759174,
      "learning_rate": 1.0031397251510382e-06,
      "log_odds_chosen": 1.4892578125,
      "log_odds_ratio": -0.3987060487270355,
      "logits/chosen": -0.963085949420929,
      "logits/rejected": -0.8248046636581421,
      "logps/chosen": -0.640917956829071,
      "logps/rejected": -1.655859351158142,
      "loss": 0.8145,
      "nll_loss": 0.7591797113418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06409911811351776,
      "rewards/margins": 0.10162963718175888,
      "rewards/rejected": -0.16569824516773224,
      "step": 6360
    },
    {
      "epoch": 0.4833813932311428,
      "grad_norm": 2.564331922881601,
      "learning_rate": 1.0023520220651762e-06,
      "log_odds_chosen": 1.2976562976837158,
      "log_odds_ratio": -0.42290037870407104,
      "logits/chosen": -0.9613281488418579,
      "logits/rejected": -0.876171886920929,
      "logps/chosen": -0.6351562738418579,
      "logps/rejected": -1.515234351158142,
      "loss": 0.8257,
      "nll_loss": 0.7596679925918579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06354980170726776,
      "rewards/margins": 0.08784790337085724,
      "rewards/rejected": -0.1514892578125,
      "step": 6370
    },
    {
      "epoch": 0.48414023372287146,
      "grad_norm": 2.470819586298383,
      "learning_rate": 1.0015661716722687e-06,
      "log_odds_chosen": 1.3885009288787842,
      "log_odds_ratio": -0.39545899629592896,
      "logits/chosen": -0.9779297113418579,
      "logits/rejected": -0.871874988079071,
      "logps/chosen": -0.6226562261581421,
      "logps/rejected": -1.562109351158142,
      "loss": 0.8322,
      "nll_loss": 0.7376953363418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06226806715130806,
      "rewards/margins": 0.09392700344324112,
      "rewards/rejected": -0.15616455674171448,
      "step": 6380
    },
    {
      "epoch": 0.4848990742146001,
      "grad_norm": 2.3090696991088766,
      "learning_rate": 1.0007821667210687e-06,
      "log_odds_chosen": 1.4011719226837158,
      "log_odds_ratio": -0.41938477754592896,
      "logits/chosen": -0.947070300579071,
      "logits/rejected": -0.853320300579071,
      "logps/chosen": -0.6456054449081421,
      "logps/rejected": -1.5625,
      "loss": 0.7884,
      "nll_loss": 0.7489258050918579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06453857570886612,
      "rewards/margins": 0.09173583984375,
      "rewards/rejected": -0.15617676079273224,
      "step": 6390
    },
    {
      "epoch": 0.48565791470632874,
      "grad_norm": 2.483183154500476,
      "learning_rate": 1e-06,
      "log_odds_chosen": 1.47705078125,
      "log_odds_ratio": -0.42402344942092896,
      "logits/chosen": -0.971875011920929,
      "logits/rejected": -0.8421875238418579,
      "logps/chosen": -0.678906261920929,
      "logps/rejected": -1.7238280773162842,
      "loss": 0.816,
      "nll_loss": 0.767382800579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06785888969898224,
      "rewards/margins": 0.10455322265625,
      "rewards/rejected": -0.17243652045726776,
      "step": 6400
    },
    {
      "epoch": 0.4864167551980574,
      "grad_norm": 3.452449471753027,
      "learning_rate": 9.992196643368784e-07,
      "log_odds_chosen": 1.265527367591858,
      "log_odds_ratio": -0.4453125,
      "logits/chosen": -0.952929675579071,
      "logits/rejected": -0.860546886920929,
      "logps/chosen": -0.7083984613418579,
      "logps/rejected": -1.617578148841858,
      "loss": 0.8028,
      "nll_loss": 0.761914074420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07075195014476776,
      "rewards/margins": 0.09090881049633026,
      "rewards/rejected": -0.16179199516773224,
      "step": 6410
    },
    {
      "epoch": 0.487175595689786,
      "grad_norm": 2.524952848845386,
      "learning_rate": 9.984411525986355e-07,
      "log_odds_chosen": 1.578515648841858,
      "log_odds_ratio": -0.37608641386032104,
      "logits/chosen": -0.9580078125,
      "logits/rejected": -0.8207031488418579,
      "logps/chosen": -0.664355456829071,
      "logps/rejected": -1.790624976158142,
      "loss": 0.8046,
      "nll_loss": 0.752148449420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06643066555261612,
      "rewards/margins": 0.11257324367761612,
      "rewards/rejected": -0.17915038764476776,
      "step": 6420
    },
    {
      "epoch": 0.48793443618151466,
      "grad_norm": 2.8427253252952314,
      "learning_rate": 9.97664457691046e-07,
      "log_odds_chosen": 1.7546875476837158,
      "log_odds_ratio": -0.3233398497104645,
      "logits/chosen": -0.9697265625,
      "logits/rejected": -0.8939453363418579,
      "logps/chosen": -0.6312500238418579,
      "logps/rejected": -1.8351562023162842,
      "loss": 0.7977,
      "nll_loss": 0.74560546875,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06314697116613388,
      "rewards/margins": 0.12041015923023224,
      "rewards/rejected": -0.18354491889476776,
      "step": 6430
    },
    {
      "epoch": 0.4886932766732433,
      "grad_norm": 2.5527692131010222,
      "learning_rate": 9.968895725584535e-07,
      "log_odds_chosen": 1.484960913658142,
      "log_odds_ratio": -0.3592285215854645,
      "logits/chosen": -0.968945324420929,
      "logits/rejected": -0.8492187261581421,
      "logps/chosen": -0.614453136920929,
      "logps/rejected": -1.6326172351837158,
      "loss": 0.8045,
      "nll_loss": 0.768359363079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06147461012005806,
      "rewards/margins": 0.10172118991613388,
      "rewards/rejected": -0.16325683891773224,
      "step": 6440
    },
    {
      "epoch": 0.48945211716497194,
      "grad_norm": 2.815766479517196,
      "learning_rate": 9.961164901835046e-07,
      "log_odds_chosen": 1.7703125476837158,
      "log_odds_ratio": -0.32661134004592896,
      "logits/chosen": -0.9476562738418579,
      "logits/rejected": -0.829296886920929,
      "logps/chosen": -0.61962890625,
      "logps/rejected": -1.865234375,
      "loss": 0.7901,
      "nll_loss": 0.714062511920929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06196289137005806,
      "rewards/margins": 0.12456054985523224,
      "rewards/rejected": -0.1865234375,
      "step": 6450
    },
    {
      "epoch": 0.4902109576567006,
      "grad_norm": 2.292149341426057,
      "learning_rate": 9.95345203586879e-07,
      "log_odds_chosen": 1.44140625,
      "log_odds_ratio": -0.36518555879592896,
      "logits/chosen": -0.9150390625,
      "logits/rejected": -0.779101550579071,
      "logps/chosen": -0.6884765625,
      "logps/rejected": -1.656640648841858,
      "loss": 0.7973,
      "nll_loss": 0.783203125,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06885986030101776,
      "rewards/margins": 0.09699706733226776,
      "rewards/rejected": -0.16594238579273224,
      "step": 6460
    },
    {
      "epoch": 0.4909697981484292,
      "grad_norm": 2.659140977437507,
      "learning_rate": 9.94575705827027e-07,
      "log_odds_chosen": 1.412866234779358,
      "log_odds_ratio": -0.4181152284145355,
      "logits/chosen": -0.9476562738418579,
      "logits/rejected": -0.8441406488418579,
      "logps/chosen": -0.655078113079071,
      "logps/rejected": -1.6271483898162842,
      "loss": 0.8285,
      "nll_loss": 0.7847656011581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06549072265625,
      "rewards/margins": 0.09720764309167862,
      "rewards/rejected": -0.16262206435203552,
      "step": 6470
    },
    {
      "epoch": 0.4917286386401578,
      "grad_norm": 2.85017131233847,
      "learning_rate": 9.938079899999065e-07,
      "log_odds_chosen": 1.4617187976837158,
      "log_odds_ratio": -0.419921875,
      "logits/chosen": -0.943554699420929,
      "logits/rejected": -0.8587890863418579,
      "logps/chosen": -0.647753894329071,
      "logps/rejected": -1.6892578601837158,
      "loss": 0.7926,
      "nll_loss": 0.7222656011581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06478271633386612,
      "rewards/margins": 0.10407714545726776,
      "rewards/rejected": -0.16892090439796448,
      "step": 6480
    },
    {
      "epoch": 0.49248747913188645,
      "grad_norm": 2.9059717253538317,
      "learning_rate": 9.930420492387219e-07,
      "log_odds_chosen": 1.4695312976837158,
      "log_odds_ratio": -0.37470704317092896,
      "logits/chosen": -0.941601574420929,
      "logits/rejected": -0.8447265625,
      "logps/chosen": -0.6338866949081421,
      "logps/rejected": -1.6521484851837158,
      "loss": 0.8109,
      "nll_loss": 0.6924804449081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06337890774011612,
      "rewards/margins": 0.101806640625,
      "rewards/rejected": -0.1651611328125,
      "step": 6490
    },
    {
      "epoch": 0.4932463196236151,
      "grad_norm": 2.691907233454959,
      "learning_rate": 9.922778767136676e-07,
      "log_odds_chosen": 1.397314429283142,
      "log_odds_ratio": -0.4439453184604645,
      "logits/chosen": -0.913281261920929,
      "logits/rejected": -0.836132824420929,
      "logps/chosen": -0.6612304449081421,
      "logps/rejected": -1.5908203125,
      "loss": 0.8096,
      "nll_loss": 0.744433581829071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06613769382238388,
      "rewards/margins": 0.09300384670495987,
      "rewards/rejected": -0.15898437798023224,
      "step": 6500
    },
    {
      "epoch": 0.49400516011534373,
      "grad_norm": 2.5959200764511006,
      "learning_rate": 9.915154656316713e-07,
      "log_odds_chosen": 1.4679687023162842,
      "log_odds_ratio": -0.4046386778354645,
      "logits/chosen": -0.9632812738418579,
      "logits/rejected": -0.8490234613418579,
      "logps/chosen": -0.5916992425918579,
      "logps/rejected": -1.601953148841858,
      "loss": 0.7864,
      "nll_loss": 0.708789050579071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.05917968600988388,
      "rewards/margins": 0.10110016167163849,
      "rewards/rejected": -0.160400390625,
      "step": 6510
    },
    {
      "epoch": 0.4947640006070724,
      "grad_norm": 3.02227371619474,
      "learning_rate": 9.907548092361398e-07,
      "log_odds_chosen": 1.3546874523162842,
      "log_odds_ratio": -0.45415037870407104,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.900195300579071,
      "logps/chosen": -0.67236328125,
      "logps/rejected": -1.646093726158142,
      "loss": 0.7978,
      "nll_loss": 0.7759765386581421,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06719970703125,
      "rewards/margins": 0.09732665866613388,
      "rewards/rejected": -0.1644287109375,
      "step": 6520
    },
    {
      "epoch": 0.495522841098801,
      "grad_norm": 2.5968513939390645,
      "learning_rate": 9.899959008067097e-07,
      "log_odds_chosen": 1.599267601966858,
      "log_odds_ratio": -0.38164061307907104,
      "logits/chosen": -0.9339843988418579,
      "logits/rejected": -0.8763672113418579,
      "logps/chosen": -0.6392577886581421,
      "logps/rejected": -1.762109398841858,
      "loss": 0.7659,
      "nll_loss": 0.692187488079071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06394042819738388,
      "rewards/margins": 0.11228637397289276,
      "rewards/rejected": -0.17622070014476776,
      "step": 6530
    },
    {
      "epoch": 0.49628168159052966,
      "grad_norm": 3.582994884785442,
      "learning_rate": 9.892387336589959e-07,
      "log_odds_chosen": 1.3213622570037842,
      "log_odds_ratio": -0.44208985567092896,
      "logits/chosen": -0.9892578125,
      "logits/rejected": -0.857617199420929,
      "logps/chosen": -0.6802734136581421,
      "logps/rejected": -1.584570288658142,
      "loss": 0.8045,
      "nll_loss": 0.7720702886581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06806640326976776,
      "rewards/margins": 0.090362548828125,
      "rewards/rejected": -0.158447265625,
      "step": 6540
    },
    {
      "epoch": 0.4970405220822583,
      "grad_norm": 2.6768237442220606,
      "learning_rate": 9.884833011443446e-07,
      "log_odds_chosen": 1.5231444835662842,
      "log_odds_ratio": -0.38959962129592896,
      "logits/chosen": -0.970507800579071,
      "logits/rejected": -0.8369140625,
      "logps/chosen": -0.6744140386581421,
      "logps/rejected": -1.736328125,
      "loss": 0.8018,
      "nll_loss": 0.8306640386581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06745605170726776,
      "rewards/margins": 0.10619811713695526,
      "rewards/rejected": -0.173583984375,
      "step": 6550
    },
    {
      "epoch": 0.49779936257398694,
      "grad_norm": 2.5392687392566686,
      "learning_rate": 9.877295966495897e-07,
      "log_odds_chosen": 1.53759765625,
      "log_odds_ratio": -0.36699217557907104,
      "logits/chosen": -0.985156238079071,
      "logits/rejected": -0.8607422113418579,
      "logps/chosen": -0.621874988079071,
      "logps/rejected": -1.692773461341858,
      "loss": 0.7968,
      "nll_loss": 0.708203136920929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06223144382238388,
      "rewards/margins": 0.10717163234949112,
      "rewards/rejected": -0.16948242485523224,
      "step": 6560
    },
    {
      "epoch": 0.4985582030657156,
      "grad_norm": 2.53456526882817,
      "learning_rate": 9.86977613596807e-07,
      "log_odds_chosen": 1.7412109375,
      "log_odds_ratio": -0.38007813692092896,
      "logits/chosen": -1.0089843273162842,
      "logits/rejected": -0.8724609613418579,
      "logps/chosen": -0.658203125,
      "logps/rejected": -1.911718726158142,
      "loss": 0.7795,
      "nll_loss": 0.7701171636581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06582031399011612,
      "rewards/margins": 0.12550048530101776,
      "rewards/rejected": -0.19113770127296448,
      "step": 6570
    },
    {
      "epoch": 0.4993170435574442,
      "grad_norm": 12.048243333811214,
      "learning_rate": 9.862273454430757e-07,
      "log_odds_chosen": 1.641015648841858,
      "log_odds_ratio": -0.3609375059604645,
      "logits/chosen": -0.935742199420929,
      "logits/rejected": -0.828906238079071,
      "logps/chosen": -0.637890636920929,
      "logps/rejected": -1.822265625,
      "loss": 0.7977,
      "nll_loss": 0.7635742425918579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06374511867761612,
      "rewards/margins": 0.11837158352136612,
      "rewards/rejected": -0.18215331435203552,
      "step": 6580
    },
    {
      "epoch": 0.5000758840491729,
      "grad_norm": 2.7112507013505214,
      "learning_rate": 9.85478785680238e-07,
      "log_odds_chosen": 1.4053223133087158,
      "log_odds_ratio": -0.4013671875,
      "logits/chosen": -0.940625011920929,
      "logits/rejected": -0.8060547113418579,
      "logps/chosen": -0.644335925579071,
      "logps/rejected": -1.6544921398162842,
      "loss": 0.8105,
      "nll_loss": 0.7601562738418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.064453125,
      "rewards/margins": 0.10089111328125,
      "rewards/rejected": -0.16542968153953552,
      "step": 6590
    },
    {
      "epoch": 0.5008347245409015,
      "grad_norm": 2.4884711966903086,
      "learning_rate": 9.847319278346618e-07,
      "log_odds_chosen": 1.386816382408142,
      "log_odds_ratio": -0.4229736328125,
      "logits/chosen": -0.931835949420929,
      "logits/rejected": -0.8599609136581421,
      "logps/chosen": -0.6346679925918579,
      "logps/rejected": -1.5812499523162842,
      "loss": 0.8052,
      "nll_loss": 0.7777343988418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06348876655101776,
      "rewards/margins": 0.094635009765625,
      "rewards/rejected": -0.15817871689796448,
      "step": 6600
    },
    {
      "epoch": 0.5015935650326302,
      "grad_norm": 2.4527795187527057,
      "learning_rate": 9.839867654670063e-07,
      "log_odds_chosen": 1.3405272960662842,
      "log_odds_ratio": -0.4638671875,
      "logits/chosen": -0.9136718511581421,
      "logits/rejected": -0.814453125,
      "logps/chosen": -0.6714843511581421,
      "logps/rejected": -1.635351538658142,
      "loss": 0.7844,
      "nll_loss": 0.7486327886581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06706543266773224,
      "rewards/margins": 0.09641418606042862,
      "rewards/rejected": -0.16340331733226776,
      "step": 6610
    },
    {
      "epoch": 0.5023524055243588,
      "grad_norm": 2.5479226445182865,
      "learning_rate": 9.832432921719876e-07,
      "log_odds_chosen": 1.3992798328399658,
      "log_odds_ratio": -0.3910156190395355,
      "logits/chosen": -0.9623047113418579,
      "logits/rejected": -0.8433593511581421,
      "logps/chosen": -0.6591796875,
      "logps/rejected": -1.641015648841858,
      "loss": 0.802,
      "nll_loss": 0.721484363079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06589355319738388,
      "rewards/margins": 0.09831543266773224,
      "rewards/rejected": -0.16408690810203552,
      "step": 6620
    },
    {
      "epoch": 0.5031112460160874,
      "grad_norm": 3.0071032712651777,
      "learning_rate": 9.825015015781493e-07,
      "log_odds_chosen": 1.422216773033142,
      "log_odds_ratio": -0.4296875,
      "logits/chosen": -0.9609375,
      "logits/rejected": -0.818554699420929,
      "logps/chosen": -0.6146484613418579,
      "logps/rejected": -1.611914038658142,
      "loss": 0.7841,
      "nll_loss": 0.7835937738418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0614013671875,
      "rewards/margins": 0.09964065253734589,
      "rewards/rejected": -0.16115722060203552,
      "step": 6630
    },
    {
      "epoch": 0.5038700865078161,
      "grad_norm": 2.5699009966789226,
      "learning_rate": 9.81761387347632e-07,
      "log_odds_chosen": 1.316015601158142,
      "log_odds_ratio": -0.45805662870407104,
      "logits/chosen": -0.978710949420929,
      "logits/rejected": -0.8802734613418579,
      "logps/chosen": -0.6343749761581421,
      "logps/rejected": -1.5720703601837158,
      "loss": 0.7949,
      "nll_loss": 0.7490234375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06349487602710724,
      "rewards/margins": 0.09367676079273224,
      "rewards/rejected": -0.15717773139476776,
      "step": 6640
    },
    {
      "epoch": 0.5046289269995446,
      "grad_norm": 3.156751365213722,
      "learning_rate": 9.810229431759452e-07,
      "log_odds_chosen": 1.333398461341858,
      "log_odds_ratio": -0.4351562559604645,
      "logits/chosen": -0.961132824420929,
      "logits/rejected": -0.8550781011581421,
      "logps/chosen": -0.6317383050918579,
      "logps/rejected": -1.5480468273162842,
      "loss": 0.7878,
      "nll_loss": 0.742968738079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06318359076976776,
      "rewards/margins": 0.09176330268383026,
      "rewards/rejected": -0.15485839545726776,
      "step": 6650
    },
    {
      "epoch": 0.5053877674912733,
      "grad_norm": 2.5289706810829857,
      "learning_rate": 9.802861627917437e-07,
      "log_odds_chosen": 1.618188500404358,
      "log_odds_ratio": -0.3940673768520355,
      "logits/chosen": -1.00390625,
      "logits/rejected": -0.8685547113418579,
      "logps/chosen": -0.682910144329071,
      "logps/rejected": -1.8113281726837158,
      "loss": 0.8064,
      "nll_loss": 0.7100585699081421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06828613579273224,
      "rewards/margins": 0.11281128227710724,
      "rewards/rejected": -0.18125000596046448,
      "step": 6660
    },
    {
      "epoch": 0.5061466079830019,
      "grad_norm": 2.7523920564694198,
      "learning_rate": 9.795510399566016e-07,
      "log_odds_chosen": 1.1799805164337158,
      "log_odds_ratio": -0.4410156309604645,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.80078125,
      "logps/chosen": -0.6509765386581421,
      "logps/rejected": -1.4345703125,
      "loss": 0.7868,
      "nll_loss": 0.713671863079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06510009616613388,
      "rewards/margins": 0.07830810546875,
      "rewards/rejected": -0.14338378608226776,
      "step": 6670
    },
    {
      "epoch": 0.5069054484747306,
      "grad_norm": 2.640847612481933,
      "learning_rate": 9.788175684647926e-07,
      "log_odds_chosen": 1.442773461341858,
      "log_odds_ratio": -0.37919920682907104,
      "logits/chosen": -0.9419921636581421,
      "logits/rejected": -0.830078125,
      "logps/chosen": -0.619824230670929,
      "logps/rejected": -1.601171851158142,
      "loss": 0.788,
      "nll_loss": 0.7431640625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06198730319738388,
      "rewards/margins": 0.09815673530101776,
      "rewards/rejected": -0.16010741889476776,
      "step": 6680
    },
    {
      "epoch": 0.5076642889664592,
      "grad_norm": 2.818182715455829,
      "learning_rate": 9.780857421430687e-07,
      "log_odds_chosen": 1.58154296875,
      "log_odds_ratio": -0.3661132752895355,
      "logits/chosen": -1.0029296875,
      "logits/rejected": -0.8433593511581421,
      "logps/chosen": -0.6211913824081421,
      "logps/rejected": -1.732812523841858,
      "loss": 0.7782,
      "nll_loss": 0.747265636920929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06218261644244194,
      "rewards/margins": 0.11107178032398224,
      "rewards/rejected": -0.17324218153953552,
      "step": 6690
    },
    {
      "epoch": 0.5084231294581879,
      "grad_norm": 2.294944830860249,
      "learning_rate": 9.773555548504417e-07,
      "log_odds_chosen": 1.3458983898162842,
      "log_odds_ratio": -0.40214842557907104,
      "logits/chosen": -0.943164050579071,
      "logits/rejected": -0.8192383050918579,
      "logps/chosen": -0.6675781011581421,
      "logps/rejected": -1.562109351158142,
      "loss": 0.7806,
      "nll_loss": 0.7562500238418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06678466498851776,
      "rewards/margins": 0.08942870795726776,
      "rewards/rejected": -0.15615233778953552,
      "step": 6700
    },
    {
      "epoch": 0.5091819699499165,
      "grad_norm": 2.6900119883631857,
      "learning_rate": 9.76627000477968e-07,
      "log_odds_chosen": 1.305078148841858,
      "log_odds_ratio": -0.4170898497104645,
      "logits/chosen": -0.9849609136581421,
      "logits/rejected": -0.876953125,
      "logps/chosen": -0.657421886920929,
      "logps/rejected": -1.5378906726837158,
      "loss": 0.791,
      "nll_loss": 0.7315429449081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06569824367761612,
      "rewards/margins": 0.087982177734375,
      "rewards/rejected": -0.15371093153953552,
      "step": 6710
    },
    {
      "epoch": 0.5099408104416452,
      "grad_norm": 2.245518934976984,
      "learning_rate": 9.75900072948533e-07,
      "log_odds_chosen": 1.2332031726837158,
      "log_odds_ratio": -0.4161621034145355,
      "logits/chosen": -0.980664074420929,
      "logits/rejected": -0.838671863079071,
      "logps/chosen": -0.646191418170929,
      "logps/rejected": -1.4474608898162842,
      "loss": 0.7994,
      "nll_loss": 0.751953125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06459961086511612,
      "rewards/margins": 0.080230712890625,
      "rewards/rejected": -0.14479979872703552,
      "step": 6720
    },
    {
      "epoch": 0.5106996509333738,
      "grad_norm": 2.3214095844998477,
      "learning_rate": 9.751747662166388e-07,
      "log_odds_chosen": 1.365820288658142,
      "log_odds_ratio": -0.41303712129592896,
      "logits/chosen": -0.975781261920929,
      "logits/rejected": -0.8677734136581421,
      "logps/chosen": -0.6664062738418579,
      "logps/rejected": -1.587304711341858,
      "loss": 0.8016,
      "nll_loss": 0.693066418170929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06666259467601776,
      "rewards/margins": 0.09200439602136612,
      "rewards/rejected": -0.15866699814796448,
      "step": 6730
    },
    {
      "epoch": 0.5114584914251025,
      "grad_norm": 2.7639231282830115,
      "learning_rate": 9.744510742681917e-07,
      "log_odds_chosen": 1.160546898841858,
      "log_odds_ratio": -0.4703125059604645,
      "logits/chosen": -1.024804711341858,
      "logits/rejected": -0.891406238079071,
      "logps/chosen": -0.6795898675918579,
      "logps/rejected": -1.43359375,
      "loss": 0.8038,
      "nll_loss": 0.7606445550918579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06793212890625,
      "rewards/margins": 0.07551880180835724,
      "rewards/rejected": -0.1434326171875,
      "step": 6740
    },
    {
      "epoch": 0.512217331916831,
      "grad_norm": 2.6184261171577625,
      "learning_rate": 9.737289911202953e-07,
      "log_odds_chosen": 1.494042992591858,
      "log_odds_ratio": -0.3863769471645355,
      "logits/chosen": -1.0046875476837158,
      "logits/rejected": -0.8541015386581421,
      "logps/chosen": -0.6490234136581421,
      "logps/rejected": -1.7179687023162842,
      "loss": 0.8003,
      "nll_loss": 0.7718750238418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06492920219898224,
      "rewards/margins": 0.10683593899011612,
      "rewards/rejected": -0.1717529296875,
      "step": 6750
    },
    {
      "epoch": 0.5129761724085597,
      "grad_norm": 3.23821521030104,
      "learning_rate": 9.730085108210398e-07,
      "log_odds_chosen": 1.39306640625,
      "log_odds_ratio": -0.40678709745407104,
      "logits/chosen": -0.9076172113418579,
      "logits/rejected": -0.849804699420929,
      "logps/chosen": -0.6294921636581421,
      "logps/rejected": -1.568750023841858,
      "loss": 0.7786,
      "nll_loss": 0.7955077886581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06291504204273224,
      "rewards/margins": 0.09385986626148224,
      "rewards/rejected": -0.15690918266773224,
      "step": 6760
    },
    {
      "epoch": 0.5137350129002883,
      "grad_norm": 2.5228606567360137,
      "learning_rate": 9.72289627449298e-07,
      "log_odds_chosen": 1.4433472156524658,
      "log_odds_ratio": -0.410888671875,
      "logits/chosen": -0.963671863079071,
      "logits/rejected": -0.877148449420929,
      "logps/chosen": -0.647656261920929,
      "logps/rejected": -1.6749999523162842,
      "loss": 0.7765,
      "nll_loss": 0.6961914300918579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06474609673023224,
      "rewards/margins": 0.10265503078699112,
      "rewards/rejected": -0.16755370795726776,
      "step": 6770
    },
    {
      "epoch": 0.514493853392017,
      "grad_norm": 2.631470233438765,
      "learning_rate": 9.715723351145206e-07,
      "log_odds_chosen": 1.5310547351837158,
      "log_odds_ratio": -0.38579100370407104,
      "logits/chosen": -0.966992199420929,
      "logits/rejected": -0.884765625,
      "logps/chosen": -0.6060546636581421,
      "logps/rejected": -1.646093726158142,
      "loss": 0.7962,
      "nll_loss": 0.700488269329071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.060546875,
      "rewards/margins": 0.10415039211511612,
      "rewards/rejected": -0.16472168266773224,
      "step": 6780
    },
    {
      "epoch": 0.5152526938837456,
      "grad_norm": 2.5546199165416588,
      "learning_rate": 9.70856627956532e-07,
      "log_odds_chosen": 1.447265625,
      "log_odds_ratio": -0.3866943418979645,
      "logits/chosen": -0.9027343988418579,
      "logits/rejected": -0.817187488079071,
      "logps/chosen": -0.643261730670929,
      "logps/rejected": -1.619140625,
      "loss": 0.801,
      "nll_loss": 0.76953125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0643310546875,
      "rewards/margins": 0.09769286960363388,
      "rewards/rejected": -0.16188964247703552,
      "step": 6790
    },
    {
      "epoch": 0.5160115343754743,
      "grad_norm": 2.5953715660511416,
      "learning_rate": 9.701425001453318e-07,
      "log_odds_chosen": 1.3435547351837158,
      "log_odds_ratio": -0.41630858182907104,
      "logits/chosen": -0.966015636920929,
      "logits/rejected": -0.863085925579071,
      "logps/chosen": -0.6380859613418579,
      "logps/rejected": -1.5373046398162842,
      "loss": 0.8171,
      "nll_loss": 0.7525390386581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06385497748851776,
      "rewards/margins": 0.08985595405101776,
      "rewards/rejected": -0.15378418564796448,
      "step": 6800
    },
    {
      "epoch": 0.5167703748672029,
      "grad_norm": 2.7439030924941736,
      "learning_rate": 9.694299458808932e-07,
      "log_odds_chosen": 1.505273461341858,
      "log_odds_ratio": -0.37578123807907104,
      "logits/chosen": -0.980273425579071,
      "logits/rejected": -0.836132824420929,
      "logps/chosen": -0.6016601324081421,
      "logps/rejected": -1.5646483898162842,
      "loss": 0.7872,
      "nll_loss": 0.69091796875,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06013183668255806,
      "rewards/margins": 0.09639892727136612,
      "rewards/rejected": -0.15645751357078552,
      "step": 6810
    },
    {
      "epoch": 0.5175292153589316,
      "grad_norm": 2.423228834350587,
      "learning_rate": 9.687189593929655e-07,
      "log_odds_chosen": 1.675390601158142,
      "log_odds_ratio": -0.3683105409145355,
      "logits/chosen": -0.9546874761581421,
      "logits/rejected": -0.810546875,
      "logps/chosen": -0.6268554925918579,
      "logps/rejected": -1.775390625,
      "loss": 0.7869,
      "nll_loss": 0.742871105670929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06267090141773224,
      "rewards/margins": 0.11484374850988388,
      "rewards/rejected": -0.177490234375,
      "step": 6820
    },
    {
      "epoch": 0.5182880558506602,
      "grad_norm": 2.828370049907918,
      "learning_rate": 9.680095349408789e-07,
      "log_odds_chosen": 1.562597632408142,
      "log_odds_ratio": -0.37763673067092896,
      "logits/chosen": -0.950976550579071,
      "logits/rejected": -0.823046863079071,
      "logps/chosen": -0.6673828363418579,
      "logps/rejected": -1.7605469226837158,
      "loss": 0.7772,
      "nll_loss": 0.768750011920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06669922173023224,
      "rewards/margins": 0.10936889797449112,
      "rewards/rejected": -0.17617186903953552,
      "step": 6830
    },
    {
      "epoch": 0.5190468963423889,
      "grad_norm": 2.7637432716296315,
      "learning_rate": 9.673016668133487e-07,
      "log_odds_chosen": 1.343652367591858,
      "log_odds_ratio": -0.3913818299770355,
      "logits/chosen": -0.938671886920929,
      "logits/rejected": -0.8287109136581421,
      "logps/chosen": -0.5897461175918579,
      "logps/rejected": -1.456640601158142,
      "loss": 0.7817,
      "nll_loss": 0.754687488079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0589599609375,
      "rewards/margins": 0.08671798557043076,
      "rewards/rejected": -0.14567871391773224,
      "step": 6840
    },
    {
      "epoch": 0.5198057368341175,
      "grad_norm": 2.941099804178467,
      "learning_rate": 9.66595349328283e-07,
      "log_odds_chosen": 1.196435570716858,
      "log_odds_ratio": -0.4708496034145355,
      "logits/chosen": -0.958984375,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -0.6904296875,
      "logps/rejected": -1.497656226158142,
      "loss": 0.7827,
      "nll_loss": 0.768359363079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06904296576976776,
      "rewards/margins": 0.08078613132238388,
      "rewards/rejected": -0.1500244140625,
      "step": 6850
    },
    {
      "epoch": 0.5205645773258462,
      "grad_norm": 3.353435743055801,
      "learning_rate": 9.658905768325902e-07,
      "log_odds_chosen": 1.538476586341858,
      "log_odds_ratio": -0.37773436307907104,
      "logits/chosen": -1.01171875,
      "logits/rejected": -0.8740234375,
      "logps/chosen": -0.6664062738418579,
      "logps/rejected": -1.7140624523162842,
      "loss": 0.785,
      "nll_loss": 0.6810547113418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.066650390625,
      "rewards/margins": 0.10468444973230362,
      "rewards/rejected": -0.17128905653953552,
      "step": 6860
    },
    {
      "epoch": 0.5213234178175747,
      "grad_norm": 2.46441643656723,
      "learning_rate": 9.651873437019902e-07,
      "log_odds_chosen": 1.263085961341858,
      "log_odds_ratio": -0.46015626192092896,
      "logits/chosen": -0.918749988079071,
      "logits/rejected": -0.8193359375,
      "logps/chosen": -0.631640613079071,
      "logps/rejected": -1.5158202648162842,
      "loss": 0.7856,
      "nll_loss": 0.7313476800918579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06308593600988388,
      "rewards/margins": 0.08848877251148224,
      "rewards/rejected": -0.15163573622703552,
      "step": 6870
    },
    {
      "epoch": 0.5220822583093034,
      "grad_norm": 2.5169144272910224,
      "learning_rate": 9.644856443408243e-07,
      "log_odds_chosen": 1.239160180091858,
      "log_odds_ratio": -0.4374023377895355,
      "logits/chosen": -0.9375,
      "logits/rejected": -0.8082031011581421,
      "logps/chosen": -0.6591796875,
      "logps/rejected": -1.454687476158142,
      "loss": 0.8084,
      "nll_loss": 0.7230468988418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06593017280101776,
      "rewards/margins": 0.07955627143383026,
      "rewards/rejected": -0.1455078125,
      "step": 6880
    },
    {
      "epoch": 0.522841098801032,
      "grad_norm": 2.7243050305242664,
      "learning_rate": 9.637854731818697e-07,
      "log_odds_chosen": 1.463281273841858,
      "log_odds_ratio": -0.4019531309604645,
      "logits/chosen": -0.9173828363418579,
      "logits/rejected": -0.8265625238418579,
      "logps/chosen": -0.660839855670929,
      "logps/rejected": -1.705664038658142,
      "loss": 0.8129,
      "nll_loss": 0.7754882574081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06612548977136612,
      "rewards/margins": 0.10466308891773224,
      "rewards/rejected": -0.17070312798023224,
      "step": 6890
    },
    {
      "epoch": 0.5235999392927606,
      "grad_norm": 2.2732163303788164,
      "learning_rate": 9.630868246861536e-07,
      "log_odds_chosen": 1.528710961341858,
      "log_odds_ratio": -0.37177735567092896,
      "logits/chosen": -0.979296863079071,
      "logits/rejected": -0.8746093511581421,
      "logps/chosen": -0.7076171636581421,
      "logps/rejected": -1.7531249523162842,
      "loss": 0.7604,
      "nll_loss": 0.7767578363418579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.07081298530101776,
      "rewards/margins": 0.10456542670726776,
      "rewards/rejected": -0.17521972954273224,
      "step": 6900
    },
    {
      "epoch": 0.5243587797844893,
      "grad_norm": 2.6619644677274734,
      "learning_rate": 9.623896933427685e-07,
      "log_odds_chosen": 1.188623070716858,
      "log_odds_ratio": -0.482666015625,
      "logits/chosen": -0.908984363079071,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.660937488079071,
      "logps/rejected": -1.4660155773162842,
      "loss": 0.793,
      "nll_loss": 0.7635742425918579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06613769382238388,
      "rewards/margins": 0.0804443359375,
      "rewards/rejected": -0.14658203721046448,
      "step": 6910
    },
    {
      "epoch": 0.5251176202762179,
      "grad_norm": 7.819901412826592,
      "learning_rate": 9.6169407366869e-07,
      "log_odds_chosen": 1.4423828125,
      "log_odds_ratio": -0.40668946504592896,
      "logits/chosen": -0.9076172113418579,
      "logits/rejected": -0.7935546636581421,
      "logps/chosen": -0.6460937261581421,
      "logps/rejected": -1.642968773841858,
      "loss": 0.7908,
      "nll_loss": 0.774609386920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06459961086511612,
      "rewards/margins": 0.09953613579273224,
      "rewards/rejected": -0.16401366889476776,
      "step": 6920
    },
    {
      "epoch": 0.5258764607679466,
      "grad_norm": 2.7180232459008815,
      "learning_rate": 9.609999602085963e-07,
      "log_odds_chosen": 1.313574194908142,
      "log_odds_ratio": -0.4557128846645355,
      "logits/chosen": -0.9195312261581421,
      "logits/rejected": -0.842968761920929,
      "logps/chosen": -0.716796875,
      "logps/rejected": -1.6369140148162842,
      "loss": 0.7905,
      "nll_loss": 0.8287109136581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07176513969898224,
      "rewards/margins": 0.09190063178539276,
      "rewards/rejected": -0.16350097954273224,
      "step": 6930
    },
    {
      "epoch": 0.5266353012596752,
      "grad_norm": 3.1306929570721507,
      "learning_rate": 9.603073475346872e-07,
      "log_odds_chosen": 1.4545409679412842,
      "log_odds_ratio": -0.40385740995407104,
      "logits/chosen": -0.9380859136581421,
      "logits/rejected": -0.819531261920929,
      "logps/chosen": -0.68017578125,
      "logps/rejected": -1.6746094226837158,
      "loss": 0.7622,
      "nll_loss": 0.69921875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06804199516773224,
      "rewards/margins": 0.09943847358226776,
      "rewards/rejected": -0.16750487685203552,
      "step": 6940
    },
    {
      "epoch": 0.5273941417514039,
      "grad_norm": 2.7259073532868507,
      "learning_rate": 9.596162302465074e-07,
      "log_odds_chosen": 1.589257836341858,
      "log_odds_ratio": -0.380615234375,
      "logits/chosen": -0.947460949420929,
      "logits/rejected": -0.8343750238418579,
      "logps/chosen": -0.6329101324081421,
      "logps/rejected": -1.7296874523162842,
      "loss": 0.7846,
      "nll_loss": 0.6851562261581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06326904147863388,
      "rewards/margins": 0.10976715385913849,
      "rewards/rejected": -0.17312011122703552,
      "step": 6950
    },
    {
      "epoch": 0.5281529822431325,
      "grad_norm": 2.4986627117247826,
      "learning_rate": 9.589266029707683e-07,
      "log_odds_chosen": 1.505859375,
      "log_odds_ratio": -0.3985351622104645,
      "logits/chosen": -0.9332031011581421,
      "logits/rejected": -0.838085949420929,
      "logps/chosen": -0.6187499761581421,
      "logps/rejected": -1.602148413658142,
      "loss": 0.8016,
      "nll_loss": 0.7486327886581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06187744066119194,
      "rewards/margins": 0.09835205227136612,
      "rewards/rejected": -0.16022948920726776,
      "step": 6960
    },
    {
      "epoch": 0.5289118227348611,
      "grad_norm": 2.457685121496299,
      "learning_rate": 9.582384603611731e-07,
      "log_odds_chosen": 1.729882836341858,
      "log_odds_ratio": -0.38330078125,
      "logits/chosen": -0.966601550579071,
      "logits/rejected": -0.871874988079071,
      "logps/chosen": -0.6161133050918579,
      "logps/rejected": -1.8074219226837158,
      "loss": 0.7659,
      "nll_loss": 0.720703125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06159668043255806,
      "rewards/margins": 0.11909179389476776,
      "rewards/rejected": -0.18083496391773224,
      "step": 6970
    },
    {
      "epoch": 0.5296706632265897,
      "grad_norm": 2.921512624034416,
      "learning_rate": 9.575517970982428e-07,
      "log_odds_chosen": 1.4179198741912842,
      "log_odds_ratio": -0.40620118379592896,
      "logits/chosen": -0.9623047113418579,
      "logits/rejected": -0.890429675579071,
      "logps/chosen": -0.638867199420929,
      "logps/rejected": -1.629296898841858,
      "loss": 0.7772,
      "nll_loss": 0.7080078125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0638427734375,
      "rewards/margins": 0.09915771335363388,
      "rewards/rejected": -0.1629638671875,
      "step": 6980
    },
    {
      "epoch": 0.5304295037183184,
      "grad_norm": 2.8538488287444066,
      "learning_rate": 9.568666078891436e-07,
      "log_odds_chosen": 1.3582031726837158,
      "log_odds_ratio": -0.39306640625,
      "logits/chosen": -0.9498046636581421,
      "logits/rejected": -0.819531261920929,
      "logps/chosen": -0.6591796875,
      "logps/rejected": -1.591406226158142,
      "loss": 0.7526,
      "nll_loss": 0.7904297113418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06595458835363388,
      "rewards/margins": 0.09334106743335724,
      "rewards/rejected": -0.15913085639476776,
      "step": 6990
    },
    {
      "epoch": 0.531188344210047,
      "grad_norm": 2.471161964707584,
      "learning_rate": 9.561828874675149e-07,
      "log_odds_chosen": 1.565820336341858,
      "log_odds_ratio": -0.38359373807907104,
      "logits/chosen": -0.9423828125,
      "logits/rejected": -0.802929699420929,
      "logps/chosen": -0.648242175579071,
      "logps/rejected": -1.710351586341858,
      "loss": 0.7897,
      "nll_loss": 0.7093750238418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06485595554113388,
      "rewards/margins": 0.10623779147863388,
      "rewards/rejected": -0.17106933891773224,
      "step": 7000
    },
    {
      "epoch": 0.5319471847017757,
      "grad_norm": 2.504221714063816,
      "learning_rate": 9.555006305933e-07,
      "log_odds_chosen": 1.356542944908142,
      "log_odds_ratio": -0.3905273377895355,
      "logits/chosen": -0.898632824420929,
      "logits/rejected": -0.8265625238418579,
      "logps/chosen": -0.6851562261581421,
      "logps/rejected": -1.6232421398162842,
      "loss": 0.7684,
      "nll_loss": 0.834765613079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06850586086511612,
      "rewards/margins": 0.09375915676355362,
      "rewards/rejected": -0.162353515625,
      "step": 7010
    },
    {
      "epoch": 0.5327060251935043,
      "grad_norm": 2.6455109230239247,
      "learning_rate": 9.548198320525771e-07,
      "log_odds_chosen": 1.588476538658142,
      "log_odds_ratio": -0.39013671875,
      "logits/chosen": -0.959179699420929,
      "logits/rejected": -0.856640636920929,
      "logps/chosen": -0.6739257574081421,
      "logps/rejected": -1.765625,
      "loss": 0.7577,
      "nll_loss": 0.716015636920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06734619289636612,
      "rewards/margins": 0.10916747897863388,
      "rewards/rejected": -0.17644043266773224,
      "step": 7020
    },
    {
      "epoch": 0.533464865685233,
      "grad_norm": 2.665659525877553,
      "learning_rate": 9.54140486657392e-07,
      "log_odds_chosen": 1.4528319835662842,
      "log_odds_ratio": -0.3826660215854645,
      "logits/chosen": -0.935546875,
      "logits/rejected": -0.815234363079071,
      "logps/chosen": -0.613476574420929,
      "logps/rejected": -1.526953101158142,
      "loss": 0.7832,
      "nll_loss": 0.7515624761581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06130371242761612,
      "rewards/margins": 0.09135131537914276,
      "rewards/rejected": -0.15273436903953552,
      "step": 7030
    },
    {
      "epoch": 0.5342237061769616,
      "grad_norm": 3.477850454953123,
      "learning_rate": 9.534625892455922e-07,
      "log_odds_chosen": 1.335852026939392,
      "log_odds_ratio": -0.421142578125,
      "logits/chosen": -0.9087890386581421,
      "logits/rejected": -0.842968761920929,
      "logps/chosen": -0.7191406488418579,
      "logps/rejected": -1.6472656726837158,
      "loss": 0.7656,
      "nll_loss": 0.7748047113418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07187499850988388,
      "rewards/margins": 0.09270019829273224,
      "rewards/rejected": -0.16469725966453552,
      "step": 7040
    },
    {
      "epoch": 0.5349825466686903,
      "grad_norm": 2.5109526690815698,
      "learning_rate": 9.527861346806618e-07,
      "log_odds_chosen": 1.248388648033142,
      "log_odds_ratio": -0.4857421815395355,
      "logits/chosen": -0.928906261920929,
      "logits/rejected": -0.7884765863418579,
      "logps/chosen": -0.633984386920929,
      "logps/rejected": -1.505468726158142,
      "loss": 0.7672,
      "nll_loss": 0.7417968511581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06337890774011612,
      "rewards/margins": 0.08709106594324112,
      "rewards/rejected": -0.15053710341453552,
      "step": 7050
    },
    {
      "epoch": 0.5357413871604189,
      "grad_norm": 2.805501394008409,
      "learning_rate": 9.521111178515582e-07,
      "log_odds_chosen": 1.509179711341858,
      "log_odds_ratio": -0.3857421875,
      "logits/chosen": -0.904296875,
      "logits/rejected": -0.7822265625,
      "logps/chosen": -0.633984386920929,
      "logps/rejected": -1.6769530773162842,
      "loss": 0.7786,
      "nll_loss": 0.756054699420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06345214694738388,
      "rewards/margins": 0.104248046875,
      "rewards/rejected": -0.16767577826976776,
      "step": 7060
    },
    {
      "epoch": 0.5365002276521476,
      "grad_norm": 2.523528338253233,
      "learning_rate": 9.514375336725502e-07,
      "log_odds_chosen": 1.3629882335662842,
      "log_odds_ratio": -0.42631834745407104,
      "logits/chosen": -0.9833984375,
      "logits/rejected": -0.8382812738418579,
      "logps/chosen": -0.656445324420929,
      "logps/rejected": -1.5763671398162842,
      "loss": 0.7856,
      "nll_loss": 0.67919921875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.065673828125,
      "rewards/margins": 0.09210205078125,
      "rewards/rejected": -0.15781250596046448,
      "step": 7070
    },
    {
      "epoch": 0.5372590681438761,
      "grad_norm": 2.347057885123951,
      "learning_rate": 9.507653770830566e-07,
      "log_odds_chosen": 1.683508276939392,
      "log_odds_ratio": -0.36762696504592896,
      "logits/chosen": -0.959179699420929,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.6412109136581421,
      "logps/rejected": -1.8171875476837158,
      "loss": 0.7849,
      "nll_loss": 0.792187511920929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06411132961511612,
      "rewards/margins": 0.11753387749195099,
      "rewards/rejected": -0.18178710341453552,
      "step": 7080
    },
    {
      "epoch": 0.5380179086356048,
      "grad_norm": 2.4445964133345646,
      "learning_rate": 9.500946430474869e-07,
      "log_odds_chosen": 1.446874976158142,
      "log_odds_ratio": -0.37883299589157104,
      "logits/chosen": -0.9027343988418579,
      "logits/rejected": -0.7875000238418579,
      "logps/chosen": -0.636425793170929,
      "logps/rejected": -1.626562476158142,
      "loss": 0.7882,
      "nll_loss": 0.75244140625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06364746391773224,
      "rewards/margins": 0.09907837212085724,
      "rewards/rejected": -0.16274413466453552,
      "step": 7090
    },
    {
      "epoch": 0.5387767491273334,
      "grad_norm": 2.730352892938648,
      "learning_rate": 9.494253265550825e-07,
      "log_odds_chosen": 1.4993164539337158,
      "log_odds_ratio": -0.3963867127895355,
      "logits/chosen": -0.9150390625,
      "logits/rejected": -0.825390636920929,
      "logps/chosen": -0.663281261920929,
      "logps/rejected": -1.6980469226837158,
      "loss": 0.7832,
      "nll_loss": 0.739062488079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0662841796875,
      "rewards/margins": 0.10351105034351349,
      "rewards/rejected": -0.16989746689796448,
      "step": 7100
    },
    {
      "epoch": 0.5395355896190621,
      "grad_norm": 2.45968923221001,
      "learning_rate": 9.4875742261976e-07,
      "log_odds_chosen": 1.5222656726837158,
      "log_odds_ratio": -0.38286131620407104,
      "logits/chosen": -0.9779297113418579,
      "logits/rejected": -0.851367175579071,
      "logps/chosen": -0.6460937261581421,
      "logps/rejected": -1.715234398841858,
      "loss": 0.7741,
      "nll_loss": 0.7432616949081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06462402641773224,
      "rewards/margins": 0.10682372748851776,
      "rewards/rejected": -0.1715087890625,
      "step": 7110
    },
    {
      "epoch": 0.5402944301107907,
      "grad_norm": 2.724854084462044,
      "learning_rate": 9.480909262799544e-07,
      "log_odds_chosen": 1.492040991783142,
      "log_odds_ratio": -0.4017578065395355,
      "logits/chosen": -0.942187488079071,
      "logits/rejected": -0.8265625238418579,
      "logps/chosen": -0.655468761920929,
      "logps/rejected": -1.6925780773162842,
      "loss": 0.7486,
      "nll_loss": 0.72607421875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06557617336511612,
      "rewards/margins": 0.10375366359949112,
      "rewards/rejected": -0.1693115234375,
      "step": 7120
    },
    {
      "epoch": 0.5410532706025194,
      "grad_norm": 2.494648878304497,
      "learning_rate": 9.47425832598465e-07,
      "log_odds_chosen": 1.3213622570037842,
      "log_odds_ratio": -0.434814453125,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.8197265863418579,
      "logps/chosen": -0.6761718988418579,
      "logps/rejected": -1.625390648841858,
      "loss": 0.7838,
      "nll_loss": 0.800976574420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.067626953125,
      "rewards/margins": 0.09482727199792862,
      "rewards/rejected": -0.16259765625,
      "step": 7130
    },
    {
      "epoch": 0.541812111094248,
      "grad_norm": 2.6638397848095696,
      "learning_rate": 9.467621366623017e-07,
      "log_odds_chosen": 1.5519530773162842,
      "log_odds_ratio": -0.39545899629592896,
      "logits/chosen": -0.921093761920929,
      "logits/rejected": -0.7964843511581421,
      "logps/chosen": -0.6626952886581421,
      "logps/rejected": -1.768945336341858,
      "loss": 0.7815,
      "nll_loss": 0.726855456829071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06627197563648224,
      "rewards/margins": 0.11057128757238388,
      "rewards/rejected": -0.17678222060203552,
      "step": 7140
    },
    {
      "epoch": 0.5425709515859767,
      "grad_norm": 2.2888844362962604,
      "learning_rate": 9.46099833582532e-07,
      "log_odds_chosen": 1.6906249523162842,
      "log_odds_ratio": -0.3595214784145355,
      "logits/chosen": -0.9134765863418579,
      "logits/rejected": -0.8218749761581421,
      "logps/chosen": -0.6312500238418579,
      "logps/rejected": -1.814062476158142,
      "loss": 0.7649,
      "nll_loss": 0.70263671875,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0631103515625,
      "rewards/margins": 0.11835937201976776,
      "rewards/rejected": -0.18137207627296448,
      "step": 7150
    },
    {
      "epoch": 0.5433297920777053,
      "grad_norm": 3.630002360712264,
      "learning_rate": 9.45438918494131e-07,
      "log_odds_chosen": 1.832617163658142,
      "log_odds_ratio": -0.3507324159145355,
      "logits/chosen": -0.929882824420929,
      "logits/rejected": -0.8212890625,
      "logps/chosen": -0.681835949420929,
      "logps/rejected": -2.0171875953674316,
      "loss": 0.7708,
      "nll_loss": 0.73046875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06818847358226776,
      "rewards/margins": 0.13364258408546448,
      "rewards/rejected": -0.20185546576976776,
      "step": 7160
    },
    {
      "epoch": 0.5440886325694338,
      "grad_norm": 2.300448851841177,
      "learning_rate": 9.447793865558291e-07,
      "log_odds_chosen": 1.4615478515625,
      "log_odds_ratio": -0.3848632872104645,
      "logits/chosen": -0.942187488079071,
      "logits/rejected": -0.8187500238418579,
      "logps/chosen": -0.65380859375,
      "logps/rejected": -1.6953125,
      "loss": 0.7932,
      "nll_loss": 0.796093761920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06539306789636612,
      "rewards/margins": 0.10420074313879013,
      "rewards/rejected": -0.16962890326976776,
      "step": 7170
    },
    {
      "epoch": 0.5448474730611625,
      "grad_norm": 2.3612558524621354,
      "learning_rate": 9.441212329499659e-07,
      "log_odds_chosen": 1.5128905773162842,
      "log_odds_ratio": -0.382080078125,
      "logits/chosen": -0.932812511920929,
      "logits/rejected": -0.801953136920929,
      "logps/chosen": -0.6617187261581421,
      "logps/rejected": -1.707617163658142,
      "loss": 0.7934,
      "nll_loss": 0.7515624761581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06614990532398224,
      "rewards/margins": 0.10458984225988388,
      "rewards/rejected": -0.1707763671875,
      "step": 7180
    },
    {
      "epoch": 0.5456063135528911,
      "grad_norm": 2.5962116569427045,
      "learning_rate": 9.434644528823399e-07,
      "log_odds_chosen": 1.394384741783142,
      "log_odds_ratio": -0.44873046875,
      "logits/chosen": -0.956250011920929,
      "logits/rejected": -0.8392578363418579,
      "logps/chosen": -0.664257824420929,
      "logps/rejected": -1.638671875,
      "loss": 0.7885,
      "nll_loss": 0.7142578363418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06644286960363388,
      "rewards/margins": 0.09754333645105362,
      "rewards/rejected": -0.1640625,
      "step": 7190
    },
    {
      "epoch": 0.5463651540446198,
      "grad_norm": 2.291738804842518,
      "learning_rate": 9.428090415820634e-07,
      "log_odds_chosen": 1.4025390148162842,
      "log_odds_ratio": -0.400390625,
      "logits/chosen": -0.975781261920929,
      "logits/rejected": -0.862500011920929,
      "logps/chosen": -0.6993163824081421,
      "logps/rejected": -1.662500023841858,
      "loss": 0.7892,
      "nll_loss": 0.7564452886581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06992187350988388,
      "rewards/margins": 0.09614257514476776,
      "rewards/rejected": -0.16618652641773224,
      "step": 7200
    },
    {
      "epoch": 0.5471239945363484,
      "grad_norm": 2.857177751327088,
      "learning_rate": 9.42154994301416e-07,
      "log_odds_chosen": 1.5506713390350342,
      "log_odds_ratio": -0.35979002714157104,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.8648437261581421,
      "logps/chosen": -0.637499988079071,
      "logps/rejected": -1.7216796875,
      "loss": 0.7661,
      "nll_loss": 0.670214831829071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06369628757238388,
      "rewards/margins": 0.10839919745922089,
      "rewards/rejected": -0.17219237983226776,
      "step": 7210
    },
    {
      "epoch": 0.5478828350280771,
      "grad_norm": 2.8996143660225693,
      "learning_rate": 9.415023063157008e-07,
      "log_odds_chosen": 1.229150414466858,
      "log_odds_ratio": -0.4491210877895355,
      "logits/chosen": -0.9974609613418579,
      "logits/rejected": -0.8580077886581421,
      "logps/chosen": -0.6839843988418579,
      "logps/rejected": -1.5398437976837158,
      "loss": 0.7583,
      "nll_loss": 0.759082019329071,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06840820610523224,
      "rewards/margins": 0.08558349311351776,
      "rewards/rejected": -0.15397949516773224,
      "step": 7220
    },
    {
      "epoch": 0.5486416755198057,
      "grad_norm": 3.03582504638056,
      "learning_rate": 9.408509729231009e-07,
      "log_odds_chosen": 1.4357421398162842,
      "log_odds_ratio": -0.37236326932907104,
      "logits/chosen": -0.938281238079071,
      "logits/rejected": -0.83984375,
      "logps/chosen": -0.6533203125,
      "logps/rejected": -1.646875023841858,
      "loss": 0.7786,
      "nll_loss": 0.7642577886581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06527099758386612,
      "rewards/margins": 0.09946288913488388,
      "rewards/rejected": -0.164794921875,
      "step": 7230
    },
    {
      "epoch": 0.5494005160115344,
      "grad_norm": 2.2835769534267847,
      "learning_rate": 9.402009894445369e-07,
      "log_odds_chosen": 1.3484375476837158,
      "log_odds_ratio": -0.4205566346645355,
      "logits/chosen": -0.9837890863418579,
      "logits/rejected": -0.84765625,
      "logps/chosen": -0.67822265625,
      "logps/rejected": -1.6003906726837158,
      "loss": 0.7593,
      "nll_loss": 0.708789050579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06776122748851776,
      "rewards/margins": 0.092132568359375,
      "rewards/rejected": -0.16008301079273224,
      "step": 7240
    },
    {
      "epoch": 0.550159356503263,
      "grad_norm": 2.5675895626757943,
      "learning_rate": 9.395523512235255e-07,
      "log_odds_chosen": 1.58642578125,
      "log_odds_ratio": -0.3625244200229645,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.8628906011581421,
      "logps/chosen": -0.6869140863418579,
      "logps/rejected": -1.8132812976837158,
      "loss": 0.8094,
      "nll_loss": 0.8158203363418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.0687255859375,
      "rewards/margins": 0.11270751804113388,
      "rewards/rejected": -0.18154296278953552,
      "step": 7250
    },
    {
      "epoch": 0.5509181969949917,
      "grad_norm": 2.8513970220491127,
      "learning_rate": 9.389050536260404e-07,
      "log_odds_chosen": 1.2263672351837158,
      "log_odds_ratio": -0.4541015625,
      "logits/chosen": -0.9242187738418579,
      "logits/rejected": -0.8033202886581421,
      "logps/chosen": -0.70263671875,
      "logps/rejected": -1.5343749523162842,
      "loss": 0.7804,
      "nll_loss": 0.833789050579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07023926079273224,
      "rewards/margins": 0.08338622748851776,
      "rewards/rejected": -0.153564453125,
      "step": 7260
    },
    {
      "epoch": 0.5516770374867203,
      "grad_norm": 2.6185408123150844,
      "learning_rate": 9.382590920403722e-07,
      "log_odds_chosen": 1.608789086341858,
      "log_odds_ratio": -0.36479490995407104,
      "logits/chosen": -0.9400390386581421,
      "logits/rejected": -0.85546875,
      "logps/chosen": -0.60595703125,
      "logps/rejected": -1.7121093273162842,
      "loss": 0.7618,
      "nll_loss": 0.6788085699081421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06058349460363388,
      "rewards/margins": 0.11055908352136612,
      "rewards/rejected": -0.171142578125,
      "step": 7270
    },
    {
      "epoch": 0.552435877978449,
      "grad_norm": 2.333030605374203,
      "learning_rate": 9.376144618769908e-07,
      "log_odds_chosen": 1.5675780773162842,
      "log_odds_ratio": -0.3802246153354645,
      "logits/chosen": -1.0050780773162842,
      "logits/rejected": -0.8599609136581421,
      "logps/chosen": -0.6201171875,
      "logps/rejected": -1.7009766101837158,
      "loss": 0.7653,
      "nll_loss": 0.7071288824081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06198730319738388,
      "rewards/margins": 0.10809326171875,
      "rewards/rejected": -0.17014160752296448,
      "step": 7280
    },
    {
      "epoch": 0.5531947184701775,
      "grad_norm": 2.6131519079953214,
      "learning_rate": 9.369711585684086e-07,
      "log_odds_chosen": 1.2822265625,
      "log_odds_ratio": -0.44013673067092896,
      "logits/chosen": -0.9886718988418579,
      "logits/rejected": -0.8525390625,
      "logps/chosen": -0.662304699420929,
      "logps/rejected": -1.5626952648162842,
      "loss": 0.7707,
      "nll_loss": 0.72900390625,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06623534858226776,
      "rewards/margins": 0.09002685546875,
      "rewards/rejected": -0.15632323920726776,
      "step": 7290
    },
    {
      "epoch": 0.5539535589619062,
      "grad_norm": 2.602124099770009,
      "learning_rate": 9.363291775690445e-07,
      "log_odds_chosen": 1.3025391101837158,
      "log_odds_ratio": -0.42451173067092896,
      "logits/chosen": -0.970507800579071,
      "logits/rejected": -0.833203136920929,
      "logps/chosen": -0.655468761920929,
      "logps/rejected": -1.5193359851837158,
      "loss": 0.7686,
      "nll_loss": 0.752734363079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06553955376148224,
      "rewards/margins": 0.08654022216796875,
      "rewards/rejected": -0.15200194716453552,
      "step": 7300
    },
    {
      "epoch": 0.5547123994536348,
      "grad_norm": 2.541079998732178,
      "learning_rate": 9.356885143550886e-07,
      "log_odds_chosen": 1.4660155773162842,
      "log_odds_ratio": -0.374267578125,
      "logits/chosen": -0.9468749761581421,
      "logits/rejected": -0.843945324420929,
      "logps/chosen": -0.6307617425918579,
      "logps/rejected": -1.593164086341858,
      "loss": 0.7722,
      "nll_loss": 0.7533203363418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06306152045726776,
      "rewards/margins": 0.09632568061351776,
      "rewards/rejected": -0.15939942002296448,
      "step": 7310
    },
    {
      "epoch": 0.5554712399453635,
      "grad_norm": 2.8886481274368596,
      "learning_rate": 9.350491644243688e-07,
      "log_odds_chosen": 1.4816405773162842,
      "log_odds_ratio": -0.40327149629592896,
      "logits/chosen": -0.955078125,
      "logits/rejected": -0.848437488079071,
      "logps/chosen": -0.647167980670929,
      "logps/rejected": -1.68359375,
      "loss": 0.7802,
      "nll_loss": 0.7841796875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06475830078125,
      "rewards/margins": 0.10378418117761612,
      "rewards/rejected": -0.16840820014476776,
      "step": 7320
    },
    {
      "epoch": 0.5562300804370921,
      "grad_norm": 4.859820064597355,
      "learning_rate": 9.344111232962179e-07,
      "log_odds_chosen": 1.6296875476837158,
      "log_odds_ratio": -0.3758789002895355,
      "logits/chosen": -0.9189453125,
      "logits/rejected": -0.7923828363418579,
      "logps/chosen": -0.6424804925918579,
      "logps/rejected": -1.816015601158142,
      "loss": 0.783,
      "nll_loss": 0.7422851324081421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06423339992761612,
      "rewards/margins": 0.11756591498851776,
      "rewards/rejected": -0.18178710341453552,
      "step": 7330
    },
    {
      "epoch": 0.5569889209288208,
      "grad_norm": 2.743787058516089,
      "learning_rate": 9.337743865113415e-07,
      "log_odds_chosen": 1.3921387195587158,
      "log_odds_ratio": -0.435791015625,
      "logits/chosen": -0.884765625,
      "logits/rejected": -0.790820300579071,
      "logps/chosen": -0.650683581829071,
      "logps/rejected": -1.6320312023162842,
      "loss": 0.7939,
      "nll_loss": 0.771679699420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06510009616613388,
      "rewards/margins": 0.09806518256664276,
      "rewards/rejected": -0.16325683891773224,
      "step": 7340
    },
    {
      "epoch": 0.5577477614205494,
      "grad_norm": 2.5182450847624196,
      "learning_rate": 9.331389496316868e-07,
      "log_odds_chosen": 1.4314453601837158,
      "log_odds_ratio": -0.4581054747104645,
      "logits/chosen": -0.9527343511581421,
      "logits/rejected": -0.865039050579071,
      "logps/chosen": -0.679882824420929,
      "logps/rejected": -1.7507812976837158,
      "loss": 0.7553,
      "nll_loss": 0.7884765863418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0679931640625,
      "rewards/margins": 0.10709228366613388,
      "rewards/rejected": -0.17495116591453552,
      "step": 7350
    },
    {
      "epoch": 0.5585066019122781,
      "grad_norm": 2.6070795396854747,
      "learning_rate": 9.325048082403138e-07,
      "log_odds_chosen": 1.6416015625,
      "log_odds_ratio": -0.3251586854457855,
      "logits/chosen": -1.013085961341858,
      "logits/rejected": -0.880664050579071,
      "logps/chosen": -0.631054699420929,
      "logps/rejected": -1.7824218273162842,
      "loss": 0.7724,
      "nll_loss": 0.7201172113418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06306152045726776,
      "rewards/margins": 0.11510010063648224,
      "rewards/rejected": -0.17802734673023224,
      "step": 7360
    },
    {
      "epoch": 0.5592654424040067,
      "grad_norm": 2.4632773844481686,
      "learning_rate": 9.318719579412648e-07,
      "log_odds_chosen": 1.5597655773162842,
      "log_odds_ratio": -0.36591798067092896,
      "logits/chosen": -1.0085937976837158,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -0.6073242425918579,
      "logps/rejected": -1.6640625,
      "loss": 0.7795,
      "nll_loss": 0.679003894329071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06072998046875,
      "rewards/margins": 0.10567627102136612,
      "rewards/rejected": -0.16643066704273224,
      "step": 7370
    },
    {
      "epoch": 0.5600242828957354,
      "grad_norm": 2.7971555454862536,
      "learning_rate": 9.312403943594374e-07,
      "log_odds_chosen": 1.55517578125,
      "log_odds_ratio": -0.37092286348342896,
      "logits/chosen": -0.9365234375,
      "logits/rejected": -0.8089843988418579,
      "logps/chosen": -0.6265624761581421,
      "logps/rejected": -1.689843773841858,
      "loss": 0.76,
      "nll_loss": 0.6898437738418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06264648586511612,
      "rewards/margins": 0.10626526176929474,
      "rewards/rejected": -0.16887207329273224,
      "step": 7380
    },
    {
      "epoch": 0.5607831233874639,
      "grad_norm": 2.18946765104726,
      "learning_rate": 9.306101131404582e-07,
      "log_odds_chosen": 1.263281226158142,
      "log_odds_ratio": -0.455810546875,
      "logits/chosen": -0.939453125,
      "logits/rejected": -0.8482421636581421,
      "logps/chosen": -0.708691418170929,
      "logps/rejected": -1.584375023841858,
      "loss": 0.7792,
      "nll_loss": 0.738574206829071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07081298530101776,
      "rewards/margins": 0.08762206882238388,
      "rewards/rejected": -0.15837402641773224,
      "step": 7390
    },
    {
      "epoch": 0.5615419638791926,
      "grad_norm": 2.4849622577410293,
      "learning_rate": 9.299811099505542e-07,
      "log_odds_chosen": 1.4933593273162842,
      "log_odds_ratio": -0.37299805879592896,
      "logits/chosen": -0.9720703363418579,
      "logits/rejected": -0.8687499761581421,
      "logps/chosen": -0.6246093511581421,
      "logps/rejected": -1.637109398841858,
      "loss": 0.7551,
      "nll_loss": 0.712109386920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06248779222369194,
      "rewards/margins": 0.10123290866613388,
      "rewards/rejected": -0.16374512016773224,
      "step": 7400
    },
    {
      "epoch": 0.5623008043709212,
      "grad_norm": 2.7129427140327294,
      "learning_rate": 9.293533804764305e-07,
      "log_odds_chosen": 1.316308617591858,
      "log_odds_ratio": -0.411376953125,
      "logits/chosen": -1.0128905773162842,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.623046875,
      "logps/rejected": -1.483789086341858,
      "loss": 0.8028,
      "nll_loss": 0.684863269329071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06226806715130806,
      "rewards/margins": 0.08603973686695099,
      "rewards/rejected": -0.14833983778953552,
      "step": 7410
    },
    {
      "epoch": 0.5630596448626499,
      "grad_norm": 2.9744175857021578,
      "learning_rate": 9.28726920425144e-07,
      "log_odds_chosen": 1.170800805091858,
      "log_odds_ratio": -0.4571777284145355,
      "logits/chosen": -0.938281238079071,
      "logits/rejected": -0.8373047113418579,
      "logps/chosen": -0.6426757574081421,
      "logps/rejected": -1.4669921398162842,
      "loss": 0.7705,
      "nll_loss": 0.788378894329071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06427001953125,
      "rewards/margins": 0.08228149265050888,
      "rewards/rejected": -0.14655761420726776,
      "step": 7420
    },
    {
      "epoch": 0.5638184853543785,
      "grad_norm": 2.5004492863317953,
      "learning_rate": 9.281017255239815e-07,
      "log_odds_chosen": 1.53466796875,
      "log_odds_ratio": -0.35053712129592896,
      "logits/chosen": -0.9234374761581421,
      "logits/rejected": -0.8046875,
      "logps/chosen": -0.594042956829071,
      "logps/rejected": -1.5988280773162842,
      "loss": 0.7576,
      "nll_loss": 0.6942383050918579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05938720703125,
      "rewards/margins": 0.10055847465991974,
      "rewards/rejected": -0.15988770127296448,
      "step": 7430
    },
    {
      "epoch": 0.5645773258461072,
      "grad_norm": 2.6145819020877896,
      "learning_rate": 9.274777915203365e-07,
      "log_odds_chosen": 1.552148461341858,
      "log_odds_ratio": -0.4241699278354645,
      "logits/chosen": -0.975390613079071,
      "logits/rejected": -0.815234363079071,
      "logps/chosen": -0.653613269329071,
      "logps/rejected": -1.755468726158142,
      "loss": 0.7526,
      "nll_loss": 0.716601550579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06539306789636612,
      "rewards/margins": 0.1102294921875,
      "rewards/rejected": -0.17563477158546448,
      "step": 7440
    },
    {
      "epoch": 0.5653361663378358,
      "grad_norm": 2.46646475946503,
      "learning_rate": 9.268551141815875e-07,
      "log_odds_chosen": 1.64697265625,
      "log_odds_ratio": -0.33745115995407104,
      "logits/chosen": -1.0177733898162842,
      "logits/rejected": -0.8833984136581421,
      "logps/chosen": -0.5980468988418579,
      "logps/rejected": -1.7195312976837158,
      "loss": 0.766,
      "nll_loss": 0.709765613079071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05979003757238388,
      "rewards/margins": 0.11211548000574112,
      "rewards/rejected": -0.1719970703125,
      "step": 7450
    },
    {
      "epoch": 0.5660950068295644,
      "grad_norm": 3.2265705628290307,
      "learning_rate": 9.262336892949784e-07,
      "log_odds_chosen": 1.1706054210662842,
      "log_odds_ratio": -0.4654296934604645,
      "logits/chosen": -0.9546874761581421,
      "logits/rejected": -0.8480468988418579,
      "logps/chosen": -0.6761718988418579,
      "logps/rejected": -1.470312476158142,
      "loss": 0.7954,
      "nll_loss": 0.75146484375,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06760253757238388,
      "rewards/margins": 0.07932128757238388,
      "rewards/rejected": -0.14697265625,
      "step": 7460
    },
    {
      "epoch": 0.5668538473212931,
      "grad_norm": 9.852639130727077,
      "learning_rate": 9.256135126674977e-07,
      "log_odds_chosen": 1.3734862804412842,
      "log_odds_ratio": -0.4510742127895355,
      "logits/chosen": -0.8988281488418579,
      "logits/rejected": -0.78515625,
      "logps/chosen": -0.650683581829071,
      "logps/rejected": -1.618749976158142,
      "loss": 0.7742,
      "nll_loss": 0.7337890863418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06513671576976776,
      "rewards/margins": 0.09698486328125,
      "rewards/rejected": -0.16196289658546448,
      "step": 7470
    },
    {
      "epoch": 0.5676126878130217,
      "grad_norm": 2.994123638724084,
      "learning_rate": 9.249945801257605e-07,
      "log_odds_chosen": 1.362207055091858,
      "log_odds_ratio": -0.4122558534145355,
      "logits/chosen": -0.955273449420929,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.645800769329071,
      "logps/rejected": -1.5632812976837158,
      "loss": 0.7589,
      "nll_loss": 0.7157226800918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06458739936351776,
      "rewards/margins": 0.09183349460363388,
      "rewards/rejected": -0.15629883110523224,
      "step": 7480
    },
    {
      "epoch": 0.5683715283047504,
      "grad_norm": 2.704469295825329,
      "learning_rate": 9.243768875158902e-07,
      "log_odds_chosen": 1.5084228515625,
      "log_odds_ratio": -0.3988281190395355,
      "logits/chosen": -0.962109386920929,
      "logits/rejected": -0.81640625,
      "logps/chosen": -0.622265636920929,
      "logps/rejected": -1.668554663658142,
      "loss": 0.7624,
      "nll_loss": 0.683886706829071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06221923977136612,
      "rewards/margins": 0.10472564399242401,
      "rewards/rejected": -0.16682128608226776,
      "step": 7490
    },
    {
      "epoch": 0.5691303687964789,
      "grad_norm": 2.655294393971407,
      "learning_rate": 9.23760430703401e-07,
      "log_odds_chosen": 1.625390648841858,
      "log_odds_ratio": -0.3265624940395355,
      "logits/chosen": -0.97265625,
      "logits/rejected": -0.8177734613418579,
      "logps/chosen": -0.593945324420929,
      "logps/rejected": -1.656640648841858,
      "loss": 0.758,
      "nll_loss": 0.705078125,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05936279147863388,
      "rewards/margins": 0.10634765774011612,
      "rewards/rejected": -0.16572265326976776,
      "step": 7500
    },
    {
      "epoch": 0.5698892092882076,
      "grad_norm": 2.810283775483581,
      "learning_rate": 9.231452055730832e-07,
      "log_odds_chosen": 1.548437476158142,
      "log_odds_ratio": -0.3837890625,
      "logits/chosen": -0.976757824420929,
      "logits/rejected": -0.861132800579071,
      "logps/chosen": -0.6836913824081421,
      "logps/rejected": -1.798437476158142,
      "loss": 0.7585,
      "nll_loss": 0.7183593511581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06831054389476776,
      "rewards/margins": 0.1116943359375,
      "rewards/rejected": -0.18007811903953552,
      "step": 7510
    },
    {
      "epoch": 0.5706480497799362,
      "grad_norm": 2.4341374442114714,
      "learning_rate": 9.225312080288851e-07,
      "log_odds_chosen": 1.733007788658142,
      "log_odds_ratio": -0.354248046875,
      "logits/chosen": -0.978710949420929,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -0.599902331829071,
      "logps/rejected": -1.8406250476837158,
      "loss": 0.7651,
      "nll_loss": 0.686328113079071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06000976637005806,
      "rewards/margins": 0.12403564155101776,
      "rewards/rejected": -0.18405762314796448,
      "step": 7520
    },
    {
      "epoch": 0.5714068902716649,
      "grad_norm": 2.199907644850221,
      "learning_rate": 9.219184339938013e-07,
      "log_odds_chosen": 1.6378905773162842,
      "log_odds_ratio": -0.39191895723342896,
      "logits/chosen": -0.9605468511581421,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -0.6197265386581421,
      "logps/rejected": -1.7966797351837158,
      "loss": 0.7591,
      "nll_loss": 0.7176758050918579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06201171875,
      "rewards/margins": 0.11766357719898224,
      "rewards/rejected": -0.1795654296875,
      "step": 7530
    },
    {
      "epoch": 0.5721657307633935,
      "grad_norm": 2.9140947393774304,
      "learning_rate": 9.213068794097574e-07,
      "log_odds_chosen": 1.7209961414337158,
      "log_odds_ratio": -0.3708252012729645,
      "logits/chosen": -1.0173828601837158,
      "logits/rejected": -0.905078113079071,
      "logps/chosen": -0.659375011920929,
      "logps/rejected": -1.873046875,
      "loss": 0.7672,
      "nll_loss": 0.686328113079071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06590576469898224,
      "rewards/margins": 0.12152099609375,
      "rewards/rejected": -0.1875,
      "step": 7540
    },
    {
      "epoch": 0.5729245712551222,
      "grad_norm": 3.004383231876261,
      "learning_rate": 9.206965402374975e-07,
      "log_odds_chosen": 1.6552734375,
      "log_odds_ratio": -0.3800292909145355,
      "logits/chosen": -0.938281238079071,
      "logits/rejected": -0.8626953363418579,
      "logps/chosen": -0.6751953363418579,
      "logps/rejected": -1.8523437976837158,
      "loss": 0.7652,
      "nll_loss": 0.758105456829071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06746826320886612,
      "rewards/margins": 0.11782226711511612,
      "rewards/rejected": -0.18540039658546448,
      "step": 7550
    },
    {
      "epoch": 0.5736834117468508,
      "grad_norm": 2.3547857415398457,
      "learning_rate": 9.200874124564723e-07,
      "log_odds_chosen": 1.6642577648162842,
      "log_odds_ratio": -0.36689454317092896,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.8695312738418579,
      "logps/chosen": -0.641894519329071,
      "logps/rejected": -1.8214843273162842,
      "loss": 0.763,
      "nll_loss": 0.710742175579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06413574516773224,
      "rewards/margins": 0.1180419921875,
      "rewards/rejected": -0.18212890625,
      "step": 7560
    },
    {
      "epoch": 0.5744422522385795,
      "grad_norm": 2.772280051032957,
      "learning_rate": 9.194794920647274e-07,
      "log_odds_chosen": 1.484375,
      "log_odds_ratio": -0.36372071504592896,
      "logits/chosen": -0.9947265386581421,
      "logits/rejected": -0.8666015863418579,
      "logps/chosen": -0.5865234136581421,
      "logps/rejected": -1.560937523841858,
      "loss": 0.7458,
      "nll_loss": 0.6258789300918579,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.05865478515625,
      "rewards/margins": 0.097503662109375,
      "rewards/rejected": -0.15605469048023224,
      "step": 7570
    },
    {
      "epoch": 0.5752010927303081,
      "grad_norm": 3.1964210191522557,
      "learning_rate": 9.188727750787932e-07,
      "log_odds_chosen": 1.5912597179412842,
      "log_odds_ratio": -0.37580567598342896,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.8505859375,
      "logps/chosen": -0.6123046875,
      "logps/rejected": -1.6921875476837158,
      "loss": 0.7539,
      "nll_loss": 0.7627929449081421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06124267727136612,
      "rewards/margins": 0.10802002251148224,
      "rewards/rejected": -0.16933593153953552,
      "step": 7580
    },
    {
      "epoch": 0.5759599332220368,
      "grad_norm": 2.6368306769769903,
      "learning_rate": 9.182672575335757e-07,
      "log_odds_chosen": 1.7109375,
      "log_odds_ratio": -0.3613037168979645,
      "logits/chosen": -1.008398413658142,
      "logits/rejected": -0.8812500238418579,
      "logps/chosen": -0.64404296875,
      "logps/rejected": -1.81640625,
      "loss": 0.7541,
      "nll_loss": 0.708691418170929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.0643310546875,
      "rewards/margins": 0.11726073920726776,
      "rewards/rejected": -0.18166503310203552,
      "step": 7590
    },
    {
      "epoch": 0.5767187737137653,
      "grad_norm": 4.500822297677708,
      "learning_rate": 9.176629354822469e-07,
      "log_odds_chosen": 1.347070336341858,
      "log_odds_ratio": -0.45947265625,
      "logits/chosen": -0.9664062261581421,
      "logits/rejected": -0.8373047113418579,
      "logps/chosen": -0.69189453125,
      "logps/rejected": -1.6886718273162842,
      "loss": 0.7851,
      "nll_loss": 0.711718738079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06920166313648224,
      "rewards/margins": 0.09984131157398224,
      "rewards/rejected": -0.16896972060203552,
      "step": 7600
    },
    {
      "epoch": 0.577477614205494,
      "grad_norm": 3.035540435406703,
      "learning_rate": 9.170598049961371e-07,
      "log_odds_chosen": 1.3517577648162842,
      "log_odds_ratio": -0.42626953125,
      "logits/chosen": -0.9671875238418579,
      "logits/rejected": -0.853710949420929,
      "logps/chosen": -0.6034179925918579,
      "logps/rejected": -1.5107421875,
      "loss": 0.7478,
      "nll_loss": 0.6773437261581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06031494215130806,
      "rewards/margins": 0.09078369289636612,
      "rewards/rejected": -0.15097656846046448,
      "step": 7610
    },
    {
      "epoch": 0.5782364546972226,
      "grad_norm": 2.9484914911615463,
      "learning_rate": 9.164578621646276e-07,
      "log_odds_chosen": 1.277441382408142,
      "log_odds_ratio": -0.4228515625,
      "logits/chosen": -0.9839843511581421,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.642578125,
      "logps/rejected": -1.5056641101837158,
      "loss": 0.76,
      "nll_loss": 0.679980456829071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06428222358226776,
      "rewards/margins": 0.08623047173023224,
      "rewards/rejected": -0.1505126953125,
      "step": 7620
    },
    {
      "epoch": 0.5789952951889513,
      "grad_norm": 2.819793441355401,
      "learning_rate": 9.15857103095044e-07,
      "log_odds_chosen": 1.2861816883087158,
      "log_odds_ratio": -0.4524902403354645,
      "logits/chosen": -0.9585937261581421,
      "logits/rejected": -0.8486328125,
      "logps/chosen": -0.722363293170929,
      "logps/rejected": -1.591406226158142,
      "loss": 0.7638,
      "nll_loss": 0.747265636920929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07225342094898224,
      "rewards/margins": 0.08688811957836151,
      "rewards/rejected": -0.15908202528953552,
      "step": 7630
    },
    {
      "epoch": 0.5797541356806799,
      "grad_norm": 2.5153252582631693,
      "learning_rate": 9.15257523912551e-07,
      "log_odds_chosen": 1.3693358898162842,
      "log_odds_ratio": -0.4251953065395355,
      "logits/chosen": -0.927539050579071,
      "logits/rejected": -0.836132824420929,
      "logps/chosen": -0.625683605670929,
      "logps/rejected": -1.5832030773162842,
      "loss": 0.7611,
      "nll_loss": 0.757617175579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06252441555261612,
      "rewards/margins": 0.09589233249425888,
      "rewards/rejected": -0.158447265625,
      "step": 7640
    },
    {
      "epoch": 0.5805129761724086,
      "grad_norm": 2.7496077089311344,
      "learning_rate": 9.146591207600472e-07,
      "log_odds_chosen": 1.626953125,
      "log_odds_ratio": -0.3426269590854645,
      "logits/chosen": -0.9466797113418579,
      "logits/rejected": -0.80078125,
      "logps/chosen": -0.6019531488418579,
      "logps/rejected": -1.703125,
      "loss": 0.7707,
      "nll_loss": 0.682324230670929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06020507961511612,
      "rewards/margins": 0.11014404147863388,
      "rewards/rejected": -0.17050781846046448,
      "step": 7650
    },
    {
      "epoch": 0.5812718166641372,
      "grad_norm": 2.837450024163419,
      "learning_rate": 9.140618897980601e-07,
      "log_odds_chosen": 1.63916015625,
      "log_odds_ratio": -0.38605958223342896,
      "logits/chosen": -0.9662109613418579,
      "logits/rejected": -0.820117175579071,
      "logps/chosen": -0.69580078125,
      "logps/rejected": -1.8357422351837158,
      "loss": 0.7624,
      "nll_loss": 0.7529296875,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06965331733226776,
      "rewards/margins": 0.11403503268957138,
      "rewards/rejected": -0.18361815810203552,
      "step": 7660
    },
    {
      "epoch": 0.5820306571558659,
      "grad_norm": 2.7485560693912507,
      "learning_rate": 9.134658272046442e-07,
      "log_odds_chosen": 1.76171875,
      "log_odds_ratio": -0.31635743379592896,
      "logits/chosen": -1.0505859851837158,
      "logits/rejected": -0.9251953363418579,
      "logps/chosen": -0.636035144329071,
      "logps/rejected": -1.891015648841858,
      "loss": 0.7707,
      "nll_loss": 0.6905273199081421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06361083686351776,
      "rewards/margins": 0.125457763671875,
      "rewards/rejected": -0.18911132216453552,
      "step": 7670
    },
    {
      "epoch": 0.5827894976475945,
      "grad_norm": 2.348448657361946,
      "learning_rate": 9.128709291752768e-07,
      "log_odds_chosen": 1.8796875476837158,
      "log_odds_ratio": -0.3135009706020355,
      "logits/chosen": -0.903515636920929,
      "logits/rejected": -0.7607421875,
      "logps/chosen": -0.6141601800918579,
      "logps/rejected": -1.969140648841858,
      "loss": 0.7625,
      "nll_loss": 0.6856445074081421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06138915941119194,
      "rewards/margins": 0.13565674424171448,
      "rewards/rejected": -0.197021484375,
      "step": 7680
    },
    {
      "epoch": 0.5835483381393232,
      "grad_norm": 2.8538406403221233,
      "learning_rate": 9.122771919227568e-07,
      "log_odds_chosen": 1.5810546875,
      "log_odds_ratio": -0.3882812559604645,
      "logits/chosen": -0.9818359613418579,
      "logits/rejected": -0.8955078125,
      "logps/chosen": -0.593554675579071,
      "logps/rejected": -1.635156273841858,
      "loss": 0.7663,
      "nll_loss": 0.7206054925918579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05935058742761612,
      "rewards/margins": 0.1041259765625,
      "rewards/rejected": -0.16352538764476776,
      "step": 7690
    },
    {
      "epoch": 0.5843071786310517,
      "grad_norm": 2.5178389786443205,
      "learning_rate": 9.116846116771035e-07,
      "log_odds_chosen": 1.4381835460662842,
      "log_odds_ratio": -0.39301759004592896,
      "logits/chosen": -0.951367199420929,
      "logits/rejected": -0.854296863079071,
      "logps/chosen": -0.6426757574081421,
      "logps/rejected": -1.6025390625,
      "loss": 0.7505,
      "nll_loss": 0.7191406488418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06423339992761612,
      "rewards/margins": 0.09596557915210724,
      "rewards/rejected": -0.16015625,
      "step": 7700
    },
    {
      "epoch": 0.5850660191227804,
      "grad_norm": 2.284825516468035,
      "learning_rate": 9.110931846854553e-07,
      "log_odds_chosen": 1.4861328601837158,
      "log_odds_ratio": -0.4158691465854645,
      "logits/chosen": -0.9330078363418579,
      "logits/rejected": -0.8111327886581421,
      "logps/chosen": -0.707714855670929,
      "logps/rejected": -1.7546875476837158,
      "loss": 0.7757,
      "nll_loss": 0.793749988079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07072754204273224,
      "rewards/margins": 0.10472412407398224,
      "rewards/rejected": -0.17534179985523224,
      "step": 7710
    },
    {
      "epoch": 0.585824859614509,
      "grad_norm": 2.8547151587086987,
      "learning_rate": 9.105029072119708e-07,
      "log_odds_chosen": 1.75146484375,
      "log_odds_ratio": -0.32563477754592896,
      "logits/chosen": -0.9185546636581421,
      "logits/rejected": -0.8388671875,
      "logps/chosen": -0.645214855670929,
      "logps/rejected": -1.889062523841858,
      "loss": 0.7611,
      "nll_loss": 0.713671863079071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06451416015625,
      "rewards/margins": 0.12446288764476776,
      "rewards/rejected": -0.18886718153953552,
      "step": 7720
    },
    {
      "epoch": 0.5865837001062376,
      "grad_norm": 2.589481144735132,
      "learning_rate": 9.099137755377291e-07,
      "log_odds_chosen": 1.5060546398162842,
      "log_odds_ratio": -0.37822264432907104,
      "logits/chosen": -0.921093761920929,
      "logits/rejected": -0.8125,
      "logps/chosen": -0.6312500238418579,
      "logps/rejected": -1.6433594226837158,
      "loss": 0.7639,
      "nll_loss": 0.772265613079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06312255561351776,
      "rewards/margins": 0.10126952826976776,
      "rewards/rejected": -0.16440430283546448,
      "step": 7730
    },
    {
      "epoch": 0.5873425405979663,
      "grad_norm": 2.842678512806397,
      "learning_rate": 9.093257859606311e-07,
      "log_odds_chosen": 1.505957007408142,
      "log_odds_ratio": -0.39775389432907104,
      "logits/chosen": -0.945507824420929,
      "logits/rejected": -0.8578125238418579,
      "logps/chosen": -0.62548828125,
      "logps/rejected": -1.662109375,
      "loss": 0.7805,
      "nll_loss": 0.772753894329071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06257323920726776,
      "rewards/margins": 0.10358276218175888,
      "rewards/rejected": -0.16621093451976776,
      "step": 7740
    },
    {
      "epoch": 0.5881013810896949,
      "grad_norm": 2.3845474551251695,
      "learning_rate": 9.087389347953037e-07,
      "log_odds_chosen": 1.724340796470642,
      "log_odds_ratio": -0.38164061307907104,
      "logits/chosen": -0.9351562261581421,
      "logits/rejected": -0.8374999761581421,
      "logps/chosen": -0.64599609375,
      "logps/rejected": -1.8640625476837158,
      "loss": 0.7421,
      "nll_loss": 0.7303711175918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06458739936351776,
      "rewards/margins": 0.12175750732421875,
      "rewards/rejected": -0.18647460639476776,
      "step": 7750
    },
    {
      "epoch": 0.5888602215814236,
      "grad_norm": 3.049628718243713,
      "learning_rate": 9.081532183729995e-07,
      "log_odds_chosen": 1.3771483898162842,
      "log_odds_ratio": -0.43896484375,
      "logits/chosen": -0.938281238079071,
      "logits/rejected": -0.8255859613418579,
      "logps/chosen": -0.696484386920929,
      "logps/rejected": -1.634179711341858,
      "loss": 0.7616,
      "nll_loss": 0.7574218511581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06977538764476776,
      "rewards/margins": 0.09376220405101776,
      "rewards/rejected": -0.163330078125,
      "step": 7760
    },
    {
      "epoch": 0.5896190620731522,
      "grad_norm": 2.9898675639163703,
      "learning_rate": 9.075686330415037e-07,
      "log_odds_chosen": 1.56689453125,
      "log_odds_ratio": -0.36835938692092896,
      "logits/chosen": -0.9673827886581421,
      "logits/rejected": -0.8238281011581421,
      "logps/chosen": -0.64990234375,
      "logps/rejected": -1.7734375,
      "loss": 0.7388,
      "nll_loss": 0.656054675579071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06495361030101776,
      "rewards/margins": 0.11245422065258026,
      "rewards/rejected": -0.17746582627296448,
      "step": 7770
    },
    {
      "epoch": 0.5903779025648809,
      "grad_norm": 2.711493764348975,
      "learning_rate": 9.069851751650364e-07,
      "log_odds_chosen": 1.5798828601837158,
      "log_odds_ratio": -0.37016600370407104,
      "logits/chosen": -1.006250023841858,
      "logits/rejected": -0.880664050579071,
      "logps/chosen": -0.5819336175918579,
      "logps/rejected": -1.6472656726837158,
      "loss": 0.7562,
      "nll_loss": 0.6475585699081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05820312350988388,
      "rewards/margins": 0.106536865234375,
      "rewards/rejected": -0.16489258408546448,
      "step": 7780
    },
    {
      "epoch": 0.5911367430566095,
      "grad_norm": 3.3006891775111766,
      "learning_rate": 9.064028411241582e-07,
      "log_odds_chosen": 1.5465819835662842,
      "log_odds_ratio": -0.3990722596645355,
      "logits/chosen": -0.957812488079071,
      "logits/rejected": -0.855664074420929,
      "logps/chosen": -0.66064453125,
      "logps/rejected": -1.7744140625,
      "loss": 0.7534,
      "nll_loss": 0.7235351800918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.0660400390625,
      "rewards/margins": 0.11158446967601776,
      "rewards/rejected": -0.17741699516773224,
      "step": 7790
    },
    {
      "epoch": 0.5918955835483382,
      "grad_norm": 2.8747180760944993,
      "learning_rate": 9.058216273156764e-07,
      "log_odds_chosen": 1.385156273841858,
      "log_odds_ratio": -0.46162110567092896,
      "logits/chosen": -0.9662109613418579,
      "logits/rejected": -0.890820324420929,
      "logps/chosen": -0.673046886920929,
      "logps/rejected": -1.666015625,
      "loss": 0.7487,
      "nll_loss": 0.728710949420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06728515774011612,
      "rewards/margins": 0.09935303032398224,
      "rewards/rejected": -0.16672363877296448,
      "step": 7800
    },
    {
      "epoch": 0.5926544240400667,
      "grad_norm": 3.1985485855872056,
      "learning_rate": 9.052415301525511e-07,
      "log_odds_chosen": 1.815039038658142,
      "log_odds_ratio": -0.3399902284145355,
      "logits/chosen": -0.9580078125,
      "logits/rejected": -0.891796886920929,
      "logps/chosen": -0.633984386920929,
      "logps/rejected": -1.9548828601837158,
      "loss": 0.7619,
      "nll_loss": 0.7357422113418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06337890774011612,
      "rewards/margins": 0.132080078125,
      "rewards/rejected": -0.19541016221046448,
      "step": 7810
    },
    {
      "epoch": 0.5934132645317954,
      "grad_norm": 2.5089583144596608,
      "learning_rate": 9.046625460638012e-07,
      "log_odds_chosen": 1.639257788658142,
      "log_odds_ratio": -0.359619140625,
      "logits/chosen": -0.9945312738418579,
      "logits/rejected": -0.8822265863418579,
      "logps/chosen": -0.6244140863418579,
      "logps/rejected": -1.753320336341858,
      "loss": 0.7484,
      "nll_loss": 0.73779296875,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06240234524011612,
      "rewards/margins": 0.11271972954273224,
      "rewards/rejected": -0.17531737685203552,
      "step": 7820
    },
    {
      "epoch": 0.594172105023524,
      "grad_norm": 2.4805759789569914,
      "learning_rate": 9.040846714944138e-07,
      "log_odds_chosen": 1.455957055091858,
      "log_odds_ratio": -0.36015623807907104,
      "logits/chosen": -0.991406261920929,
      "logits/rejected": -0.8359375,
      "logps/chosen": -0.625292956829071,
      "logps/rejected": -1.5916016101837158,
      "loss": 0.7739,
      "nll_loss": 0.70556640625,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06253661960363388,
      "rewards/margins": 0.09675903618335724,
      "rewards/rejected": -0.15915527939796448,
      "step": 7830
    },
    {
      "epoch": 0.5949309455152527,
      "grad_norm": 2.821660680248788,
      "learning_rate": 9.035079029052513e-07,
      "log_odds_chosen": 1.501562476158142,
      "log_odds_ratio": -0.38007813692092896,
      "logits/chosen": -0.9839843511581421,
      "logits/rejected": -0.895703136920929,
      "logps/chosen": -0.63818359375,
      "logps/rejected": -1.644140601158142,
      "loss": 0.7391,
      "nll_loss": 0.749218761920929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06376953423023224,
      "rewards/margins": 0.10068969428539276,
      "rewards/rejected": -0.1644287109375,
      "step": 7840
    },
    {
      "epoch": 0.5956897860069813,
      "grad_norm": 2.6384636226315252,
      "learning_rate": 9.029322367729605e-07,
      "log_odds_chosen": 1.4099609851837158,
      "log_odds_ratio": -0.41801756620407104,
      "logits/chosen": -0.9906250238418579,
      "logits/rejected": -0.8970702886581421,
      "logps/chosen": -0.678027331829071,
      "logps/rejected": -1.642187476158142,
      "loss": 0.7719,
      "nll_loss": 0.724414050579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06778564304113388,
      "rewards/margins": 0.09635009616613388,
      "rewards/rejected": -0.1641845703125,
      "step": 7850
    },
    {
      "epoch": 0.59644862649871,
      "grad_norm": 2.9699807727782486,
      "learning_rate": 9.02357669589883e-07,
      "log_odds_chosen": 1.49609375,
      "log_odds_ratio": -0.40751951932907104,
      "logits/chosen": -1.0207030773162842,
      "logits/rejected": -0.8851562738418579,
      "logps/chosen": -0.633593738079071,
      "logps/rejected": -1.6726562976837158,
      "loss": 0.7629,
      "nll_loss": 0.765820324420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06331787258386612,
      "rewards/margins": 0.10396728664636612,
      "rewards/rejected": -0.167236328125,
      "step": 7860
    },
    {
      "epoch": 0.5972074669904386,
      "grad_norm": 11.36507148656882,
      "learning_rate": 9.017841978639643e-07,
      "log_odds_chosen": 1.2036621570587158,
      "log_odds_ratio": -0.46928709745407104,
      "logits/chosen": -0.9251953363418579,
      "logits/rejected": -0.84765625,
      "logps/chosen": -0.646484375,
      "logps/rejected": -1.45703125,
      "loss": 0.7505,
      "nll_loss": 0.7432616949081421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06470946967601776,
      "rewards/margins": 0.08107452094554901,
      "rewards/rejected": -0.14573974907398224,
      "step": 7870
    },
    {
      "epoch": 0.5979663074821673,
      "grad_norm": 2.6416555828030654,
      "learning_rate": 9.012118181186658e-07,
      "log_odds_chosen": 2.001953125,
      "log_odds_ratio": -0.2942871153354645,
      "logits/chosen": -0.9624999761581421,
      "logits/rejected": -0.8707031011581421,
      "logps/chosen": -0.59619140625,
      "logps/rejected": -2.0082030296325684,
      "loss": 0.7552,
      "nll_loss": 0.7476562261581421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.05964355543255806,
      "rewards/margins": 0.14122314751148224,
      "rewards/rejected": -0.2008056640625,
      "step": 7880
    },
    {
      "epoch": 0.5987251479738959,
      "grad_norm": 3.7616505179711432,
      "learning_rate": 9.00640526892875e-07,
      "log_odds_chosen": 1.847265601158142,
      "log_odds_ratio": -0.3622070252895355,
      "logits/chosen": -0.9478515386581421,
      "logits/rejected": -0.8232421875,
      "logps/chosen": -0.7217773199081421,
      "logps/rejected": -2.071093797683716,
      "loss": 0.7506,
      "nll_loss": 0.716992199420929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.07220458984375,
      "rewards/margins": 0.13499756157398224,
      "rewards/rejected": -0.20712891221046448,
      "step": 7890
    },
    {
      "epoch": 0.5994839884656246,
      "grad_norm": 2.5231640032615923,
      "learning_rate": 9.000703207408191e-07,
      "log_odds_chosen": 1.419921875,
      "log_odds_ratio": -0.41948240995407104,
      "logits/chosen": -0.9810546636581421,
      "logits/rejected": -0.8177734613418579,
      "logps/chosen": -0.7158203125,
      "logps/rejected": -1.7648437023162842,
      "loss": 0.7354,
      "nll_loss": 0.777148425579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07145996391773224,
      "rewards/margins": 0.10487060248851776,
      "rewards/rejected": -0.17634277045726776,
      "step": 7900
    },
    {
      "epoch": 0.6002428289573531,
      "grad_norm": 2.4294931927903898,
      "learning_rate": 8.995011962319761e-07,
      "log_odds_chosen": 1.544824242591858,
      "log_odds_ratio": -0.4046630859375,
      "logits/chosen": -0.958789050579071,
      "logits/rejected": -0.864062488079071,
      "logps/chosen": -0.6341797113418579,
      "logps/rejected": -1.691015601158142,
      "loss": 0.77,
      "nll_loss": 0.7001953125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06340332329273224,
      "rewards/margins": 0.10576172173023224,
      "rewards/rejected": -0.16911621391773224,
      "step": 7910
    },
    {
      "epoch": 0.6010016694490818,
      "grad_norm": 2.5209477310139157,
      "learning_rate": 8.989331499509894e-07,
      "log_odds_chosen": 1.433203101158142,
      "log_odds_ratio": -0.3963378965854645,
      "logits/chosen": -0.981249988079071,
      "logits/rejected": -0.8931640386581421,
      "logps/chosen": -0.662792980670929,
      "logps/rejected": -1.6417968273162842,
      "loss": 0.7434,
      "nll_loss": 0.7215820550918579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06621094048023224,
      "rewards/margins": 0.09779052436351776,
      "rewards/rejected": -0.16416016221046448,
      "step": 7920
    },
    {
      "epoch": 0.6017605099408104,
      "grad_norm": 2.7461637562871686,
      "learning_rate": 8.983661784975812e-07,
      "log_odds_chosen": 1.5466797351837158,
      "log_odds_ratio": -0.36066895723342896,
      "logits/chosen": -0.9654296636581421,
      "logits/rejected": -0.8125,
      "logps/chosen": -0.6265624761581421,
      "logps/rejected": -1.687109351158142,
      "loss": 0.7737,
      "nll_loss": 0.767382800579071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06264648586511612,
      "rewards/margins": 0.10602416843175888,
      "rewards/rejected": -0.16865234076976776,
      "step": 7930
    },
    {
      "epoch": 0.6025193504325391,
      "grad_norm": 2.5384272995695034,
      "learning_rate": 8.97800278486467e-07,
      "log_odds_chosen": 1.696874976158142,
      "log_odds_ratio": -0.33732908964157104,
      "logits/chosen": -0.960742175579071,
      "logits/rejected": -0.837890625,
      "logps/chosen": -0.5869140625,
      "logps/rejected": -1.716796875,
      "loss": 0.7409,
      "nll_loss": 0.6884765625,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05874023586511612,
      "rewards/margins": 0.11293945461511612,
      "rewards/rejected": -0.17158202826976776,
      "step": 7940
    },
    {
      "epoch": 0.6032781909242677,
      "grad_norm": 2.493644451819072,
      "learning_rate": 8.972354465472708e-07,
      "log_odds_chosen": 1.6525390148162842,
      "log_odds_ratio": -0.32763671875,
      "logits/chosen": -0.9765625,
      "logits/rejected": -0.8423827886581421,
      "logps/chosen": -0.6025390625,
      "logps/rejected": -1.734765648841858,
      "loss": 0.7652,
      "nll_loss": 0.708300769329071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06024169921875,
      "rewards/margins": 0.11328125,
      "rewards/rejected": -0.17360839247703552,
      "step": 7950
    },
    {
      "epoch": 0.6040370314159964,
      "grad_norm": 2.8487068023888122,
      "learning_rate": 8.966716793244405e-07,
      "log_odds_chosen": 1.623437523841858,
      "log_odds_ratio": -0.38330078125,
      "logits/chosen": -0.967968761920929,
      "logits/rejected": -0.8353515863418579,
      "logps/chosen": -0.65625,
      "logps/rejected": -1.798828125,
      "loss": 0.7419,
      "nll_loss": 0.6820312738418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06566162407398224,
      "rewards/margins": 0.11431884765625,
      "rewards/rejected": -0.17983397841453552,
      "step": 7960
    },
    {
      "epoch": 0.604795871907725,
      "grad_norm": 2.5073907734682837,
      "learning_rate": 8.96108973477165e-07,
      "log_odds_chosen": 1.2568359375,
      "log_odds_ratio": -0.4188476502895355,
      "logits/chosen": -0.947070300579071,
      "logits/rejected": -0.8677734136581421,
      "logps/chosen": -0.6322265863418579,
      "logps/rejected": -1.4666016101837158,
      "loss": 0.7561,
      "nll_loss": 0.697460949420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06324462592601776,
      "rewards/margins": 0.08325805515050888,
      "rewards/rejected": -0.14653320610523224,
      "step": 7970
    },
    {
      "epoch": 0.6055547123994537,
      "grad_norm": 3.2528673875615794,
      "learning_rate": 8.955473256792899e-07,
      "log_odds_chosen": 1.5939452648162842,
      "log_odds_ratio": -0.3775878846645355,
      "logits/chosen": -0.9515625238418579,
      "logits/rejected": -0.838671863079071,
      "logps/chosen": -0.666796863079071,
      "logps/rejected": -1.766015648841858,
      "loss": 0.7594,
      "nll_loss": 0.7408202886581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06668701022863388,
      "rewards/margins": 0.10986328125,
      "rewards/rejected": -0.17658691108226776,
      "step": 7980
    },
    {
      "epoch": 0.6063135528911823,
      "grad_norm": 3.404393843374074,
      "learning_rate": 8.949867326192358e-07,
      "log_odds_chosen": 1.6707031726837158,
      "log_odds_ratio": -0.35498046875,
      "logits/chosen": -1.0046875476837158,
      "logits/rejected": -0.8548828363418579,
      "logps/chosen": -0.5908203125,
      "logps/rejected": -1.740625023841858,
      "loss": 0.7356,
      "nll_loss": 0.6864258050918579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.05908203125,
      "rewards/margins": 0.11496581882238388,
      "rewards/rejected": -0.17404785752296448,
      "step": 7990
    },
    {
      "epoch": 0.6070723933829109,
      "grad_norm": 2.582456484827545,
      "learning_rate": 8.944271909999158e-07,
      "log_odds_chosen": 1.7126953601837158,
      "log_odds_ratio": -0.37041014432907104,
      "logits/chosen": -1.015234351158142,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.6416991949081421,
      "logps/rejected": -1.874609351158142,
      "loss": 0.7337,
      "nll_loss": 0.674121081829071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06411132961511612,
      "rewards/margins": 0.12339477241039276,
      "rewards/rejected": -0.18747559189796448,
      "step": 8000
    },
    {
      "epoch": 0.6078312338746396,
      "grad_norm": 2.4714549139507325,
      "learning_rate": 8.938686975386545e-07,
      "log_odds_chosen": 1.644921898841858,
      "log_odds_ratio": -0.3622070252895355,
      "logits/chosen": -0.9818359613418579,
      "logits/rejected": -0.8648437261581421,
      "logps/chosen": -0.6283203363418579,
      "logps/rejected": -1.785546898841858,
      "loss": 0.7388,
      "nll_loss": 0.66650390625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06279297173023224,
      "rewards/margins": 0.11582031100988388,
      "rewards/rejected": -0.1787109375,
      "step": 8010
    },
    {
      "epoch": 0.6085900743663681,
      "grad_norm": 2.492341880398822,
      "learning_rate": 8.933112489671067e-07,
      "log_odds_chosen": 1.431249976158142,
      "log_odds_ratio": -0.40974122285842896,
      "logits/chosen": -0.9873046875,
      "logits/rejected": -0.8990234136581421,
      "logps/chosen": -0.671093761920929,
      "logps/rejected": -1.6572265625,
      "loss": 0.7557,
      "nll_loss": 0.71630859375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06705322116613388,
      "rewards/margins": 0.09859009087085724,
      "rewards/rejected": -0.16560058295726776,
      "step": 8020
    },
    {
      "epoch": 0.6093489148580968,
      "grad_norm": 3.057374669441468,
      "learning_rate": 8.927548420311771e-07,
      "log_odds_chosen": 1.4990234375,
      "log_odds_ratio": -0.3926757872104645,
      "logits/chosen": -1.005859375,
      "logits/rejected": -0.8863281011581421,
      "logps/chosen": -0.6031249761581421,
      "logps/rejected": -1.5988280773162842,
      "loss": 0.7314,
      "nll_loss": 0.7215820550918579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06029052659869194,
      "rewards/margins": 0.09958495944738388,
      "rewards/rejected": -0.1597900390625,
      "step": 8030
    },
    {
      "epoch": 0.6101077553498254,
      "grad_norm": 2.774171691721351,
      "learning_rate": 8.921994734909409e-07,
      "log_odds_chosen": 1.3442871570587158,
      "log_odds_ratio": -0.44404298067092896,
      "logits/chosen": -0.9140625,
      "logits/rejected": -0.859375,
      "logps/chosen": -0.675976574420929,
      "logps/rejected": -1.6130859851837158,
      "loss": 0.7615,
      "nll_loss": 0.731640636920929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06757812201976776,
      "rewards/margins": 0.09364929050207138,
      "rewards/rejected": -0.16123047471046448,
      "step": 8040
    },
    {
      "epoch": 0.6108665958415541,
      "grad_norm": 2.933552988268182,
      "learning_rate": 8.916451401205645e-07,
      "log_odds_chosen": 1.589257836341858,
      "log_odds_ratio": -0.3824706971645355,
      "logits/chosen": -1.013281226158142,
      "logits/rejected": -0.8515625,
      "logps/chosen": -0.640332043170929,
      "logps/rejected": -1.775781273841858,
      "loss": 0.7682,
      "nll_loss": 0.7958984375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06406249850988388,
      "rewards/margins": 0.11343993991613388,
      "rewards/rejected": -0.17744140326976776,
      "step": 8050
    },
    {
      "epoch": 0.6116254363332827,
      "grad_norm": 2.857921231159211,
      "learning_rate": 8.91091838708226e-07,
      "log_odds_chosen": 1.3947265148162842,
      "log_odds_ratio": -0.40986329317092896,
      "logits/chosen": -1.0076172351837158,
      "logits/rejected": -0.862500011920929,
      "logps/chosen": -0.652539074420929,
      "logps/rejected": -1.630859375,
      "loss": 0.7746,
      "nll_loss": 0.7197265625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06525878608226776,
      "rewards/margins": 0.09781493991613388,
      "rewards/rejected": -0.16306152939796448,
      "step": 8060
    },
    {
      "epoch": 0.6123842768250114,
      "grad_norm": 2.837229213150366,
      "learning_rate": 8.905395660560378e-07,
      "log_odds_chosen": 1.504296898841858,
      "log_odds_ratio": -0.385986328125,
      "logits/chosen": -0.9537109136581421,
      "logits/rejected": -0.8519531488418579,
      "logps/chosen": -0.6470702886581421,
      "logps/rejected": -1.6808593273162842,
      "loss": 0.7559,
      "nll_loss": 0.692675769329071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06475830078125,
      "rewards/margins": 0.10333251953125,
      "rewards/rejected": -0.16806641221046448,
      "step": 8070
    },
    {
      "epoch": 0.61314311731674,
      "grad_norm": 2.4942308815223537,
      "learning_rate": 8.899883189799695e-07,
      "log_odds_chosen": 1.417578101158142,
      "log_odds_ratio": -0.36015623807907104,
      "logits/chosen": -0.982617199420929,
      "logits/rejected": -0.87890625,
      "logps/chosen": -0.6324218511581421,
      "logps/rejected": -1.5691406726837158,
      "loss": 0.7358,
      "nll_loss": 0.7281249761581421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06330566108226776,
      "rewards/margins": 0.09346923977136612,
      "rewards/rejected": -0.1568603515625,
      "step": 8080
    },
    {
      "epoch": 0.6139019578084687,
      "grad_norm": 2.7121767516735242,
      "learning_rate": 8.894380943097694e-07,
      "log_odds_chosen": 1.674414038658142,
      "log_odds_ratio": -0.3631347715854645,
      "logits/chosen": -0.968554675579071,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.62158203125,
      "logps/rejected": -1.8341796398162842,
      "loss": 0.772,
      "nll_loss": 0.726855456829071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06214599683880806,
      "rewards/margins": 0.12135620415210724,
      "rewards/rejected": -0.18332520127296448,
      "step": 8090
    },
    {
      "epoch": 0.6146607983001973,
      "grad_norm": 2.390556532345029,
      "learning_rate": 8.888888888888888e-07,
      "log_odds_chosen": 1.4236328601837158,
      "log_odds_ratio": -0.4300781190395355,
      "logits/chosen": -0.9994140863418579,
      "logits/rejected": -0.893359363079071,
      "logps/chosen": -0.627246081829071,
      "logps/rejected": -1.621484398841858,
      "loss": 0.742,
      "nll_loss": 0.7144531011581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06267090141773224,
      "rewards/margins": 0.09934081882238388,
      "rewards/rejected": -0.16203613579273224,
      "step": 8100
    },
    {
      "epoch": 0.615419638791926,
      "grad_norm": 2.577970320731745,
      "learning_rate": 8.883406995744061e-07,
      "log_odds_chosen": 1.5,
      "log_odds_ratio": -0.419921875,
      "logits/chosen": -0.9488281011581421,
      "logits/rejected": -0.860156238079071,
      "logps/chosen": -0.6480468511581421,
      "logps/rejected": -1.7058594226837158,
      "loss": 0.7694,
      "nll_loss": 0.743359386920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06479492038488388,
      "rewards/margins": 0.10576172173023224,
      "rewards/rejected": -0.17062988877296448,
      "step": 8110
    },
    {
      "epoch": 0.6161784792836545,
      "grad_norm": 2.447946322540896,
      "learning_rate": 8.877935232369506e-07,
      "log_odds_chosen": 1.5734374523162842,
      "log_odds_ratio": -0.3714843690395355,
      "logits/chosen": -1.0812499523162842,
      "logits/rejected": -0.9462890625,
      "logps/chosen": -0.61181640625,
      "logps/rejected": -1.683203101158142,
      "loss": 0.7284,
      "nll_loss": 0.633496105670929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06105957180261612,
      "rewards/margins": 0.10723876953125,
      "rewards/rejected": -0.16831055283546448,
      "step": 8120
    },
    {
      "epoch": 0.6169373197753832,
      "grad_norm": 3.306034152526167,
      "learning_rate": 8.872473567606276e-07,
      "log_odds_chosen": 1.40234375,
      "log_odds_ratio": -0.3980956971645355,
      "logits/chosen": -0.969531238079071,
      "logits/rejected": -0.8519531488418579,
      "logps/chosen": -0.6717773675918579,
      "logps/rejected": -1.608789086341858,
      "loss": 0.7549,
      "nll_loss": 0.7886718511581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06724853813648224,
      "rewards/margins": 0.09385986626148224,
      "rewards/rejected": -0.16091307997703552,
      "step": 8130
    },
    {
      "epoch": 0.6176961602671118,
      "grad_norm": 3.149915089419751,
      "learning_rate": 8.867021970429453e-07,
      "log_odds_chosen": 1.9333984851837158,
      "log_odds_ratio": -0.29780274629592896,
      "logits/chosen": -0.9957031011581421,
      "logits/rejected": -0.889843761920929,
      "logps/chosen": -0.6250976324081421,
      "logps/rejected": -1.990820288658142,
      "loss": 0.737,
      "nll_loss": 0.663867175579071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.0625,
      "rewards/margins": 0.13642577826976776,
      "rewards/rejected": -0.19895020127296448,
      "step": 8140
    },
    {
      "epoch": 0.6184550007588405,
      "grad_norm": 2.7878921038913322,
      "learning_rate": 8.86158040994738e-07,
      "log_odds_chosen": 1.447363257408142,
      "log_odds_ratio": -0.3819824159145355,
      "logits/chosen": -1.0300781726837158,
      "logits/rejected": -0.919726550579071,
      "logps/chosen": -0.6255859136581421,
      "logps/rejected": -1.5720703601837158,
      "loss": 0.7494,
      "nll_loss": 0.727832019329071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06258545070886612,
      "rewards/margins": 0.09465332329273224,
      "rewards/rejected": -0.1572265625,
      "step": 8150
    },
    {
      "epoch": 0.6192138412505691,
      "grad_norm": 3.0476600032940997,
      "learning_rate": 8.856148855400954e-07,
      "log_odds_chosen": 1.740625023841858,
      "log_odds_ratio": -0.3084472715854645,
      "logits/chosen": -0.963085949420929,
      "logits/rejected": -0.841992199420929,
      "logps/chosen": -0.600292980670929,
      "logps/rejected": -1.8113281726837158,
      "loss": 0.7538,
      "nll_loss": 0.75,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06000976637005806,
      "rewards/margins": 0.12105713039636612,
      "rewards/rejected": -0.18115234375,
      "step": 8160
    },
    {
      "epoch": 0.6199726817422978,
      "grad_norm": 2.5533530605362635,
      "learning_rate": 8.850727276162873e-07,
      "log_odds_chosen": 1.46875,
      "log_odds_ratio": -0.4081054627895355,
      "logits/chosen": -1.0302734375,
      "logits/rejected": -0.8910156488418579,
      "logps/chosen": -0.679882824420929,
      "logps/rejected": -1.683984398841858,
      "loss": 0.7257,
      "nll_loss": 0.661914050579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06789550930261612,
      "rewards/margins": 0.10047607123851776,
      "rewards/rejected": -0.16835936903953552,
      "step": 8170
    },
    {
      "epoch": 0.6207315222340264,
      "grad_norm": 3.2848117284318272,
      "learning_rate": 8.845315641736929e-07,
      "log_odds_chosen": 1.520605444908142,
      "log_odds_ratio": -0.4037109315395355,
      "logits/chosen": -0.9814453125,
      "logits/rejected": -0.8617187738418579,
      "logps/chosen": -0.6546875238418579,
      "logps/rejected": -1.7429687976837158,
      "loss": 0.7459,
      "nll_loss": 0.7669922113418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06551513820886612,
      "rewards/margins": 0.10893554985523224,
      "rewards/rejected": -0.17431640625,
      "step": 8180
    },
    {
      "epoch": 0.6214903627257551,
      "grad_norm": 2.791595021621251,
      "learning_rate": 8.839913921757278e-07,
      "log_odds_chosen": 1.4835937023162842,
      "log_odds_ratio": -0.4066406190395355,
      "logits/chosen": -0.9996093511581421,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.615234375,
      "logps/rejected": -1.641015648841858,
      "loss": 0.7326,
      "nll_loss": 0.6529296636581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06151122972369194,
      "rewards/margins": 0.10253296047449112,
      "rewards/rejected": -0.1640625,
      "step": 8190
    },
    {
      "epoch": 0.6222492032174837,
      "grad_norm": 3.048746147281647,
      "learning_rate": 8.834522085987722e-07,
      "log_odds_chosen": 1.454199194908142,
      "log_odds_ratio": -0.4039062559604645,
      "logits/chosen": -1.033789038658142,
      "logits/rejected": -0.9232422113418579,
      "logps/chosen": -0.6869140863418579,
      "logps/rejected": -1.698632836341858,
      "loss": 0.7428,
      "nll_loss": 0.6998046636581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06870117038488388,
      "rewards/margins": 0.10119018703699112,
      "rewards/rejected": -0.169921875,
      "step": 8200
    },
    {
      "epoch": 0.6230080437092124,
      "grad_norm": 3.4131145719597917,
      "learning_rate": 8.829140104321008e-07,
      "log_odds_chosen": 1.7082030773162842,
      "log_odds_ratio": -0.35747069120407104,
      "logits/chosen": -0.9951171875,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.627246081829071,
      "logps/rejected": -1.8542969226837158,
      "loss": 0.7528,
      "nll_loss": 0.690136730670929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06275634467601776,
      "rewards/margins": 0.12246093899011612,
      "rewards/rejected": -0.185302734375,
      "step": 8210
    },
    {
      "epoch": 0.623766884200941,
      "grad_norm": 2.35008977369702,
      "learning_rate": 8.82376794677811e-07,
      "log_odds_chosen": 1.544824242591858,
      "log_odds_ratio": -0.3555664122104645,
      "logits/chosen": -1.0048828125,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.6446288824081421,
      "logps/rejected": -1.700781226158142,
      "loss": 0.7389,
      "nll_loss": 0.717089831829071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06447754055261612,
      "rewards/margins": 0.10557861626148224,
      "rewards/rejected": -0.17014160752296448,
      "step": 8220
    },
    {
      "epoch": 0.6245257246926696,
      "grad_norm": 2.644380570827783,
      "learning_rate": 8.818405583507537e-07,
      "log_odds_chosen": 1.7898437976837158,
      "log_odds_ratio": -0.3233886659145355,
      "logits/chosen": -0.9800781011581421,
      "logits/rejected": -0.845507800579071,
      "logps/chosen": -0.608691394329071,
      "logps/rejected": -1.8406250476837158,
      "loss": 0.7442,
      "nll_loss": 0.6659179925918579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06083984300494194,
      "rewards/margins": 0.12324218451976776,
      "rewards/rejected": -0.18422850966453552,
      "step": 8230
    },
    {
      "epoch": 0.6252845651843982,
      "grad_norm": 3.0243830901341244,
      "learning_rate": 8.813052984784634e-07,
      "log_odds_chosen": 1.5166015625,
      "log_odds_ratio": -0.40559083223342896,
      "logits/chosen": -0.9458984136581421,
      "logits/rejected": -0.8583984375,
      "logps/chosen": -0.616894543170929,
      "logps/rejected": -1.646875023841858,
      "loss": 0.7313,
      "nll_loss": 0.680468738079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06174316257238388,
      "rewards/margins": 0.102996826171875,
      "rewards/rejected": -0.16475829482078552,
      "step": 8240
    },
    {
      "epoch": 0.6260434056761269,
      "grad_norm": 3.2306600805265977,
      "learning_rate": 8.807710121010885e-07,
      "log_odds_chosen": 1.6636962890625,
      "log_odds_ratio": -0.35820311307907104,
      "logits/chosen": -0.9722656011581421,
      "logits/rejected": -0.835156261920929,
      "logps/chosen": -0.6083008050918579,
      "logps/rejected": -1.7146484851837158,
      "loss": 0.7438,
      "nll_loss": 0.730175793170929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06075439602136612,
      "rewards/margins": 0.11063003540039062,
      "rewards/rejected": -0.17136231064796448,
      "step": 8250
    },
    {
      "epoch": 0.6268022461678555,
      "grad_norm": 2.9141471270555765,
      "learning_rate": 8.802376962713231e-07,
      "log_odds_chosen": 1.4658203125,
      "log_odds_ratio": -0.36909180879592896,
      "logits/chosen": -0.9457031488418579,
      "logits/rejected": -0.824999988079071,
      "logps/chosen": -0.648632824420929,
      "logps/rejected": -1.61328125,
      "loss": 0.7468,
      "nll_loss": 0.7123047113418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06486816704273224,
      "rewards/margins": 0.09639892727136612,
      "rewards/rejected": -0.1611328125,
      "step": 8260
    },
    {
      "epoch": 0.6275610866595842,
      "grad_norm": 3.1878050398742612,
      "learning_rate": 8.797053480543386e-07,
      "log_odds_chosen": 1.7580077648162842,
      "log_odds_ratio": -0.3594970703125,
      "logits/chosen": -0.962890625,
      "logits/rejected": -0.8568359613418579,
      "logps/chosen": -0.630664050579071,
      "logps/rejected": -1.872460961341858,
      "loss": 0.7527,
      "nll_loss": 0.675000011920929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06308593600988388,
      "rewards/margins": 0.12403564155101776,
      "rewards/rejected": -0.18723145127296448,
      "step": 8270
    },
    {
      "epoch": 0.6283199271513128,
      "grad_norm": 3.0743558173968313,
      "learning_rate": 8.79173964527716e-07,
      "log_odds_chosen": 1.362939476966858,
      "log_odds_ratio": -0.37578123807907104,
      "logits/chosen": -0.9712890386581421,
      "logits/rejected": -0.8773437738418579,
      "logps/chosen": -0.5775390863418579,
      "logps/rejected": -1.437890648841858,
      "loss": 0.726,
      "nll_loss": 0.6591796875,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05767822265625,
      "rewards/margins": 0.08607711642980576,
      "rewards/rejected": -0.14382323622703552,
      "step": 8280
    },
    {
      "epoch": 0.6290787676430414,
      "grad_norm": 2.651832353171311,
      "learning_rate": 8.78643542781378e-07,
      "log_odds_chosen": 1.3501465320587158,
      "log_odds_ratio": -0.42548829317092896,
      "logits/chosen": -0.982617199420929,
      "logits/rejected": -0.891796886920929,
      "logps/chosen": -0.633105456829071,
      "logps/rejected": -1.5535156726837158,
      "loss": 0.7413,
      "nll_loss": 0.7281249761581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06330566108226776,
      "rewards/margins": 0.09206543117761612,
      "rewards/rejected": -0.15541991591453552,
      "step": 8290
    },
    {
      "epoch": 0.6298376081347701,
      "grad_norm": 2.9512012017296425,
      "learning_rate": 8.781140799175228e-07,
      "log_odds_chosen": 1.624414086341858,
      "log_odds_ratio": -0.3788085877895355,
      "logits/chosen": -0.9400390386581421,
      "logits/rejected": -0.8447265625,
      "logps/chosen": -0.6796875,
      "logps/rejected": -1.859765648841858,
      "loss": 0.7517,
      "nll_loss": 0.7349609136581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06798096001148224,
      "rewards/margins": 0.11801757663488388,
      "rewards/rejected": -0.18603515625,
      "step": 8300
    },
    {
      "epoch": 0.6305964486264987,
      "grad_norm": 2.6873314333789105,
      "learning_rate": 8.775855730505568e-07,
      "log_odds_chosen": 1.347900390625,
      "log_odds_ratio": -0.4040771424770355,
      "logits/chosen": -0.990234375,
      "logits/rejected": -0.871874988079071,
      "logps/chosen": -0.6451171636581421,
      "logps/rejected": -1.57421875,
      "loss": 0.7653,
      "nll_loss": 0.7845703363418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06453857570886612,
      "rewards/margins": 0.09277953952550888,
      "rewards/rejected": -0.15729980170726776,
      "step": 8310
    },
    {
      "epoch": 0.6313552891182274,
      "grad_norm": 2.597865177201747,
      "learning_rate": 8.770580193070291e-07,
      "log_odds_chosen": 1.535253882408142,
      "log_odds_ratio": -0.4124999940395355,
      "logits/chosen": -1.043359398841858,
      "logits/rejected": -0.915234386920929,
      "logps/chosen": -0.6656249761581421,
      "logps/rejected": -1.754296898841858,
      "loss": 0.7431,
      "nll_loss": 0.694531261920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06660155951976776,
      "rewards/margins": 0.10898437350988388,
      "rewards/rejected": -0.17539063096046448,
      "step": 8320
    },
    {
      "epoch": 0.6321141296099559,
      "grad_norm": 2.6636255315470523,
      "learning_rate": 8.765314158255661e-07,
      "log_odds_chosen": 1.629296898841858,
      "log_odds_ratio": -0.35380858182907104,
      "logits/chosen": -1.015039086341858,
      "logits/rejected": -0.900390625,
      "logps/chosen": -0.60302734375,
      "logps/rejected": -1.716406226158142,
      "loss": 0.7289,
      "nll_loss": 0.688183605670929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06025390699505806,
      "rewards/margins": 0.11134032905101776,
      "rewards/rejected": -0.17172852158546448,
      "step": 8330
    },
    {
      "epoch": 0.6328729701016846,
      "grad_norm": 3.086899896839651,
      "learning_rate": 8.760057597568057e-07,
      "log_odds_chosen": 1.525781273841858,
      "log_odds_ratio": -0.3667968809604645,
      "logits/chosen": -0.9691406488418579,
      "logits/rejected": -0.85546875,
      "logps/chosen": -0.6664062738418579,
      "logps/rejected": -1.6960937976837158,
      "loss": 0.7533,
      "nll_loss": 0.7625976800918579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06668701022863388,
      "rewards/margins": 0.10299072414636612,
      "rewards/rejected": -0.16960449516773224,
      "step": 8340
    },
    {
      "epoch": 0.6336318105934132,
      "grad_norm": 2.9739790332142473,
      "learning_rate": 8.754810482633324e-07,
      "log_odds_chosen": 1.609960913658142,
      "log_odds_ratio": -0.37421876192092896,
      "logits/chosen": -0.9654296636581421,
      "logits/rejected": -0.863085925579071,
      "logps/chosen": -0.571484386920929,
      "logps/rejected": -1.644921898841858,
      "loss": 0.7429,
      "nll_loss": 0.6396484375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.05717773362994194,
      "rewards/margins": 0.10746459662914276,
      "rewards/rejected": -0.16452637314796448,
      "step": 8350
    },
    {
      "epoch": 0.6343906510851419,
      "grad_norm": 2.9560606544370613,
      "learning_rate": 8.749572785196142e-07,
      "log_odds_chosen": 1.9353516101837158,
      "log_odds_ratio": -0.28803712129592896,
      "logits/chosen": -1.0578124523162842,
      "logits/rejected": -0.893359363079071,
      "logps/chosen": -0.5663086175918579,
      "logps/rejected": -1.8341796398162842,
      "loss": 0.7365,
      "nll_loss": 0.625,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.05665283277630806,
      "rewards/margins": 0.12680664658546448,
      "rewards/rejected": -0.18339844048023224,
      "step": 8360
    },
    {
      "epoch": 0.6351494915768705,
      "grad_norm": 2.568163559209062,
      "learning_rate": 8.744344477119373e-07,
      "log_odds_chosen": 1.5021483898162842,
      "log_odds_ratio": -0.41474610567092896,
      "logits/chosen": -0.970898449420929,
      "logits/rejected": -0.892382800579071,
      "logps/chosen": -0.631152331829071,
      "logps/rejected": -1.679296851158142,
      "loss": 0.7646,
      "nll_loss": 0.6807616949081421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06303711235523224,
      "rewards/margins": 0.10488434135913849,
      "rewards/rejected": -0.16787108778953552,
      "step": 8370
    },
    {
      "epoch": 0.6359083320685992,
      "grad_norm": 3.0448048864642905,
      "learning_rate": 8.739125530383433e-07,
      "log_odds_chosen": 1.4322998523712158,
      "log_odds_ratio": -0.4263671934604645,
      "logits/chosen": -0.9722656011581421,
      "logits/rejected": -0.853515625,
      "logps/chosen": -0.6661132574081421,
      "logps/rejected": -1.634179711341858,
      "loss": 0.751,
      "nll_loss": 0.7247070074081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06662597507238388,
      "rewards/margins": 0.09671630710363388,
      "rewards/rejected": -0.16340331733226776,
      "step": 8380
    },
    {
      "epoch": 0.6366671725603278,
      "grad_norm": 3.1788330711321717,
      "learning_rate": 8.733915917085661e-07,
      "log_odds_chosen": 1.391943335533142,
      "log_odds_ratio": -0.4283203184604645,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.876757800579071,
      "logps/chosen": -0.64208984375,
      "logps/rejected": -1.5900390148162842,
      "loss": 0.741,
      "nll_loss": 0.728320300579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06422118842601776,
      "rewards/margins": 0.09482421725988388,
      "rewards/rejected": -0.15903320908546448,
      "step": 8390
    },
    {
      "epoch": 0.6374260130520565,
      "grad_norm": 2.750893058680582,
      "learning_rate": 8.728715609439695e-07,
      "log_odds_chosen": 1.3787109851837158,
      "log_odds_ratio": -0.4112304747104645,
      "logits/chosen": -0.9349609613418579,
      "logits/rejected": -0.852734386920929,
      "logps/chosen": -0.600878894329071,
      "logps/rejected": -1.517968773841858,
      "loss": 0.7427,
      "nll_loss": 0.7120116949081421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06007080152630806,
      "rewards/margins": 0.09171142429113388,
      "rewards/rejected": -0.15180663764476776,
      "step": 8400
    },
    {
      "epoch": 0.6381848535437851,
      "grad_norm": 3.0718974356666156,
      "learning_rate": 8.72352457977484e-07,
      "log_odds_chosen": 1.9513671398162842,
      "log_odds_ratio": -0.3036254942417145,
      "logits/chosen": -0.9458984136581421,
      "logits/rejected": -0.851367175579071,
      "logps/chosen": -0.6241210699081421,
      "logps/rejected": -1.9650390148162842,
      "loss": 0.7135,
      "nll_loss": 0.69970703125,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06243896484375,
      "rewards/margins": 0.133941650390625,
      "rewards/rejected": -0.19648437201976776,
      "step": 8410
    },
    {
      "epoch": 0.6389436940355138,
      "grad_norm": 3.311457488876305,
      "learning_rate": 8.718342800535456e-07,
      "log_odds_chosen": 1.5027344226837158,
      "log_odds_ratio": -0.4173828065395355,
      "logits/chosen": -1.012109398841858,
      "logits/rejected": -0.901562511920929,
      "logps/chosen": -0.666308581829071,
      "logps/rejected": -1.7080078125,
      "loss": 0.7208,
      "nll_loss": 0.6729491949081421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06658935546875,
      "rewards/margins": 0.10408935695886612,
      "rewards/rejected": -0.170654296875,
      "step": 8420
    },
    {
      "epoch": 0.6397025345272424,
      "grad_norm": 3.262866200217403,
      "learning_rate": 8.713170244280353e-07,
      "log_odds_chosen": 1.824609398841858,
      "log_odds_ratio": -0.34624022245407104,
      "logits/chosen": -0.997265636920929,
      "logits/rejected": -0.908007800579071,
      "logps/chosen": -0.6078125238418579,
      "logps/rejected": -1.884374976158142,
      "loss": 0.7523,
      "nll_loss": 0.7369140386581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06077880784869194,
      "rewards/margins": 0.12769775092601776,
      "rewards/rejected": -0.18852539360523224,
      "step": 8430
    },
    {
      "epoch": 0.640461375018971,
      "grad_norm": 2.8713086281177147,
      "learning_rate": 8.708006883682162e-07,
      "log_odds_chosen": 1.7332031726837158,
      "log_odds_ratio": -0.3373046815395355,
      "logits/chosen": -1.033593773841858,
      "logits/rejected": -0.886914074420929,
      "logps/chosen": -0.6158202886581421,
      "logps/rejected": -1.820703148841858,
      "loss": 0.7409,
      "nll_loss": 0.629199206829071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06154785305261612,
      "rewards/margins": 0.12059326469898224,
      "rewards/rejected": -0.18203124403953552,
      "step": 8440
    },
    {
      "epoch": 0.6412202155106996,
      "grad_norm": 2.9310215190540623,
      "learning_rate": 8.702852691526739e-07,
      "log_odds_chosen": 1.4729492664337158,
      "log_odds_ratio": -0.396728515625,
      "logits/chosen": -0.963085949420929,
      "logits/rejected": -0.8636718988418579,
      "logps/chosen": -0.599804699420929,
      "logps/rejected": -1.566015601158142,
      "loss": 0.7518,
      "nll_loss": 0.7314453125,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.05996093899011612,
      "rewards/margins": 0.09669189155101776,
      "rewards/rejected": -0.15666504204273224,
      "step": 8450
    },
    {
      "epoch": 0.6419790560024283,
      "grad_norm": 2.9271110648833347,
      "learning_rate": 8.697707640712562e-07,
      "log_odds_chosen": 1.4552733898162842,
      "log_odds_ratio": -0.3949218690395355,
      "logits/chosen": -0.966992199420929,
      "logits/rejected": -0.859570324420929,
      "logps/chosen": -0.6366211175918579,
      "logps/rejected": -1.603124976158142,
      "loss": 0.726,
      "nll_loss": 0.683886706829071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06368408352136612,
      "rewards/margins": 0.09665527194738388,
      "rewards/rejected": -0.16030272841453552,
      "step": 8460
    },
    {
      "epoch": 0.6427378964941569,
      "grad_norm": 3.6924975175153034,
      "learning_rate": 8.692571704250135e-07,
      "log_odds_chosen": 1.582373023033142,
      "log_odds_ratio": -0.40437012910842896,
      "logits/chosen": -0.9736328125,
      "logits/rejected": -0.8726562261581421,
      "logps/chosen": -0.6644531488418579,
      "logps/rejected": -1.7654297351837158,
      "loss": 0.7284,
      "nll_loss": 0.6695312261581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06644286960363388,
      "rewards/margins": 0.1100616455078125,
      "rewards/rejected": -0.17658691108226776,
      "step": 8470
    },
    {
      "epoch": 0.6434967369858856,
      "grad_norm": 2.677405253580464,
      "learning_rate": 8.687444855261388e-07,
      "log_odds_chosen": 1.5798828601837158,
      "log_odds_ratio": -0.34428709745407104,
      "logits/chosen": -1.045507788658142,
      "logits/rejected": -0.908007800579071,
      "logps/chosen": -0.67431640625,
      "logps/rejected": -1.753515601158142,
      "loss": 0.7573,
      "nll_loss": 0.7320312261581421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06737060844898224,
      "rewards/margins": 0.10798339545726776,
      "rewards/rejected": -0.17536620795726776,
      "step": 8480
    },
    {
      "epoch": 0.6442555774776142,
      "grad_norm": 2.8686430304565627,
      "learning_rate": 8.682327066979084e-07,
      "log_odds_chosen": 1.7863280773162842,
      "log_odds_ratio": -0.35395509004592896,
      "logits/chosen": -1.0158202648162842,
      "logits/rejected": -0.909960925579071,
      "logps/chosen": -0.6407226324081421,
      "logps/rejected": -1.8449218273162842,
      "loss": 0.7105,
      "nll_loss": 0.664843738079071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06411132961511612,
      "rewards/margins": 0.12052612006664276,
      "rewards/rejected": -0.18452148139476776,
      "step": 8490
    },
    {
      "epoch": 0.6450144179693429,
      "grad_norm": 3.535271780732417,
      "learning_rate": 8.677218312746247e-07,
      "log_odds_chosen": 1.3644530773162842,
      "log_odds_ratio": -0.42656248807907104,
      "logits/chosen": -0.934374988079071,
      "logits/rejected": -0.867968738079071,
      "logps/chosen": -0.6304687261581421,
      "logps/rejected": -1.5177733898162842,
      "loss": 0.7168,
      "nll_loss": 0.693164050579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06307373195886612,
      "rewards/margins": 0.088623046875,
      "rewards/rejected": -0.15166015923023224,
      "step": 8500
    },
    {
      "epoch": 0.6457732584610715,
      "grad_norm": 2.5067323346276043,
      "learning_rate": 8.672118566015558e-07,
      "log_odds_chosen": 1.441992163658142,
      "log_odds_ratio": -0.384033203125,
      "logits/chosen": -0.9912109375,
      "logits/rejected": -0.8990234136581421,
      "logps/chosen": -0.68408203125,
      "logps/rejected": -1.667382836341858,
      "loss": 0.7498,
      "nll_loss": 0.703906238079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06840820610523224,
      "rewards/margins": 0.09831543266773224,
      "rewards/rejected": -0.16667480766773224,
      "step": 8510
    },
    {
      "epoch": 0.6465320989528002,
      "grad_norm": 2.6543785612719284,
      "learning_rate": 8.667027800348789e-07,
      "log_odds_chosen": 1.536718726158142,
      "log_odds_ratio": -0.420166015625,
      "logits/chosen": -1.006445288658142,
      "logits/rejected": -0.896679699420929,
      "logps/chosen": -0.6268554925918579,
      "logps/rejected": -1.687109351158142,
      "loss": 0.737,
      "nll_loss": 0.7401367425918579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06271972507238388,
      "rewards/margins": 0.10617981106042862,
      "rewards/rejected": -0.16877441108226776,
      "step": 8520
    },
    {
      "epoch": 0.6472909394445288,
      "grad_norm": 2.762159306136293,
      "learning_rate": 8.661945989416229e-07,
      "log_odds_chosen": 1.3839843273162842,
      "log_odds_ratio": -0.38188475370407104,
      "logits/chosen": -1.0050780773162842,
      "logits/rejected": -0.916210949420929,
      "logps/chosen": -0.634472668170929,
      "logps/rejected": -1.5574219226837158,
      "loss": 0.7218,
      "nll_loss": 0.675488293170929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06340332329273224,
      "rewards/margins": 0.09228515625,
      "rewards/rejected": -0.15581054985523224,
      "step": 8530
    },
    {
      "epoch": 0.6480497799362575,
      "grad_norm": 2.8951048651776854,
      "learning_rate": 8.6568731069961e-07,
      "log_odds_chosen": 1.61328125,
      "log_odds_ratio": -0.40544432401657104,
      "logits/chosen": -0.9619140625,
      "logits/rejected": -0.884570300579071,
      "logps/chosen": -0.647656261920929,
      "logps/rejected": -1.798828125,
      "loss": 0.7604,
      "nll_loss": 0.7176758050918579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06474609673023224,
      "rewards/margins": 0.11499633640050888,
      "rewards/rejected": -0.17973633110523224,
      "step": 8540
    },
    {
      "epoch": 0.648808620427986,
      "grad_norm": 2.3552558261504704,
      "learning_rate": 8.651809126974002e-07,
      "log_odds_chosen": 1.507226586341858,
      "log_odds_ratio": -0.3956542909145355,
      "logits/chosen": -1.0255858898162842,
      "logits/rejected": -0.929492175579071,
      "logps/chosen": -0.6234375238418579,
      "logps/rejected": -1.64453125,
      "loss": 0.7381,
      "nll_loss": 0.7227538824081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06236572191119194,
      "rewards/margins": 0.10211181640625,
      "rewards/rejected": -0.1644287109375,
      "step": 8550
    },
    {
      "epoch": 0.6495674609197146,
      "grad_norm": 3.096614392145739,
      "learning_rate": 8.646754023342339e-07,
      "log_odds_chosen": 1.7849609851837158,
      "log_odds_ratio": -0.3517089784145355,
      "logits/chosen": -1.0060546398162842,
      "logits/rejected": -0.908007800579071,
      "logps/chosen": -0.6517578363418579,
      "logps/rejected": -1.9578125476837158,
      "loss": 0.731,
      "nll_loss": 0.7100585699081421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06523437798023224,
      "rewards/margins": 0.13050536811351776,
      "rewards/rejected": -0.19589844346046448,
      "step": 8560
    },
    {
      "epoch": 0.6503263014114433,
      "grad_norm": 3.16796873792443,
      "learning_rate": 8.64170777019976e-07,
      "log_odds_chosen": 1.470434546470642,
      "log_odds_ratio": -0.419921875,
      "logits/chosen": -0.982226550579071,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.679980456829071,
      "logps/rejected": -1.7218749523162842,
      "loss": 0.7338,
      "nll_loss": 0.739062488079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06800536811351776,
      "rewards/margins": 0.10419921576976776,
      "rewards/rejected": -0.17219237983226776,
      "step": 8570
    },
    {
      "epoch": 0.6510851419031719,
      "grad_norm": 2.6695115140682044,
      "learning_rate": 8.636670341750609e-07,
      "log_odds_chosen": 1.385839819908142,
      "log_odds_ratio": -0.4066406190395355,
      "logits/chosen": -1.0009765625,
      "logits/rejected": -0.8802734613418579,
      "logps/chosen": -0.6244140863418579,
      "logps/rejected": -1.5412108898162842,
      "loss": 0.7031,
      "nll_loss": 0.6615234613418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06245117262005806,
      "rewards/margins": 0.0916748046875,
      "rewards/rejected": -0.15415039658546448,
      "step": 8580
    },
    {
      "epoch": 0.6518439823949006,
      "grad_norm": 3.1696301000680327,
      "learning_rate": 8.631641712304359e-07,
      "log_odds_chosen": 1.8359375,
      "log_odds_ratio": -0.33056640625,
      "logits/chosen": -1.0232422351837158,
      "logits/rejected": -0.9052734375,
      "logps/chosen": -0.6185547113418579,
      "logps/rejected": -1.9460937976837158,
      "loss": 0.7555,
      "nll_loss": 0.686230480670929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06185302883386612,
      "rewards/margins": 0.13284912705421448,
      "rewards/rejected": -0.19472655653953552,
      "step": 8590
    },
    {
      "epoch": 0.6526028228866292,
      "grad_norm": 2.8968920212170555,
      "learning_rate": 8.626621856275073e-07,
      "log_odds_chosen": 1.299218773841858,
      "log_odds_ratio": -0.4683593809604645,
      "logits/chosen": -0.9886718988418579,
      "logits/rejected": -0.884765625,
      "logps/chosen": -0.6806640625,
      "logps/rejected": -1.560156226158142,
      "loss": 0.7466,
      "nll_loss": 0.73974609375,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06805419921875,
      "rewards/margins": 0.08784179389476776,
      "rewards/rejected": -0.15593262016773224,
      "step": 8600
    },
    {
      "epoch": 0.6533616633783579,
      "grad_norm": 2.552751142554736,
      "learning_rate": 8.621610748180847e-07,
      "log_odds_chosen": 1.638769507408142,
      "log_odds_ratio": -0.3624511659145355,
      "logits/chosen": -1.0068359375,
      "logits/rejected": -0.897656261920929,
      "logps/chosen": -0.631152331829071,
      "logps/rejected": -1.762109398841858,
      "loss": 0.734,
      "nll_loss": 0.666210949420929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06315918266773224,
      "rewards/margins": 0.11306152492761612,
      "rewards/rejected": -0.17631836235523224,
      "step": 8610
    },
    {
      "epoch": 0.6541205038700865,
      "grad_norm": 2.753626232114711,
      "learning_rate": 8.616608362643274e-07,
      "log_odds_chosen": 1.6326172351837158,
      "log_odds_ratio": -0.3794189393520355,
      "logits/chosen": -1.000585913658142,
      "logits/rejected": -0.8974609375,
      "logps/chosen": -0.686328113079071,
      "logps/rejected": -1.773046851158142,
      "loss": 0.7381,
      "nll_loss": 0.7484375238418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06859131157398224,
      "rewards/margins": 0.10865478217601776,
      "rewards/rejected": -0.17722168564796448,
      "step": 8620
    },
    {
      "epoch": 0.6548793443618152,
      "grad_norm": 2.689379678521682,
      "learning_rate": 8.611614674386904e-07,
      "log_odds_chosen": 1.426660180091858,
      "log_odds_ratio": -0.38823240995407104,
      "logits/chosen": -0.972460925579071,
      "logits/rejected": -0.898242175579071,
      "logps/chosen": -0.6400390863418579,
      "logps/rejected": -1.603124976158142,
      "loss": 0.7334,
      "nll_loss": 0.6958984136581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06398925930261612,
      "rewards/margins": 0.09627075493335724,
      "rewards/rejected": -0.16020508110523224,
      "step": 8630
    },
    {
      "epoch": 0.6556381848535437,
      "grad_norm": 2.677306624529923,
      "learning_rate": 8.606629658238703e-07,
      "log_odds_chosen": 1.325585961341858,
      "log_odds_ratio": -0.40888673067092896,
      "logits/chosen": -0.9624999761581421,
      "logits/rejected": -0.8707031011581421,
      "logps/chosen": -0.620410144329071,
      "logps/rejected": -1.503515601158142,
      "loss": 0.7296,
      "nll_loss": 0.6761718988418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06203613430261612,
      "rewards/margins": 0.08827514946460724,
      "rewards/rejected": -0.15034179389476776,
      "step": 8640
    },
    {
      "epoch": 0.6563970253452724,
      "grad_norm": 2.895149978178067,
      "learning_rate": 8.601653289127525e-07,
      "log_odds_chosen": 1.5421874523162842,
      "log_odds_ratio": -0.3878417909145355,
      "logits/chosen": -1.0207030773162842,
      "logits/rejected": -0.9361327886581421,
      "logps/chosen": -0.608203113079071,
      "logps/rejected": -1.676367163658142,
      "loss": 0.7097,
      "nll_loss": 0.677734375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06082763522863388,
      "rewards/margins": 0.10679320991039276,
      "rewards/rejected": -0.16767577826976776,
      "step": 8650
    },
    {
      "epoch": 0.657155865837001,
      "grad_norm": 2.753880165644991,
      "learning_rate": 8.596685542083577e-07,
      "log_odds_chosen": 1.6876952648162842,
      "log_odds_ratio": -0.3546386659145355,
      "logits/chosen": -1.0119140148162842,
      "logits/rejected": -0.9166015386581421,
      "logps/chosen": -0.5708984136581421,
      "logps/rejected": -1.743749976158142,
      "loss": 0.7311,
      "nll_loss": 0.6666015386581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05710449069738388,
      "rewards/margins": 0.11738280951976776,
      "rewards/rejected": -0.17441406846046448,
      "step": 8660
    },
    {
      "epoch": 0.6579147063287297,
      "grad_norm": 2.9148157983710514,
      "learning_rate": 8.591726392237899e-07,
      "log_odds_chosen": 1.8271484375,
      "log_odds_ratio": -0.36079102754592896,
      "logits/chosen": -0.9859374761581421,
      "logits/rejected": -0.8843749761581421,
      "logps/chosen": -0.6114257574081421,
      "logps/rejected": -1.8933594226837158,
      "loss": 0.742,
      "nll_loss": 0.689746081829071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06110839918255806,
      "rewards/margins": 0.12818603217601776,
      "rewards/rejected": -0.18935546278953552,
      "step": 8670
    },
    {
      "epoch": 0.6586735468204583,
      "grad_norm": 2.7111935790151516,
      "learning_rate": 8.586775814821837e-07,
      "log_odds_chosen": 1.3165161609649658,
      "log_odds_ratio": -0.46293944120407104,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.8892577886581421,
      "logps/chosen": -0.6864258050918579,
      "logps/rejected": -1.635156273841858,
      "loss": 0.7611,
      "nll_loss": 0.7523437738418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06866455078125,
      "rewards/margins": 0.09486083686351776,
      "rewards/rejected": -0.163330078125,
      "step": 8680
    },
    {
      "epoch": 0.659432387312187,
      "grad_norm": 2.6280862213836933,
      "learning_rate": 8.58183378516652e-07,
      "log_odds_chosen": 1.5119140148162842,
      "log_odds_ratio": -0.4378906190395355,
      "logits/chosen": -0.971875011920929,
      "logits/rejected": -0.873046875,
      "logps/chosen": -0.630175769329071,
      "logps/rejected": -1.699609398841858,
      "loss": 0.7368,
      "nll_loss": 0.674023449420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06298828125,
      "rewards/margins": 0.107025146484375,
      "rewards/rejected": -0.17001953721046448,
      "step": 8690
    },
    {
      "epoch": 0.6601912278039156,
      "grad_norm": 2.720790730426448,
      "learning_rate": 8.576900278702358e-07,
      "log_odds_chosen": 1.4738280773162842,
      "log_odds_ratio": -0.39497071504592896,
      "logits/chosen": -1.0126953125,
      "logits/rejected": -0.9164062738418579,
      "logps/chosen": -0.6498047113418579,
      "logps/rejected": -1.675390601158142,
      "loss": 0.6992,
      "nll_loss": 0.6756836175918579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06499023735523224,
      "rewards/margins": 0.10251464694738388,
      "rewards/rejected": -0.16745606064796448,
      "step": 8700
    },
    {
      "epoch": 0.6609500682956443,
      "grad_norm": 2.5856309339344405,
      "learning_rate": 8.57197527095851e-07,
      "log_odds_chosen": 1.5299804210662842,
      "log_odds_ratio": -0.39042967557907104,
      "logits/chosen": -0.9970703125,
      "logits/rejected": -0.8843749761581421,
      "logps/chosen": -0.611523449420929,
      "logps/rejected": -1.680078148841858,
      "loss": 0.727,
      "nll_loss": 0.6763671636581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06114501878619194,
      "rewards/margins": 0.10677490383386612,
      "rewards/rejected": -0.16816405951976776,
      "step": 8710
    },
    {
      "epoch": 0.6617089087873729,
      "grad_norm": 2.64453440804158,
      "learning_rate": 8.567058737562385e-07,
      "log_odds_chosen": 1.338964819908142,
      "log_odds_ratio": -0.45068359375,
      "logits/chosen": -1.004492163658142,
      "logits/rejected": -0.904296875,
      "logps/chosen": -0.6468750238418579,
      "logps/rejected": -1.526757836341858,
      "loss": 0.7073,
      "nll_loss": 0.6474609375,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06473388522863388,
      "rewards/margins": 0.08798141777515411,
      "rewards/rejected": -0.15273436903953552,
      "step": 8720
    },
    {
      "epoch": 0.6624677492791016,
      "grad_norm": 2.405005952527545,
      "learning_rate": 8.562150654239141e-07,
      "log_odds_chosen": 1.528906226158142,
      "log_odds_ratio": -0.399658203125,
      "logits/chosen": -1.008398413658142,
      "logits/rejected": -0.908203125,
      "logps/chosen": -0.65576171875,
      "logps/rejected": -1.742578148841858,
      "loss": 0.7386,
      "nll_loss": 0.7059570550918579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06556396186351776,
      "rewards/margins": 0.10859374701976776,
      "rewards/rejected": -0.17434081435203552,
      "step": 8730
    },
    {
      "epoch": 0.6632265897708302,
      "grad_norm": 2.6175396641657516,
      "learning_rate": 8.55725099681116e-07,
      "log_odds_chosen": 1.3904297351837158,
      "log_odds_ratio": -0.4146484434604645,
      "logits/chosen": -1.0255858898162842,
      "logits/rejected": -0.9332031011581421,
      "logps/chosen": -0.654492199420929,
      "logps/rejected": -1.6095702648162842,
      "loss": 0.7259,
      "nll_loss": 0.6895507574081421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06549072265625,
      "rewards/margins": 0.09559936821460724,
      "rewards/rejected": -0.16103515028953552,
      "step": 8740
    },
    {
      "epoch": 0.6639854302625589,
      "grad_norm": 2.6906800972208758,
      "learning_rate": 8.552359741197579e-07,
      "log_odds_chosen": 1.6115233898162842,
      "log_odds_ratio": -0.38212889432907104,
      "logits/chosen": -1.016015648841858,
      "logits/rejected": -0.901562511920929,
      "logps/chosen": -0.61474609375,
      "logps/rejected": -1.6902344226837158,
      "loss": 0.7212,
      "nll_loss": 0.647656261920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06151122972369194,
      "rewards/margins": 0.10777588188648224,
      "rewards/rejected": -0.16923828423023224,
      "step": 8750
    },
    {
      "epoch": 0.6647442707542874,
      "grad_norm": 3.1965397296377285,
      "learning_rate": 8.547476863413765e-07,
      "log_odds_chosen": 1.517968773841858,
      "log_odds_ratio": -0.375732421875,
      "logits/chosen": -1.025781273841858,
      "logits/rejected": -0.910351574420929,
      "logps/chosen": -0.626953125,
      "logps/rejected": -1.685156226158142,
      "loss": 0.721,
      "nll_loss": 0.61767578125,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06269530951976776,
      "rewards/margins": 0.10578612983226776,
      "rewards/rejected": -0.16845703125,
      "step": 8760
    },
    {
      "epoch": 0.6655031112460161,
      "grad_norm": 3.1745774121709993,
      "learning_rate": 8.54260233957083e-07,
      "log_odds_chosen": 1.410253882408142,
      "log_odds_ratio": -0.41484373807907104,
      "logits/chosen": -1.0271484851837158,
      "logits/rejected": -0.905078113079071,
      "logps/chosen": -0.639941394329071,
      "logps/rejected": -1.6300780773162842,
      "loss": 0.7232,
      "nll_loss": 0.722460925579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06402587890625,
      "rewards/margins": 0.09885253757238388,
      "rewards/rejected": -0.16291503608226776,
      "step": 8770
    },
    {
      "epoch": 0.6662619517377447,
      "grad_norm": 3.3473736434386168,
      "learning_rate": 8.537736145875154e-07,
      "log_odds_chosen": 1.4734375476837158,
      "log_odds_ratio": -0.4004882872104645,
      "logits/chosen": -1.0300781726837158,
      "logits/rejected": -0.8910156488418579,
      "logps/chosen": -0.65380859375,
      "logps/rejected": -1.6845703125,
      "loss": 0.7195,
      "nll_loss": 0.731640636920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06539306789636612,
      "rewards/margins": 0.10313721001148224,
      "rewards/rejected": -0.16843262314796448,
      "step": 8780
    },
    {
      "epoch": 0.6670207922294734,
      "grad_norm": 2.7101377956102914,
      "learning_rate": 8.532878258627874e-07,
      "log_odds_chosen": 1.5070312023162842,
      "log_odds_ratio": -0.39033204317092896,
      "logits/chosen": -1.0041015148162842,
      "logits/rejected": -0.8978515863418579,
      "logps/chosen": -0.6019531488418579,
      "logps/rejected": -1.6140625476837158,
      "loss": 0.7291,
      "nll_loss": 0.70166015625,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06022949144244194,
      "rewards/margins": 0.10114135593175888,
      "rewards/rejected": -0.161376953125,
      "step": 8790
    },
    {
      "epoch": 0.667779632721202,
      "grad_norm": 3.1457287417296307,
      "learning_rate": 8.528028654224416e-07,
      "log_odds_chosen": 1.606835961341858,
      "log_odds_ratio": -0.35566407442092896,
      "logits/chosen": -1.040624976158142,
      "logits/rejected": -0.895703136920929,
      "logps/chosen": -0.637988269329071,
      "logps/rejected": -1.7429687976837158,
      "loss": 0.7173,
      "nll_loss": 0.7562500238418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06373290717601776,
      "rewards/margins": 0.11043701320886612,
      "rewards/rejected": -0.1741943359375,
      "step": 8800
    },
    {
      "epoch": 0.6685384732129307,
      "grad_norm": 2.4979785314087897,
      "learning_rate": 8.523187309154008e-07,
      "log_odds_chosen": 1.6749999523162842,
      "log_odds_ratio": -0.3563476502895355,
      "logits/chosen": -1.016992211341858,
      "logits/rejected": -0.9253906011581421,
      "logps/chosen": -0.6712890863418579,
      "logps/rejected": -1.8273437023162842,
      "loss": 0.7068,
      "nll_loss": 0.613574206829071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06718750298023224,
      "rewards/margins": 0.11564941704273224,
      "rewards/rejected": -0.18283692002296448,
      "step": 8810
    },
    {
      "epoch": 0.6692973137046593,
      "grad_norm": 2.89145165275318,
      "learning_rate": 8.518354199999198e-07,
      "log_odds_chosen": 1.4490234851837158,
      "log_odds_ratio": -0.36884766817092896,
      "logits/chosen": -1.058007836341858,
      "logits/rejected": -0.927734375,
      "logps/chosen": -0.631542980670929,
      "logps/rejected": -1.5812499523162842,
      "loss": 0.7171,
      "nll_loss": 0.643359363079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06312255561351776,
      "rewards/margins": 0.09495849907398224,
      "rewards/rejected": -0.15803222358226776,
      "step": 8820
    },
    {
      "epoch": 0.6700561541963879,
      "grad_norm": 2.7949277153698695,
      "learning_rate": 8.513529303435386e-07,
      "log_odds_chosen": 1.6535155773162842,
      "log_odds_ratio": -0.3829101622104645,
      "logits/chosen": -1.033203125,
      "logits/rejected": -0.938281238079071,
      "logps/chosen": -0.6231445074081421,
      "logps/rejected": -1.771484375,
      "loss": 0.7095,
      "nll_loss": 0.680371105670929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06230468675494194,
      "rewards/margins": 0.11488189548254013,
      "rewards/rejected": -0.17722168564796448,
      "step": 8830
    },
    {
      "epoch": 0.6708149946881166,
      "grad_norm": 2.6022467229642148,
      "learning_rate": 8.50871259623034e-07,
      "log_odds_chosen": 1.805273413658142,
      "log_odds_ratio": -0.3316406309604645,
      "logits/chosen": -0.982421875,
      "logits/rejected": -0.8453124761581421,
      "logps/chosen": -0.578906238079071,
      "logps/rejected": -1.7998046875,
      "loss": 0.717,
      "nll_loss": 0.693066418170929,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.05787353590130806,
      "rewards/margins": 0.12221679836511612,
      "rewards/rejected": -0.18012695014476776,
      "step": 8840
    },
    {
      "epoch": 0.6715738351798451,
      "grad_norm": 2.5418111643811905,
      "learning_rate": 8.503904055243742e-07,
      "log_odds_chosen": 1.670019507408142,
      "log_odds_ratio": -0.365234375,
      "logits/chosen": -0.9320312738418579,
      "logits/rejected": -0.8900390863418579,
      "logps/chosen": -0.57763671875,
      "logps/rejected": -1.7355468273162842,
      "loss": 0.7158,
      "nll_loss": 0.622363269329071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.0577392578125,
      "rewards/margins": 0.11576537787914276,
      "rewards/rejected": -0.17355957627296448,
      "step": 8850
    },
    {
      "epoch": 0.6723326756715738,
      "grad_norm": 3.052844854817002,
      "learning_rate": 8.499103657426704e-07,
      "log_odds_chosen": 1.416601538658142,
      "log_odds_ratio": -0.42431640625,
      "logits/chosen": -1.056054711341858,
      "logits/rejected": -0.9154297113418579,
      "logps/chosen": -0.685742199420929,
      "logps/rejected": -1.661523461341858,
      "loss": 0.7174,
      "nll_loss": 0.71533203125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06862793117761612,
      "rewards/margins": 0.09736938774585724,
      "rewards/rejected": -0.16611328721046448,
      "step": 8860
    },
    {
      "epoch": 0.6730915161633024,
      "grad_norm": 2.511251534832671,
      "learning_rate": 8.494311379821314e-07,
      "log_odds_chosen": 1.158789038658142,
      "log_odds_ratio": -0.44999998807907104,
      "logits/chosen": -0.9912109375,
      "logits/rejected": -0.880859375,
      "logps/chosen": -0.630566418170929,
      "logps/rejected": -1.350000023841858,
      "loss": 0.7238,
      "nll_loss": 0.6845703125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06308593600988388,
      "rewards/margins": 0.07196655124425888,
      "rewards/rejected": -0.13498535752296448,
      "step": 8870
    },
    {
      "epoch": 0.6738503566550311,
      "grad_norm": 2.412780939469374,
      "learning_rate": 8.489527199560178e-07,
      "log_odds_chosen": 1.5443847179412842,
      "log_odds_ratio": -0.4092773497104645,
      "logits/chosen": -1.009179711341858,
      "logits/rejected": -0.914257824420929,
      "logps/chosen": -0.613574206829071,
      "logps/rejected": -1.6824219226837158,
      "loss": 0.7251,
      "nll_loss": 0.674023449420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06130371242761612,
      "rewards/margins": 0.10692749172449112,
      "rewards/rejected": -0.16813965141773224,
      "step": 8880
    },
    {
      "epoch": 0.6746091971467597,
      "grad_norm": 3.0254601092370574,
      "learning_rate": 8.484751093865948e-07,
      "log_odds_chosen": 1.3079102039337158,
      "log_odds_ratio": -0.43115234375,
      "logits/chosen": -1.008398413658142,
      "logits/rejected": -0.904296875,
      "logps/chosen": -0.6478515863418579,
      "logps/rejected": -1.4953124523162842,
      "loss": 0.7184,
      "nll_loss": 0.7030273675918579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06484375149011612,
      "rewards/margins": 0.08479003608226776,
      "rewards/rejected": -0.14960937201976776,
      "step": 8890
    },
    {
      "epoch": 0.6753680376384884,
      "grad_norm": 3.9457403315097856,
      "learning_rate": 8.47998304005088e-07,
      "log_odds_chosen": 1.51513671875,
      "log_odds_ratio": -0.4247070252895355,
      "logits/chosen": -0.949023425579071,
      "logits/rejected": -0.873046875,
      "logps/chosen": -0.719921886920929,
      "logps/rejected": -1.7775390148162842,
      "loss": 0.7391,
      "nll_loss": 0.7147461175918579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07200928032398224,
      "rewards/margins": 0.10576172173023224,
      "rewards/rejected": -0.17766113579273224,
      "step": 8900
    },
    {
      "epoch": 0.676126878130217,
      "grad_norm": 3.1192413134893493,
      "learning_rate": 8.475223015516377e-07,
      "log_odds_chosen": 1.552343726158142,
      "log_odds_ratio": -0.3772949278354645,
      "logits/chosen": -0.994335949420929,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.620800793170929,
      "logps/rejected": -1.673828125,
      "loss": 0.7235,
      "nll_loss": 0.69287109375,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06203613430261612,
      "rewards/margins": 0.10540771484375,
      "rewards/rejected": -0.16730956733226776,
      "step": 8910
    },
    {
      "epoch": 0.6768857186219457,
      "grad_norm": 2.6214655270353266,
      "learning_rate": 8.470470997752534e-07,
      "log_odds_chosen": 1.460546851158142,
      "log_odds_ratio": -0.40434569120407104,
      "logits/chosen": -1.045312523841858,
      "logits/rejected": -0.9146484136581421,
      "logps/chosen": -0.6294921636581421,
      "logps/rejected": -1.6281249523162842,
      "loss": 0.7257,
      "nll_loss": 0.6561523675918579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06297607719898224,
      "rewards/margins": 0.09989013522863388,
      "rewards/rejected": -0.16279296576976776,
      "step": 8920
    },
    {
      "epoch": 0.6776445591136743,
      "grad_norm": 2.87762511585744,
      "learning_rate": 8.465726964337702e-07,
      "log_odds_chosen": 1.945703148841858,
      "log_odds_ratio": -0.3311523497104645,
      "logits/chosen": -1.008203148841858,
      "logits/rejected": -0.8978515863418579,
      "logps/chosen": -0.610546886920929,
      "logps/rejected": -2.012500047683716,
      "loss": 0.7284,
      "nll_loss": 0.7041015625,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06104736402630806,
      "rewards/margins": 0.14020995795726776,
      "rewards/rejected": -0.20107421278953552,
      "step": 8930
    },
    {
      "epoch": 0.678403399605403,
      "grad_norm": 2.683763592444103,
      "learning_rate": 8.460990892938031e-07,
      "log_odds_chosen": 1.658789038658142,
      "log_odds_ratio": -0.37397462129592896,
      "logits/chosen": -1.0085937976837158,
      "logits/rejected": -0.882617175579071,
      "logps/chosen": -0.609179675579071,
      "logps/rejected": -1.791015625,
      "loss": 0.7071,
      "nll_loss": 0.689257800579071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06092529371380806,
      "rewards/margins": 0.11809692531824112,
      "rewards/rejected": -0.1790771484375,
      "step": 8940
    },
    {
      "epoch": 0.6791622400971316,
      "grad_norm": 2.449182932081057,
      "learning_rate": 8.456262761307038e-07,
      "log_odds_chosen": 1.5183594226837158,
      "log_odds_ratio": -0.38395994901657104,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.885546863079071,
      "logps/chosen": -0.669726550579071,
      "logps/rejected": -1.7228515148162842,
      "loss": 0.7217,
      "nll_loss": 0.712109386920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06695556640625,
      "rewards/margins": 0.10532226413488388,
      "rewards/rejected": -0.1722412109375,
      "step": 8950
    },
    {
      "epoch": 0.6799210805888602,
      "grad_norm": 2.3708588018010195,
      "learning_rate": 8.451542547285166e-07,
      "log_odds_chosen": 1.4533202648162842,
      "log_odds_ratio": -0.41425782442092896,
      "logits/chosen": -1.015234351158142,
      "logits/rejected": -0.8861328363418579,
      "logps/chosen": -0.63037109375,
      "logps/rejected": -1.6416015625,
      "loss": 0.7285,
      "nll_loss": 0.674511730670929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06312255561351776,
      "rewards/margins": 0.10125732421875,
      "rewards/rejected": -0.1641845703125,
      "step": 8960
    },
    {
      "epoch": 0.6806799210805888,
      "grad_norm": 3.5430099084984334,
      "learning_rate": 8.44683022879934e-07,
      "log_odds_chosen": 1.6541016101837158,
      "log_odds_ratio": -0.3408203125,
      "logits/chosen": -1.025976538658142,
      "logits/rejected": -0.925000011920929,
      "logps/chosen": -0.60009765625,
      "logps/rejected": -1.7423827648162842,
      "loss": 0.736,
      "nll_loss": 0.6495116949081421,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.05999755859375,
      "rewards/margins": 0.11427001655101776,
      "rewards/rejected": -0.17426757514476776,
      "step": 8970
    },
    {
      "epoch": 0.6814387615723175,
      "grad_norm": 3.102706773563467,
      "learning_rate": 8.442125783862544e-07,
      "log_odds_chosen": 1.7042968273162842,
      "log_odds_ratio": -0.35102540254592896,
      "logits/chosen": -1.0712890625,
      "logits/rejected": -0.9183593988418579,
      "logps/chosen": -0.6201171875,
      "logps/rejected": -1.8410155773162842,
      "loss": 0.7214,
      "nll_loss": 0.669628918170929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06193847581744194,
      "rewards/margins": 0.12209472805261612,
      "rewards/rejected": -0.18408203125,
      "step": 8980
    },
    {
      "epoch": 0.6821976020640461,
      "grad_norm": 2.9053453555705984,
      "learning_rate": 8.437429190573388e-07,
      "log_odds_chosen": 1.7033202648162842,
      "log_odds_ratio": -0.3802246153354645,
      "logits/chosen": -1.0107421875,
      "logits/rejected": -0.9296875,
      "logps/chosen": -0.61279296875,
      "logps/rejected": -1.8103516101837158,
      "loss": 0.7214,
      "nll_loss": 0.6630859375,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06125488132238388,
      "rewards/margins": 0.11966552585363388,
      "rewards/rejected": -0.18088379502296448,
      "step": 8990
    },
    {
      "epoch": 0.6829564425557748,
      "grad_norm": 2.490135537579069,
      "learning_rate": 8.432740427115678e-07,
      "log_odds_chosen": 1.4577147960662842,
      "log_odds_ratio": -0.39594727754592896,
      "logits/chosen": -0.9605468511581421,
      "logits/rejected": -0.8550781011581421,
      "logps/chosen": -0.6131836175918579,
      "logps/rejected": -1.596093773841858,
      "loss": 0.7053,
      "nll_loss": 0.696972668170929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.061279296875,
      "rewards/margins": 0.09818573296070099,
      "rewards/rejected": -0.15944823622703552,
      "step": 9000
    },
    {
      "epoch": 0.6837152830475034,
      "grad_norm": 3.8422780431570813,
      "learning_rate": 8.428059471757984e-07,
      "log_odds_chosen": 1.509765625,
      "log_odds_ratio": -0.3612060546875,
      "logits/chosen": -0.9937499761581421,
      "logits/rejected": -0.902148425579071,
      "logps/chosen": -0.6751953363418579,
      "logps/rejected": -1.7332031726837158,
      "loss": 0.7349,
      "nll_loss": 0.7255859375,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06752929836511612,
      "rewards/margins": 0.10584716498851776,
      "rewards/rejected": -0.17343750596046448,
      "step": 9010
    },
    {
      "epoch": 0.6844741235392321,
      "grad_norm": 3.8217632772985533,
      "learning_rate": 8.423386302853226e-07,
      "log_odds_chosen": 1.5412108898162842,
      "log_odds_ratio": -0.3634277284145355,
      "logits/chosen": -1.0535156726837158,
      "logits/rejected": -0.940234363079071,
      "logps/chosen": -0.604785144329071,
      "logps/rejected": -1.644140601158142,
      "loss": 0.7039,
      "nll_loss": 0.687304675579071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06046142429113388,
      "rewards/margins": 0.10375366359949112,
      "rewards/rejected": -0.16440430283546448,
      "step": 9020
    },
    {
      "epoch": 0.6852329640309607,
      "grad_norm": 2.868969089731345,
      "learning_rate": 8.418720898838254e-07,
      "log_odds_chosen": 1.5802733898162842,
      "log_odds_ratio": -0.3963378965854645,
      "logits/chosen": -0.9775390625,
      "logits/rejected": -0.896484375,
      "logps/chosen": -0.645800769329071,
      "logps/rejected": -1.7529296875,
      "loss": 0.7625,
      "nll_loss": 0.67333984375,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06461181491613388,
      "rewards/margins": 0.11072997748851776,
      "rewards/rejected": -0.17524413764476776,
      "step": 9030
    },
    {
      "epoch": 0.6859918045226894,
      "grad_norm": 2.9932422667148457,
      "learning_rate": 8.414063238233425e-07,
      "log_odds_chosen": 1.869140625,
      "log_odds_ratio": -0.29838865995407104,
      "logits/chosen": -0.987500011920929,
      "logits/rejected": -0.8609374761581421,
      "logps/chosen": -0.6407226324081421,
      "logps/rejected": -1.9460937976837158,
      "loss": 0.72,
      "nll_loss": 0.7079101800918579,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.0640869140625,
      "rewards/margins": 0.13059082627296448,
      "rewards/rejected": -0.194580078125,
      "step": 9040
    },
    {
      "epoch": 0.686750645014418,
      "grad_norm": 3.080579444219386,
      "learning_rate": 8.409413299642188e-07,
      "log_odds_chosen": 1.582617163658142,
      "log_odds_ratio": -0.3634277284145355,
      "logits/chosen": -1.0291016101837158,
      "logits/rejected": -0.892773449420929,
      "logps/chosen": -0.6392577886581421,
      "logps/rejected": -1.748632788658142,
      "loss": 0.7379,
      "nll_loss": 0.687207043170929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06386718899011612,
      "rewards/margins": 0.11099853366613388,
      "rewards/rejected": -0.17478027939796448,
      "step": 9050
    },
    {
      "epoch": 0.6875094855061467,
      "grad_norm": 2.48095153866512,
      "learning_rate": 8.404771061750672e-07,
      "log_odds_chosen": 1.6251952648162842,
      "log_odds_ratio": -0.3843750059604645,
      "logits/chosen": -0.9576171636581421,
      "logits/rejected": -0.845703125,
      "logps/chosen": -0.6758788824081421,
      "logps/rejected": -1.826757788658142,
      "loss": 0.7342,
      "nll_loss": 0.716992199420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06765136867761612,
      "rewards/margins": 0.11514892429113388,
      "rewards/rejected": -0.18276366591453552,
      "step": 9060
    },
    {
      "epoch": 0.6882683259978752,
      "grad_norm": 3.138066548353663,
      "learning_rate": 8.400136503327277e-07,
      "log_odds_chosen": 1.494531273841858,
      "log_odds_ratio": -0.4088134765625,
      "logits/chosen": -1.018164038658142,
      "logits/rejected": -0.9740234613418579,
      "logps/chosen": -0.651074230670929,
      "logps/rejected": -1.631250023841858,
      "loss": 0.7057,
      "nll_loss": 0.5933593511581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06508789211511612,
      "rewards/margins": 0.09808349609375,
      "rewards/rejected": -0.16330567002296448,
      "step": 9070
    },
    {
      "epoch": 0.6890271664896039,
      "grad_norm": 2.6656921736577415,
      "learning_rate": 8.395509603222271e-07,
      "log_odds_chosen": 1.689062476158142,
      "log_odds_ratio": -0.3423095643520355,
      "logits/chosen": -1.023828148841858,
      "logits/rejected": -0.921875,
      "logps/chosen": -0.623339831829071,
      "logps/rejected": -1.80078125,
      "loss": 0.7244,
      "nll_loss": 0.718066394329071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06226806715130806,
      "rewards/margins": 0.11767578125,
      "rewards/rejected": -0.1800537109375,
      "step": 9080
    },
    {
      "epoch": 0.6897860069813325,
      "grad_norm": 2.818770206694993,
      "learning_rate": 8.390890340367368e-07,
      "log_odds_chosen": 1.294335961341858,
      "log_odds_ratio": -0.39106446504592896,
      "logits/chosen": -1.0060546398162842,
      "logits/rejected": -0.911328136920929,
      "logps/chosen": -0.6317383050918579,
      "logps/rejected": -1.4919922351837158,
      "loss": 0.7159,
      "nll_loss": 0.7132812738418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06312255561351776,
      "rewards/margins": 0.086029052734375,
      "rewards/rejected": -0.14924316108226776,
      "step": 9090
    },
    {
      "epoch": 0.6905448474730611,
      "grad_norm": 2.7946045339632963,
      "learning_rate": 8.386278693775346e-07,
      "log_odds_chosen": 1.460546851158142,
      "log_odds_ratio": -0.38300782442092896,
      "logits/chosen": -0.9693359136581421,
      "logits/rejected": -0.891406238079071,
      "logps/chosen": -0.6612304449081421,
      "logps/rejected": -1.6355469226837158,
      "loss": 0.7402,
      "nll_loss": 0.6854492425918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06613769382238388,
      "rewards/margins": 0.09750976413488388,
      "rewards/rejected": -0.16352538764476776,
      "step": 9100
    },
    {
      "epoch": 0.6913036879647898,
      "grad_norm": 3.4993736100208173,
      "learning_rate": 8.381674642539632e-07,
      "log_odds_chosen": 1.458398461341858,
      "log_odds_ratio": -0.4149414002895355,
      "logits/chosen": -1.0343749523162842,
      "logits/rejected": -0.9302734136581421,
      "logps/chosen": -0.636914074420929,
      "logps/rejected": -1.6232421398162842,
      "loss": 0.7265,
      "nll_loss": 0.678906261920929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06370849907398224,
      "rewards/margins": 0.09853820502758026,
      "rewards/rejected": -0.16232910752296448,
      "step": 9110
    },
    {
      "epoch": 0.6920625284565184,
      "grad_norm": 2.87806956546888,
      "learning_rate": 8.37707816583391e-07,
      "log_odds_chosen": 1.818750023841858,
      "log_odds_ratio": -0.3299316465854645,
      "logits/chosen": -1.082617163658142,
      "logits/rejected": -0.9185546636581421,
      "logps/chosen": -0.662890613079071,
      "logps/rejected": -1.958593726158142,
      "loss": 0.7081,
      "nll_loss": 0.699999988079071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06630859524011612,
      "rewards/margins": 0.12961426377296448,
      "rewards/rejected": -0.19589844346046448,
      "step": 9120
    },
    {
      "epoch": 0.6928213689482471,
      "grad_norm": 2.976141220633325,
      "learning_rate": 8.372489242911724e-07,
      "log_odds_chosen": 1.623632788658142,
      "log_odds_ratio": -0.3701171875,
      "logits/chosen": -1.049218773841858,
      "logits/rejected": -0.901171863079071,
      "logps/chosen": -0.587109386920929,
      "logps/rejected": -1.689453125,
      "loss": 0.7078,
      "nll_loss": 0.5948241949081421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05867920070886612,
      "rewards/margins": 0.11011962592601776,
      "rewards/rejected": -0.16887207329273224,
      "step": 9130
    },
    {
      "epoch": 0.6935802094399757,
      "grad_norm": 3.495959990506981,
      "learning_rate": 8.367907853106078e-07,
      "log_odds_chosen": 1.5183594226837158,
      "log_odds_ratio": -0.3849121034145355,
      "logits/chosen": -1.039453148841858,
      "logits/rejected": -0.9058593511581421,
      "logps/chosen": -0.6591796875,
      "logps/rejected": -1.702539086341858,
      "loss": 0.7543,
      "nll_loss": 0.73583984375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06594238430261612,
      "rewards/margins": 0.10428466647863388,
      "rewards/rejected": -0.17019042372703552,
      "step": 9140
    },
    {
      "epoch": 0.6943390499317044,
      "grad_norm": 3.15168909268087,
      "learning_rate": 8.363333975829066e-07,
      "log_odds_chosen": 1.76953125,
      "log_odds_ratio": -0.3524414002895355,
      "logits/chosen": -1.012109398841858,
      "logits/rejected": -0.8880859613418579,
      "logps/chosen": -0.624804675579071,
      "logps/rejected": -1.8757812976837158,
      "loss": 0.7177,
      "nll_loss": 0.6651366949081421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06248779222369194,
      "rewards/margins": 0.12497558444738388,
      "rewards/rejected": -0.1875,
      "step": 9150
    },
    {
      "epoch": 0.695097890423433,
      "grad_norm": 3.0190535757643326,
      "learning_rate": 8.358767590571457e-07,
      "log_odds_chosen": 1.661523461341858,
      "log_odds_ratio": -0.355224609375,
      "logits/chosen": -1.0380859375,
      "logits/rejected": -0.912109375,
      "logps/chosen": -0.5850585699081421,
      "logps/rejected": -1.651953101158142,
      "loss": 0.6982,
      "nll_loss": 0.642285168170929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.05849609524011612,
      "rewards/margins": 0.10675048828125,
      "rewards/rejected": -0.16518554091453552,
      "step": 9160
    },
    {
      "epoch": 0.6958567309151616,
      "grad_norm": 2.6297503660881665,
      "learning_rate": 8.354208676902326e-07,
      "log_odds_chosen": 1.6213867664337158,
      "log_odds_ratio": -0.3669677674770355,
      "logits/chosen": -1.0037109851837158,
      "logits/rejected": -0.890625,
      "logps/chosen": -0.5731445550918579,
      "logps/rejected": -1.680078148841858,
      "loss": 0.7222,
      "nll_loss": 0.642285168170929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05728759616613388,
      "rewards/margins": 0.11073608696460724,
      "rewards/rejected": -0.16796875,
      "step": 9170
    },
    {
      "epoch": 0.6966155714068902,
      "grad_norm": 3.1619887361778525,
      "learning_rate": 8.349657214468659e-07,
      "log_odds_chosen": 1.474023461341858,
      "log_odds_ratio": -0.3697753846645355,
      "logits/chosen": -1.0623047351837158,
      "logits/rejected": -0.9458984136581421,
      "logps/chosen": -0.5767577886581421,
      "logps/rejected": -1.539453148841858,
      "loss": 0.7018,
      "nll_loss": 0.640332043170929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05766601487994194,
      "rewards/margins": 0.09627685695886612,
      "rewards/rejected": -0.1539306640625,
      "step": 9180
    },
    {
      "epoch": 0.6973744118986189,
      "grad_norm": 2.9357100092718236,
      "learning_rate": 8.345113182994988e-07,
      "log_odds_chosen": 1.493749976158142,
      "log_odds_ratio": -0.3768066465854645,
      "logits/chosen": -1.0568358898162842,
      "logits/rejected": -0.921679675579071,
      "logps/chosen": -0.5972656011581421,
      "logps/rejected": -1.613671898841858,
      "loss": 0.6904,
      "nll_loss": 0.6214843988418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.05975341796875,
      "rewards/margins": 0.10163573920726776,
      "rewards/rejected": -0.16145019233226776,
      "step": 9190
    },
    {
      "epoch": 0.6981332523903475,
      "grad_norm": 2.604420840193982,
      "learning_rate": 8.34057656228299e-07,
      "log_odds_chosen": 1.535742163658142,
      "log_odds_ratio": -0.4088378846645355,
      "logits/chosen": -1.037109375,
      "logits/rejected": -0.9345703125,
      "logps/chosen": -0.64697265625,
      "logps/rejected": -1.7136719226837158,
      "loss": 0.7144,
      "nll_loss": 0.651562511920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06472168117761612,
      "rewards/margins": 0.10678710788488388,
      "rewards/rejected": -0.17136231064796448,
      "step": 9200
    },
    {
      "epoch": 0.6988920928820762,
      "grad_norm": 3.200157985640524,
      "learning_rate": 8.336047332211128e-07,
      "log_odds_chosen": 1.6169922351837158,
      "log_odds_ratio": -0.34624022245407104,
      "logits/chosen": -1.0525391101837158,
      "logits/rejected": -0.942187488079071,
      "logps/chosen": -0.62646484375,
      "logps/rejected": -1.7042968273162842,
      "loss": 0.71,
      "nll_loss": 0.6533203125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06267090141773224,
      "rewards/margins": 0.10789795219898224,
      "rewards/rejected": -0.17041015625,
      "step": 9210
    },
    {
      "epoch": 0.6996509333738048,
      "grad_norm": 3.2156469672490178,
      "learning_rate": 8.331525472734267e-07,
      "log_odds_chosen": 1.89453125,
      "log_odds_ratio": -0.33203125,
      "logits/chosen": -1.049218773841858,
      "logits/rejected": -0.9134765863418579,
      "logps/chosen": -0.6415039300918579,
      "logps/rejected": -2.007031202316284,
      "loss": 0.7091,
      "nll_loss": 0.67333984375,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06411132961511612,
      "rewards/margins": 0.136566162109375,
      "rewards/rejected": -0.20078125596046448,
      "step": 9220
    },
    {
      "epoch": 0.7004097738655335,
      "grad_norm": 3.4145794329670482,
      "learning_rate": 8.327010963883302e-07,
      "log_odds_chosen": 1.4153320789337158,
      "log_odds_ratio": -0.4208007752895355,
      "logits/chosen": -1.0205078125,
      "logits/rejected": -0.932421863079071,
      "logps/chosen": -0.61962890625,
      "logps/rejected": -1.5984375476837158,
      "loss": 0.7182,
      "nll_loss": 0.70166015625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06198730319738388,
      "rewards/margins": 0.097930908203125,
      "rewards/rejected": -0.15986327826976776,
      "step": 9230
    },
    {
      "epoch": 0.7011686143572621,
      "grad_norm": 3.1446396732274375,
      "learning_rate": 8.322503785764789e-07,
      "log_odds_chosen": 1.369726538658142,
      "log_odds_ratio": -0.4117675721645355,
      "logits/chosen": -0.993359386920929,
      "logits/rejected": -0.887499988079071,
      "logps/chosen": -0.6258789300918579,
      "logps/rejected": -1.572265625,
      "loss": 0.7209,
      "nll_loss": 0.689648449420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06258545070886612,
      "rewards/margins": 0.094696044921875,
      "rewards/rejected": -0.15737304091453552,
      "step": 9240
    },
    {
      "epoch": 0.7019274548489908,
      "grad_norm": 2.7808709633900044,
      "learning_rate": 8.318003918560583e-07,
      "log_odds_chosen": 1.7595703601837158,
      "log_odds_ratio": -0.3587402403354645,
      "logits/chosen": -0.9945312738418579,
      "logits/rejected": -0.9126952886581421,
      "logps/chosen": -0.629687488079071,
      "logps/rejected": -1.862695336341858,
      "loss": 0.7204,
      "nll_loss": 0.708691418170929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06302490085363388,
      "rewards/margins": 0.12322998046875,
      "rewards/rejected": -0.18605956435203552,
      "step": 9250
    },
    {
      "epoch": 0.7026862953407194,
      "grad_norm": 3.199962352835074,
      "learning_rate": 8.313511342527453e-07,
      "log_odds_chosen": 1.432519555091858,
      "log_odds_ratio": -0.45488280057907104,
      "logits/chosen": -1.012304663658142,
      "logits/rejected": -0.9175781011581421,
      "logps/chosen": -0.640429675579071,
      "logps/rejected": -1.6814453601837158,
      "loss": 0.7062,
      "nll_loss": 0.6934570074081421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06409911811351776,
      "rewards/margins": 0.10413818061351776,
      "rewards/rejected": -0.16818848252296448,
      "step": 9260
    },
    {
      "epoch": 0.703445135832448,
      "grad_norm": 3.0159947288223945,
      "learning_rate": 8.309026037996745e-07,
      "log_odds_chosen": 1.75146484375,
      "log_odds_ratio": -0.35882568359375,
      "logits/chosen": -1.0076172351837158,
      "logits/rejected": -0.9087890386581421,
      "logps/chosen": -0.5914062261581421,
      "logps/rejected": -1.811914086341858,
      "loss": 0.7019,
      "nll_loss": 0.6719726324081421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05911865085363388,
      "rewards/margins": 0.12203369289636612,
      "rewards/rejected": -0.18107910454273224,
      "step": 9270
    },
    {
      "epoch": 0.7042039763241766,
      "grad_norm": 3.052470283610497,
      "learning_rate": 8.304547985373996e-07,
      "log_odds_chosen": 1.763574242591858,
      "log_odds_ratio": -0.3483032286167145,
      "logits/chosen": -1.0441405773162842,
      "logits/rejected": -0.939453125,
      "logps/chosen": -0.6279296875,
      "logps/rejected": -1.82421875,
      "loss": 0.685,
      "nll_loss": 0.633007824420929,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06272582709789276,
      "rewards/margins": 0.11971130222082138,
      "rewards/rejected": -0.1824951171875,
      "step": 9280
    },
    {
      "epoch": 0.7049628168159053,
      "grad_norm": 3.4082513013731766,
      "learning_rate": 8.300077165138592e-07,
      "log_odds_chosen": 1.7136719226837158,
      "log_odds_ratio": -0.3604492247104645,
      "logits/chosen": -1.038476586341858,
      "logits/rejected": -0.924609363079071,
      "logps/chosen": -0.620410144329071,
      "logps/rejected": -1.804296851158142,
      "loss": 0.6991,
      "nll_loss": 0.687304675579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06206054612994194,
      "rewards/margins": 0.11837158352136612,
      "rewards/rejected": -0.18034668266773224,
      "step": 9290
    },
    {
      "epoch": 0.7057216573076339,
      "grad_norm": 2.9015258805794555,
      "learning_rate": 8.295613557843402e-07,
      "log_odds_chosen": 1.6443359851837158,
      "log_odds_ratio": -0.3910156190395355,
      "logits/chosen": -0.9951171875,
      "logits/rejected": -0.9296875,
      "logps/chosen": -0.67236328125,
      "logps/rejected": -1.8214843273162842,
      "loss": 0.7376,
      "nll_loss": 0.7109375,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06719970703125,
      "rewards/margins": 0.11496581882238388,
      "rewards/rejected": -0.1822509765625,
      "step": 9300
    },
    {
      "epoch": 0.7064804977993626,
      "grad_norm": 2.939785147512604,
      "learning_rate": 8.291157144114419e-07,
      "log_odds_chosen": 1.4240233898162842,
      "log_odds_ratio": -0.4503173828125,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.904296875,
      "logps/chosen": -0.712109386920929,
      "logps/rejected": -1.7039062976837158,
      "loss": 0.7093,
      "nll_loss": 0.709179699420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07125244289636612,
      "rewards/margins": 0.09906616061925888,
      "rewards/rejected": -0.1702880859375,
      "step": 9310
    },
    {
      "epoch": 0.7072393382910912,
      "grad_norm": 2.600650906164271,
      "learning_rate": 8.286707904650417e-07,
      "log_odds_chosen": 1.435156226158142,
      "log_odds_ratio": -0.41533201932907104,
      "logits/chosen": -1.022070288658142,
      "logits/rejected": -0.9398437738418579,
      "logps/chosen": -0.6942383050918579,
      "logps/rejected": -1.678125023841858,
      "loss": 0.7282,
      "nll_loss": 0.76171875,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06944580376148224,
      "rewards/margins": 0.09825439751148224,
      "rewards/rejected": -0.16779784858226776,
      "step": 9320
    },
    {
      "epoch": 0.7079981787828199,
      "grad_norm": 3.2070573741611685,
      "learning_rate": 8.282265820222593e-07,
      "log_odds_chosen": 1.4519531726837158,
      "log_odds_ratio": -0.402587890625,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.8941406011581421,
      "logps/chosen": -0.6220703125,
      "logps/rejected": -1.6042969226837158,
      "loss": 0.7007,
      "nll_loss": 0.659472644329071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06219482421875,
      "rewards/margins": 0.09814453125,
      "rewards/rejected": -0.16032715141773224,
      "step": 9330
    },
    {
      "epoch": 0.7087570192745485,
      "grad_norm": 2.8976755408991886,
      "learning_rate": 8.277830871674222e-07,
      "log_odds_chosen": 1.535058617591858,
      "log_odds_ratio": -0.38789063692092896,
      "logits/chosen": -0.9917968511581421,
      "logits/rejected": -0.8589843511581421,
      "logps/chosen": -0.6283203363418579,
      "logps/rejected": -1.6736328601837158,
      "loss": 0.7335,
      "nll_loss": 0.61328125,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06282959133386612,
      "rewards/margins": 0.10449524223804474,
      "rewards/rejected": -0.16738280653953552,
      "step": 9340
    },
    {
      "epoch": 0.7095158597662772,
      "grad_norm": 3.3987506799493086,
      "learning_rate": 8.273403039920306e-07,
      "log_odds_chosen": 1.495996117591858,
      "log_odds_ratio": -0.37250977754592896,
      "logits/chosen": -1.0451171398162842,
      "logits/rejected": -0.9439452886581421,
      "logps/chosen": -0.6737304925918579,
      "logps/rejected": -1.664453148841858,
      "loss": 0.7109,
      "nll_loss": 0.725781261920929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06737060844898224,
      "rewards/margins": 0.09909667819738388,
      "rewards/rejected": -0.16655273735523224,
      "step": 9350
    },
    {
      "epoch": 0.7102747002580058,
      "grad_norm": 3.7816459933046587,
      "learning_rate": 8.268982305947231e-07,
      "log_odds_chosen": 1.4901854991912842,
      "log_odds_ratio": -0.39619141817092896,
      "logits/chosen": -1.0166015625,
      "logits/rejected": -0.8984375,
      "logps/chosen": -0.6463867425918579,
      "logps/rejected": -1.638085961341858,
      "loss": 0.7177,
      "nll_loss": 0.7064453363418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06459961086511612,
      "rewards/margins": 0.09928588569164276,
      "rewards/rejected": -0.16384276747703552,
      "step": 9360
    },
    {
      "epoch": 0.7110335407497345,
      "grad_norm": 2.6035328018623867,
      "learning_rate": 8.264568650812423e-07,
      "log_odds_chosen": 1.539648413658142,
      "log_odds_ratio": -0.3548828065395355,
      "logits/chosen": -1.0623047351837158,
      "logits/rejected": -0.9449218511581421,
      "logps/chosen": -0.638964831829071,
      "logps/rejected": -1.673828125,
      "loss": 0.7036,
      "nll_loss": 0.6361328363418579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06395263969898224,
      "rewards/margins": 0.10349731147289276,
      "rewards/rejected": -0.1673583984375,
      "step": 9370
    },
    {
      "epoch": 0.711792381241463,
      "grad_norm": 3.1513163902945993,
      "learning_rate": 8.26016205564401e-07,
      "log_odds_chosen": 1.645898461341858,
      "log_odds_ratio": -0.36396485567092896,
      "logits/chosen": -1.0388672351837158,
      "logits/rejected": -0.927929699420929,
      "logps/chosen": -0.639355480670929,
      "logps/rejected": -1.76953125,
      "loss": 0.6799,
      "nll_loss": 0.664843738079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06398925930261612,
      "rewards/margins": 0.11285400390625,
      "rewards/rejected": -0.17683105170726776,
      "step": 9380
    },
    {
      "epoch": 0.7125512217331916,
      "grad_norm": 2.9624290128543147,
      "learning_rate": 8.25576250164048e-07,
      "log_odds_chosen": 1.588281273841858,
      "log_odds_ratio": -0.3607177734375,
      "logits/chosen": -1.052148461341858,
      "logits/rejected": -0.9164062738418579,
      "logps/chosen": -0.6328125,
      "logps/rejected": -1.719140648841858,
      "loss": 0.7075,
      "nll_loss": 0.6480468511581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06333007663488388,
      "rewards/margins": 0.10847778618335724,
      "rewards/rejected": -0.17189940810203552,
      "step": 9390
    },
    {
      "epoch": 0.7133100622249203,
      "grad_norm": 3.098491141706027,
      "learning_rate": 8.251369970070346e-07,
      "log_odds_chosen": 1.573632836341858,
      "log_odds_ratio": -0.404296875,
      "logits/chosen": -1.037695288658142,
      "logits/rejected": -0.8955078125,
      "logps/chosen": -0.636425793170929,
      "logps/rejected": -1.751953125,
      "loss": 0.7252,
      "nll_loss": 0.6932617425918579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06368408352136612,
      "rewards/margins": 0.11151123046875,
      "rewards/rejected": -0.17529296875,
      "step": 9400
    },
    {
      "epoch": 0.7140689027166489,
      "grad_norm": 3.4785135599316175,
      "learning_rate": 8.246984442271813e-07,
      "log_odds_chosen": 1.3253905773162842,
      "log_odds_ratio": -0.4205566346645355,
      "logits/chosen": -1.0634765625,
      "logits/rejected": -0.9535156488418579,
      "logps/chosen": -0.626660168170929,
      "logps/rejected": -1.505273461341858,
      "loss": 0.7136,
      "nll_loss": 0.676464855670929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06265868991613388,
      "rewards/margins": 0.08788452297449112,
      "rewards/rejected": -0.1505126953125,
      "step": 9410
    },
    {
      "epoch": 0.7148277432083776,
      "grad_norm": 4.066118106757687,
      "learning_rate": 8.242605899652435e-07,
      "log_odds_chosen": 1.559960961341858,
      "log_odds_ratio": -0.3536376953125,
      "logits/chosen": -1.0441405773162842,
      "logits/rejected": -0.917187511920929,
      "logps/chosen": -0.595507800579071,
      "logps/rejected": -1.647070288658142,
      "loss": 0.7127,
      "nll_loss": 0.6332031488418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.0595703125,
      "rewards/margins": 0.10518188774585724,
      "rewards/rejected": -0.16462402045726776,
      "step": 9420
    },
    {
      "epoch": 0.7155865837001062,
      "grad_norm": 2.778220956023162,
      "learning_rate": 8.238234323688798e-07,
      "log_odds_chosen": 1.290380835533142,
      "log_odds_ratio": -0.43852537870407104,
      "logits/chosen": -0.970507800579071,
      "logits/rejected": -0.879687488079071,
      "logps/chosen": -0.62255859375,
      "logps/rejected": -1.532812476158142,
      "loss": 0.7116,
      "nll_loss": 0.7496093511581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06223144382238388,
      "rewards/margins": 0.09091796725988388,
      "rewards/rejected": -0.15324707329273224,
      "step": 9430
    },
    {
      "epoch": 0.7163454241918349,
      "grad_norm": 3.0909960504122536,
      "learning_rate": 8.233869695926182e-07,
      "log_odds_chosen": 1.1931641101837158,
      "log_odds_ratio": -0.4408203065395355,
      "logits/chosen": -1.0041015148162842,
      "logits/rejected": -0.9169921875,
      "logps/chosen": -0.65625,
      "logps/rejected": -1.42578125,
      "loss": 0.7176,
      "nll_loss": 0.7022460699081421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06563720852136612,
      "rewards/margins": 0.07687988132238388,
      "rewards/rejected": -0.14260253310203552,
      "step": 9440
    },
    {
      "epoch": 0.7171042646835635,
      "grad_norm": 2.8911447642832693,
      "learning_rate": 8.229511997978235e-07,
      "log_odds_chosen": 1.635351538658142,
      "log_odds_ratio": -0.340087890625,
      "logits/chosen": -1.026953101158142,
      "logits/rejected": -0.8980468511581421,
      "logps/chosen": -0.627734363079071,
      "logps/rejected": -1.747656226158142,
      "loss": 0.7057,
      "nll_loss": 0.7240234613418579,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06282959133386612,
      "rewards/margins": 0.11191406100988388,
      "rewards/rejected": -0.1746826171875,
      "step": 9450
    },
    {
      "epoch": 0.7178631051752922,
      "grad_norm": 2.8737807569712803,
      "learning_rate": 8.22516121152665e-07,
      "log_odds_chosen": 1.576269507408142,
      "log_odds_ratio": -0.3546386659145355,
      "logits/chosen": -1.055078148841858,
      "logits/rejected": -0.9457031488418579,
      "logps/chosen": -0.65966796875,
      "logps/rejected": -1.761328101158142,
      "loss": 0.688,
      "nll_loss": 0.6517578363418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06590576469898224,
      "rewards/margins": 0.11024169623851776,
      "rewards/rejected": -0.17629393935203552,
      "step": 9460
    },
    {
      "epoch": 0.7186219456670208,
      "grad_norm": 2.5498810925172175,
      "learning_rate": 8.220817318320836e-07,
      "log_odds_chosen": 1.574804663658142,
      "log_odds_ratio": -0.41093748807907104,
      "logits/chosen": -1.0486328601837158,
      "logits/rejected": -0.9603515863418579,
      "logps/chosen": -0.638378918170929,
      "logps/rejected": -1.734765648841858,
      "loss": 0.7249,
      "nll_loss": 0.6749023199081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0638427734375,
      "rewards/margins": 0.10963134467601776,
      "rewards/rejected": -0.17338867485523224,
      "step": 9470
    },
    {
      "epoch": 0.7193807861587495,
      "grad_norm": 2.5597559134110615,
      "learning_rate": 8.216480300177611e-07,
      "log_odds_chosen": 1.4949219226837158,
      "log_odds_ratio": -0.4126953184604645,
      "logits/chosen": -0.9994140863418579,
      "logits/rejected": -0.9130859375,
      "logps/chosen": -0.6241210699081421,
      "logps/rejected": -1.637109398841858,
      "loss": 0.7042,
      "nll_loss": 0.671679675579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06241454929113388,
      "rewards/margins": 0.10139159858226776,
      "rewards/rejected": -0.1636962890625,
      "step": 9480
    },
    {
      "epoch": 0.720139626650478,
      "grad_norm": 2.942311840429656,
      "learning_rate": 8.212150138980857e-07,
      "log_odds_chosen": 1.49365234375,
      "log_odds_ratio": -0.39448243379592896,
      "logits/chosen": -1.0380859375,
      "logits/rejected": -0.9300781488418579,
      "logps/chosen": -0.6373046636581421,
      "logps/rejected": -1.6593749523162842,
      "loss": 0.6918,
      "nll_loss": 0.686328113079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06378173828125,
      "rewards/margins": 0.10233154147863388,
      "rewards/rejected": -0.166015625,
      "step": 9490
    },
    {
      "epoch": 0.7208984671422067,
      "grad_norm": 2.745660460923844,
      "learning_rate": 8.207826816681233e-07,
      "log_odds_chosen": 1.2062499523162842,
      "log_odds_ratio": -0.46992188692092896,
      "logits/chosen": -1.0085937976837158,
      "logits/rejected": -0.9095703363418579,
      "logps/chosen": -0.667773425579071,
      "logps/rejected": -1.4835937023162842,
      "loss": 0.7297,
      "nll_loss": 0.725781261920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06674804538488388,
      "rewards/margins": 0.08168945461511612,
      "rewards/rejected": -0.14841309189796448,
      "step": 9500
    },
    {
      "epoch": 0.7216573076339353,
      "grad_norm": 4.0009419229614,
      "learning_rate": 8.203510315295829e-07,
      "log_odds_chosen": 1.691796898841858,
      "log_odds_ratio": -0.31611329317092896,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.8960937261581421,
      "logps/chosen": -0.6041015386581421,
      "logps/rejected": -1.7589843273162842,
      "loss": 0.7152,
      "nll_loss": 0.687792956829071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06040038913488388,
      "rewards/margins": 0.11550293117761612,
      "rewards/rejected": -0.17592772841453552,
      "step": 9510
    },
    {
      "epoch": 0.722416148125664,
      "grad_norm": 3.1941615556516125,
      "learning_rate": 8.199200616907878e-07,
      "log_odds_chosen": 1.6658203601837158,
      "log_odds_ratio": -0.35419923067092896,
      "logits/chosen": -1.046289086341858,
      "logits/rejected": -0.9287109375,
      "logps/chosen": -0.6255859136581421,
      "logps/rejected": -1.803125023841858,
      "loss": 0.7029,
      "nll_loss": 0.6260741949081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06258545070886612,
      "rewards/margins": 0.11777343600988388,
      "rewards/rejected": -0.18037109076976776,
      "step": 9520
    },
    {
      "epoch": 0.7231749886173926,
      "grad_norm": 3.049993025984763,
      "learning_rate": 8.194897703666421e-07,
      "log_odds_chosen": 1.5374023914337158,
      "log_odds_ratio": -0.35200196504592896,
      "logits/chosen": -1.004296898841858,
      "logits/rejected": -0.8941406011581421,
      "logps/chosen": -0.619335949420929,
      "logps/rejected": -1.7029297351837158,
      "loss": 0.6998,
      "nll_loss": 0.6654297113418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06191406399011612,
      "rewards/margins": 0.10847778618335724,
      "rewards/rejected": -0.17036132514476776,
      "step": 9530
    },
    {
      "epoch": 0.7239338291091213,
      "grad_norm": 2.871771558134182,
      "learning_rate": 8.190601557786015e-07,
      "log_odds_chosen": 1.481347680091858,
      "log_odds_ratio": -0.40869140625,
      "logits/chosen": -1.020898461341858,
      "logits/rejected": -0.909960925579071,
      "logps/chosen": -0.626953125,
      "logps/rejected": -1.6613280773162842,
      "loss": 0.7111,
      "nll_loss": 0.688281238079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06268310546875,
      "rewards/margins": 0.10338135063648224,
      "rewards/rejected": -0.16608886420726776,
      "step": 9540
    },
    {
      "epoch": 0.7246926696008499,
      "grad_norm": 2.9624145394028956,
      "learning_rate": 8.186312161546413e-07,
      "log_odds_chosen": 1.67919921875,
      "log_odds_ratio": -0.3866210877895355,
      "logits/chosen": -1.064062476158142,
      "logits/rejected": -0.9556640386581421,
      "logps/chosen": -0.6200195550918579,
      "logps/rejected": -1.801171898841858,
      "loss": 0.6751,
      "nll_loss": 0.6309570074081421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06196289137005806,
      "rewards/margins": 0.11810302734375,
      "rewards/rejected": -0.1800537109375,
      "step": 9550
    },
    {
      "epoch": 0.7254515100925786,
      "grad_norm": 2.5334843972987664,
      "learning_rate": 8.182029497292262e-07,
      "log_odds_chosen": 1.605371117591858,
      "log_odds_ratio": -0.4095703065395355,
      "logits/chosen": -1.086523413658142,
      "logits/rejected": -0.967578113079071,
      "logps/chosen": -0.695507824420929,
      "logps/rejected": -1.8291015625,
      "loss": 0.7053,
      "nll_loss": 0.693652331829071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06959228217601776,
      "rewards/margins": 0.11314697563648224,
      "rewards/rejected": -0.18266601860523224,
      "step": 9560
    },
    {
      "epoch": 0.7262103505843072,
      "grad_norm": 4.016065116443712,
      "learning_rate": 8.177753547432792e-07,
      "log_odds_chosen": 1.692968726158142,
      "log_odds_ratio": -0.35295408964157104,
      "logits/chosen": -1.01953125,
      "logits/rejected": -0.9164062738418579,
      "logps/chosen": -0.6377929449081421,
      "logps/rejected": -1.80859375,
      "loss": 0.7054,
      "nll_loss": 0.682910144329071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06374511867761612,
      "rewards/margins": 0.11716308444738388,
      "rewards/rejected": -0.18085937201976776,
      "step": 9570
    },
    {
      "epoch": 0.7269691910760359,
      "grad_norm": 4.45337210149427,
      "learning_rate": 8.173484294441524e-07,
      "log_odds_chosen": 1.3463866710662842,
      "log_odds_ratio": -0.4344726502895355,
      "logits/chosen": -1.000585913658142,
      "logits/rejected": -0.912109375,
      "logps/chosen": -0.6314452886581421,
      "logps/rejected": -1.586523413658142,
      "loss": 0.7228,
      "nll_loss": 0.7105468511581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06313476711511612,
      "rewards/margins": 0.09547729790210724,
      "rewards/rejected": -0.15849609673023224,
      "step": 9580
    },
    {
      "epoch": 0.7277280315677644,
      "grad_norm": 3.093014359022252,
      "learning_rate": 8.169221720855952e-07,
      "log_odds_chosen": 1.443212866783142,
      "log_odds_ratio": -0.37128907442092896,
      "logits/chosen": -1.045507788658142,
      "logits/rejected": -0.950976550579071,
      "logps/chosen": -0.588085949420929,
      "logps/rejected": -1.5234375,
      "loss": 0.7095,
      "nll_loss": 0.6708008050918579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05874023586511612,
      "rewards/margins": 0.09354247897863388,
      "rewards/rejected": -0.15234375,
      "step": 9590
    },
    {
      "epoch": 0.7284868720594931,
      "grad_norm": 2.8045353815731273,
      "learning_rate": 8.164965809277261e-07,
      "log_odds_chosen": 1.424414038658142,
      "log_odds_ratio": -0.37666016817092896,
      "logits/chosen": -1.0439453125,
      "logits/rejected": -0.9107421636581421,
      "logps/chosen": -0.6259765625,
      "logps/rejected": -1.5779297351837158,
      "loss": 0.6969,
      "nll_loss": 0.6421874761581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06264648586511612,
      "rewards/margins": 0.09520263969898224,
      "rewards/rejected": -0.15773925185203552,
      "step": 9600
    },
    {
      "epoch": 0.7292457125512217,
      "grad_norm": 2.761496333392702,
      "learning_rate": 8.160716542370011e-07,
      "log_odds_chosen": 1.6564452648162842,
      "log_odds_ratio": -0.3475585877895355,
      "logits/chosen": -1.098046898841858,
      "logits/rejected": -0.9603515863418579,
      "logps/chosen": -0.6324218511581421,
      "logps/rejected": -1.783593773841858,
      "loss": 0.7157,
      "nll_loss": 0.6709960699081421,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06326904147863388,
      "rewards/margins": 0.11512450873851776,
      "rewards/rejected": -0.17827148735523224,
      "step": 9610
    },
    {
      "epoch": 0.7300045530429504,
      "grad_norm": 3.015306363526461,
      "learning_rate": 8.156473902861856e-07,
      "log_odds_chosen": 1.65771484375,
      "log_odds_ratio": -0.3919921815395355,
      "logits/chosen": -1.0265624523162842,
      "logits/rejected": -0.9195312261581421,
      "logps/chosen": -0.6045898199081421,
      "logps/rejected": -1.729882836341858,
      "loss": 0.6856,
      "nll_loss": 0.664746105670929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06038818508386612,
      "rewards/margins": 0.11270751804113388,
      "rewards/rejected": -0.17307129502296448,
      "step": 9620
    },
    {
      "epoch": 0.730763393534679,
      "grad_norm": 3.3445443161524615,
      "learning_rate": 8.152237873543241e-07,
      "log_odds_chosen": 1.431640625,
      "log_odds_ratio": -0.4449218809604645,
      "logits/chosen": -0.990039050579071,
      "logits/rejected": -0.8568359613418579,
      "logps/chosen": -0.6485351324081421,
      "logps/rejected": -1.675390601158142,
      "loss": 0.7148,
      "nll_loss": 0.7191406488418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06489257514476776,
      "rewards/margins": 0.10255126655101776,
      "rewards/rejected": -0.16757813096046448,
      "step": 9630
    },
    {
      "epoch": 0.7315222340264077,
      "grad_norm": 3.750217860575627,
      "learning_rate": 8.148008437267104e-07,
      "log_odds_chosen": 1.6515624523162842,
      "log_odds_ratio": -0.3653808534145355,
      "logits/chosen": -1.0380859375,
      "logits/rejected": -0.95703125,
      "logps/chosen": -0.6327148675918579,
      "logps/rejected": -1.7820312976837158,
      "loss": 0.7153,
      "nll_loss": 0.673828125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06325683742761612,
      "rewards/margins": 0.11501464992761612,
      "rewards/rejected": -0.17824706435203552,
      "step": 9640
    },
    {
      "epoch": 0.7322810745181363,
      "grad_norm": 3.7236221175889663,
      "learning_rate": 8.143785576948602e-07,
      "log_odds_chosen": 1.3380858898162842,
      "log_odds_ratio": -0.38300782442092896,
      "logits/chosen": -0.994140625,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -0.5947265625,
      "logps/rejected": -1.4792969226837158,
      "loss": 0.7217,
      "nll_loss": 0.673535168170929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05947265774011612,
      "rewards/margins": 0.08846435695886612,
      "rewards/rejected": -0.14794921875,
      "step": 9650
    },
    {
      "epoch": 0.7330399150098649,
      "grad_norm": 3.6288611870242407,
      "learning_rate": 8.139569275564796e-07,
      "log_odds_chosen": 1.61181640625,
      "log_odds_ratio": -0.3910156190395355,
      "logits/chosen": -0.9994140863418579,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.719531238079071,
      "logps/rejected": -1.8894531726837158,
      "loss": 0.7069,
      "nll_loss": 0.728515625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.072021484375,
      "rewards/margins": 0.11671753227710724,
      "rewards/rejected": -0.188720703125,
      "step": 9660
    },
    {
      "epoch": 0.7337987555015936,
      "grad_norm": 2.950044313068542,
      "learning_rate": 8.135359516154388e-07,
      "log_odds_chosen": 1.6580078601837158,
      "log_odds_ratio": -0.33281248807907104,
      "logits/chosen": -1.054101586341858,
      "logits/rejected": -0.9556640386581421,
      "logps/chosen": -0.6055663824081421,
      "logps/rejected": -1.7160155773162842,
      "loss": 0.7187,
      "nll_loss": 0.6265624761581421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06060791015625,
      "rewards/margins": 0.11098632961511612,
      "rewards/rejected": -0.1715087890625,
      "step": 9670
    },
    {
      "epoch": 0.7345575959933222,
      "grad_norm": 3.0822414004750263,
      "learning_rate": 8.131156281817418e-07,
      "log_odds_chosen": 1.7685546875,
      "log_odds_ratio": -0.32861328125,
      "logits/chosen": -1.0246093273162842,
      "logits/rejected": -0.902148425579071,
      "logps/chosen": -0.5926758050918579,
      "logps/rejected": -1.786718726158142,
      "loss": 0.7261,
      "nll_loss": 0.655468761920929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05927734449505806,
      "rewards/margins": 0.11955566704273224,
      "rewards/rejected": -0.17875976860523224,
      "step": 9680
    },
    {
      "epoch": 0.7353164364850509,
      "grad_norm": 2.615596741963579,
      "learning_rate": 8.126959555714979e-07,
      "log_odds_chosen": 1.173095703125,
      "log_odds_ratio": -0.4927734434604645,
      "logits/chosen": -1.0300781726837158,
      "logits/rejected": -0.912109375,
      "logps/chosen": -0.6563476324081421,
      "logps/rejected": -1.4714844226837158,
      "loss": 0.7003,
      "nll_loss": 0.63525390625,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06563720852136612,
      "rewards/margins": 0.0815223678946495,
      "rewards/rejected": -0.14711913466453552,
      "step": 9690
    },
    {
      "epoch": 0.7360752769767794,
      "grad_norm": 3.360149813748261,
      "learning_rate": 8.122769321068952e-07,
      "log_odds_chosen": 1.746484398841858,
      "log_odds_ratio": -0.3385253846645355,
      "logits/chosen": -0.9775390625,
      "logits/rejected": -0.888671875,
      "logps/chosen": -0.6509765386581421,
      "logps/rejected": -1.8640625476837158,
      "loss": 0.6869,
      "nll_loss": 0.673828125,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06505737453699112,
      "rewards/margins": 0.12130127102136612,
      "rewards/rejected": -0.18649902939796448,
      "step": 9700
    },
    {
      "epoch": 0.7368341174685081,
      "grad_norm": 2.909685109657297,
      "learning_rate": 8.118585561161698e-07,
      "log_odds_chosen": 1.688574194908142,
      "log_odds_ratio": -0.3746093809604645,
      "logits/chosen": -1.053320288658142,
      "logits/rejected": -0.9369140863418579,
      "logps/chosen": -0.5865234136581421,
      "logps/rejected": -1.7595703601837158,
      "loss": 0.7144,
      "nll_loss": 0.708691418170929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.05860595777630806,
      "rewards/margins": 0.11747436225414276,
      "rewards/rejected": -0.176025390625,
      "step": 9710
    },
    {
      "epoch": 0.7375929579602367,
      "grad_norm": 3.189014729667494,
      "learning_rate": 8.114408259335793e-07,
      "log_odds_chosen": 1.705468773841858,
      "log_odds_ratio": -0.35576170682907104,
      "logits/chosen": -1.022070288658142,
      "logits/rejected": -0.9437500238418579,
      "logps/chosen": -0.5956054925918579,
      "logps/rejected": -1.735937476158142,
      "loss": 0.6708,
      "nll_loss": 0.6220703125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05959472805261612,
      "rewards/margins": 0.11409912258386612,
      "rewards/rejected": -0.17365722358226776,
      "step": 9720
    },
    {
      "epoch": 0.7383517984519654,
      "grad_norm": 3.077063349879149,
      "learning_rate": 8.110237398993754e-07,
      "log_odds_chosen": 1.367919921875,
      "log_odds_ratio": -0.43510740995407104,
      "logits/chosen": -1.018164038658142,
      "logits/rejected": -0.95703125,
      "logps/chosen": -0.6788085699081421,
      "logps/rejected": -1.609960913658142,
      "loss": 0.7031,
      "nll_loss": 0.65380859375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06787109375,
      "rewards/margins": 0.09309692680835724,
      "rewards/rejected": -0.1611328125,
      "step": 9730
    },
    {
      "epoch": 0.739110638943694,
      "grad_norm": 2.748602718265524,
      "learning_rate": 8.106072963597751e-07,
      "log_odds_chosen": 1.61865234375,
      "log_odds_ratio": -0.3550781309604645,
      "logits/chosen": -1.0451171398162842,
      "logits/rejected": -0.933789074420929,
      "logps/chosen": -0.582714855670929,
      "logps/rejected": -1.6845703125,
      "loss": 0.7167,
      "nll_loss": 0.63525390625,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05832519382238388,
      "rewards/margins": 0.11011962592601776,
      "rewards/rejected": -0.16845703125,
      "step": 9740
    },
    {
      "epoch": 0.7398694794354227,
      "grad_norm": 3.268717899241557,
      "learning_rate": 8.101914936669332e-07,
      "log_odds_chosen": 1.5139648914337158,
      "log_odds_ratio": -0.43901365995407104,
      "logits/chosen": -1.012304663658142,
      "logits/rejected": -0.916015625,
      "logps/chosen": -0.6890624761581421,
      "logps/rejected": -1.73828125,
      "loss": 0.7166,
      "nll_loss": 0.74951171875,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0689697265625,
      "rewards/margins": 0.10474853217601776,
      "rewards/rejected": -0.17365722358226776,
      "step": 9750
    },
    {
      "epoch": 0.7406283199271513,
      "grad_norm": 2.639596921121034,
      "learning_rate": 8.09776330178916e-07,
      "log_odds_chosen": 1.7824218273162842,
      "log_odds_ratio": -0.35649412870407104,
      "logits/chosen": -1.0369141101837158,
      "logits/rejected": -0.915234386920929,
      "logps/chosen": -0.634960949420929,
      "logps/rejected": -1.90234375,
      "loss": 0.7067,
      "nll_loss": 0.61767578125,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06353759765625,
      "rewards/margins": 0.12664794921875,
      "rewards/rejected": -0.19020995497703552,
      "step": 9760
    },
    {
      "epoch": 0.74138716041888,
      "grad_norm": 2.727969831386695,
      "learning_rate": 8.093618042596727e-07,
      "log_odds_chosen": 1.7810547351837158,
      "log_odds_ratio": -0.3067871034145355,
      "logits/chosen": -0.9916015863418579,
      "logits/rejected": -0.900390625,
      "logps/chosen": -0.61669921875,
      "logps/rejected": -1.8624999523162842,
      "loss": 0.6931,
      "nll_loss": 0.700390636920929,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.06169433519244194,
      "rewards/margins": 0.12442626804113388,
      "rewards/rejected": -0.18623046576976776,
      "step": 9770
    },
    {
      "epoch": 0.7421460009106086,
      "grad_norm": 2.95089832822571,
      "learning_rate": 8.089479142790095e-07,
      "log_odds_chosen": 1.5273926258087158,
      "log_odds_ratio": -0.3857665956020355,
      "logits/chosen": -1.022851586341858,
      "logits/rejected": -0.910351574420929,
      "logps/chosen": -0.615527331829071,
      "logps/rejected": -1.6533203125,
      "loss": 0.7232,
      "nll_loss": 0.693359375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06160888820886612,
      "rewards/margins": 0.10381469875574112,
      "rewards/rejected": -0.165283203125,
      "step": 9780
    },
    {
      "epoch": 0.7429048414023373,
      "grad_norm": 3.3794191970100678,
      "learning_rate": 8.085346586125621e-07,
      "log_odds_chosen": 1.370214819908142,
      "log_odds_ratio": -0.3920654356479645,
      "logits/chosen": -1.0105469226837158,
      "logits/rejected": -0.9052734375,
      "logps/chosen": -0.617871105670929,
      "logps/rejected": -1.521093726158142,
      "loss": 0.6906,
      "nll_loss": 0.639453113079071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06179199367761612,
      "rewards/margins": 0.09033813327550888,
      "rewards/rejected": -0.15212401747703552,
      "step": 9790
    },
    {
      "epoch": 0.7436636818940658,
      "grad_norm": 3.8719705893956666,
      "learning_rate": 8.081220356417685e-07,
      "log_odds_chosen": 1.421484351158142,
      "log_odds_ratio": -0.43310546875,
      "logits/chosen": -1.0009765625,
      "logits/rejected": -0.937695324420929,
      "logps/chosen": -0.619921863079071,
      "logps/rejected": -1.566015601158142,
      "loss": 0.6978,
      "nll_loss": 0.6644531488418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06198730319738388,
      "rewards/margins": 0.09454345703125,
      "rewards/rejected": -0.15642090141773224,
      "step": 9800
    },
    {
      "epoch": 0.7444225223857945,
      "grad_norm": 2.9859368077072466,
      "learning_rate": 8.077100437538435e-07,
      "log_odds_chosen": 1.490820288658142,
      "log_odds_ratio": -0.3731445372104645,
      "logits/chosen": -1.071679711341858,
      "logits/rejected": -0.959765613079071,
      "logps/chosen": -0.6390625238418579,
      "logps/rejected": -1.635351538658142,
      "loss": 0.7135,
      "nll_loss": 0.611523449420929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06390380859375,
      "rewards/margins": 0.09953613579273224,
      "rewards/rejected": -0.16350097954273224,
      "step": 9810
    },
    {
      "epoch": 0.7451813628775231,
      "grad_norm": 2.9458682593614287,
      "learning_rate": 8.072986813417512e-07,
      "log_odds_chosen": 1.2968261241912842,
      "log_odds_ratio": -0.40947264432907104,
      "logits/chosen": -1.021484375,
      "logits/rejected": -0.9173828363418579,
      "logps/chosen": -0.640917956829071,
      "logps/rejected": -1.514257788658142,
      "loss": 0.7177,
      "nll_loss": 0.6927734613418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06409911811351776,
      "rewards/margins": 0.08727417141199112,
      "rewards/rejected": -0.15144042670726776,
      "step": 9820
    },
    {
      "epoch": 0.7459402033692518,
      "grad_norm": 2.7256634641769493,
      "learning_rate": 8.068879468041791e-07,
      "log_odds_chosen": 1.596777319908142,
      "log_odds_ratio": -0.37910157442092896,
      "logits/chosen": -1.0333983898162842,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.6444336175918579,
      "logps/rejected": -1.7619140148162842,
      "loss": 0.7021,
      "nll_loss": 0.6631835699081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06450195610523224,
      "rewards/margins": 0.11162719875574112,
      "rewards/rejected": -0.1761474609375,
      "step": 9830
    },
    {
      "epoch": 0.7466990438609804,
      "grad_norm": 2.461950975465807,
      "learning_rate": 8.064778385455118e-07,
      "log_odds_chosen": 1.403710961341858,
      "log_odds_ratio": -0.42656248807907104,
      "logits/chosen": -1.055273413658142,
      "logits/rejected": -0.931640625,
      "logps/chosen": -0.662109375,
      "logps/rejected": -1.67578125,
      "loss": 0.6893,
      "nll_loss": 0.674023449420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06617431342601776,
      "rewards/margins": 0.10143432766199112,
      "rewards/rejected": -0.16770020127296448,
      "step": 9840
    },
    {
      "epoch": 0.7474578843527091,
      "grad_norm": 2.9381430618600692,
      "learning_rate": 8.060683549758054e-07,
      "log_odds_chosen": 1.464453101158142,
      "log_odds_ratio": -0.39643555879592896,
      "logits/chosen": -1.0138671398162842,
      "logits/rejected": -0.916015625,
      "logps/chosen": -0.6651366949081421,
      "logps/rejected": -1.657812476158142,
      "loss": 0.7252,
      "nll_loss": 0.696582019329071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06650390475988388,
      "rewards/margins": 0.09929809719324112,
      "rewards/rejected": -0.16567382216453552,
      "step": 9850
    },
    {
      "epoch": 0.7482167248444377,
      "grad_norm": 3.3001354673190106,
      "learning_rate": 8.056594945107608e-07,
      "log_odds_chosen": 1.5481445789337158,
      "log_odds_ratio": -0.412353515625,
      "logits/chosen": -1.0177733898162842,
      "logits/rejected": -0.9375,
      "logps/chosen": -0.6268554925918579,
      "logps/rejected": -1.6853516101837158,
      "loss": 0.7111,
      "nll_loss": 0.6719726324081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06265868991613388,
      "rewards/margins": 0.10568847507238388,
      "rewards/rejected": -0.16846923530101776,
      "step": 9860
    },
    {
      "epoch": 0.7489755653361664,
      "grad_norm": 3.5461519330612243,
      "learning_rate": 8.052512555716987e-07,
      "log_odds_chosen": 1.743554711341858,
      "log_odds_ratio": -0.3538574278354645,
      "logits/chosen": -1.0109374523162842,
      "logits/rejected": -0.945507824420929,
      "logps/chosen": -0.659960925579071,
      "logps/rejected": -1.8671875,
      "loss": 0.6991,
      "nll_loss": 0.6568359136581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06595458835363388,
      "rewards/margins": 0.12074585258960724,
      "rewards/rejected": -0.18659667670726776,
      "step": 9870
    },
    {
      "epoch": 0.749734405827895,
      "grad_norm": 2.67020379560676,
      "learning_rate": 8.048436365855337e-07,
      "log_odds_chosen": 1.1943359375,
      "log_odds_ratio": -0.4874511659145355,
      "logits/chosen": -0.988476574420929,
      "logits/rejected": -0.89453125,
      "logps/chosen": -0.6356445550918579,
      "logps/rejected": -1.4871094226837158,
      "loss": 0.6852,
      "nll_loss": 0.6796875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06356201320886612,
      "rewards/margins": 0.08510742336511612,
      "rewards/rejected": -0.14865723252296448,
      "step": 9880
    },
    {
      "epoch": 0.7504932463196237,
      "grad_norm": 2.8341309625851516,
      "learning_rate": 8.044366359847486e-07,
      "log_odds_chosen": 1.6677734851837158,
      "log_odds_ratio": -0.38178712129592896,
      "logits/chosen": -1.039648413658142,
      "logits/rejected": -0.911328136920929,
      "logps/chosen": -0.6180664300918579,
      "logps/rejected": -1.732812523841858,
      "loss": 0.6864,
      "nll_loss": 0.610058605670929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06181640550494194,
      "rewards/margins": 0.11149291694164276,
      "rewards/rejected": -0.1732177734375,
      "step": 9890
    },
    {
      "epoch": 0.7512520868113522,
      "grad_norm": 2.8976394764539113,
      "learning_rate": 8.040302522073696e-07,
      "log_odds_chosen": 1.7229492664337158,
      "log_odds_ratio": -0.3226074278354645,
      "logits/chosen": -0.993945300579071,
      "logits/rejected": -0.865039050579071,
      "logps/chosen": -0.601367175579071,
      "logps/rejected": -1.783203125,
      "loss": 0.6997,
      "nll_loss": 0.6250976324081421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06004638597369194,
      "rewards/margins": 0.11831054836511612,
      "rewards/rejected": -0.17836913466453552,
      "step": 9900
    },
    {
      "epoch": 0.752010927303081,
      "grad_norm": 3.3050263093017582,
      "learning_rate": 8.036244836969407e-07,
      "log_odds_chosen": 1.7648437023162842,
      "log_odds_ratio": -0.3529296815395355,
      "logits/chosen": -1.0378906726837158,
      "logits/rejected": -0.9375,
      "logps/chosen": -0.5956054925918579,
      "logps/rejected": -1.8035156726837158,
      "loss": 0.7065,
      "nll_loss": 0.6626952886581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05949706956744194,
      "rewards/margins": 0.12080077826976776,
      "rewards/rejected": -0.18034668266773224,
      "step": 9910
    },
    {
      "epoch": 0.7527697677948095,
      "grad_norm": 2.70363681667673,
      "learning_rate": 8.032193289024989e-07,
      "log_odds_chosen": 1.8629882335662842,
      "log_odds_ratio": -0.3124755918979645,
      "logits/chosen": -1.049218773841858,
      "logits/rejected": -0.894726574420929,
      "logps/chosen": -0.6024414300918579,
      "logps/rejected": -1.8781249523162842,
      "loss": 0.7,
      "nll_loss": 0.6827148199081421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06024169921875,
      "rewards/margins": 0.12754973769187927,
      "rewards/rejected": -0.187744140625,
      "step": 9920
    },
    {
      "epoch": 0.7535286082865381,
      "grad_norm": 3.6720386590938063,
      "learning_rate": 8.02814786278549e-07,
      "log_odds_chosen": 1.5441405773162842,
      "log_odds_ratio": -0.41064453125,
      "logits/chosen": -1.015039086341858,
      "logits/rejected": -0.919921875,
      "logps/chosen": -0.623339831829071,
      "logps/rejected": -1.6804687976837158,
      "loss": 0.7135,
      "nll_loss": 0.7216796875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06236572191119194,
      "rewards/margins": 0.105712890625,
      "rewards/rejected": -0.16799315810203552,
      "step": 9930
    },
    {
      "epoch": 0.7542874487782668,
      "grad_norm": 3.2424502309904764,
      "learning_rate": 8.024108542850394e-07,
      "log_odds_chosen": 1.529296875,
      "log_odds_ratio": -0.3722167909145355,
      "logits/chosen": -1.028710961341858,
      "logits/rejected": -0.923828125,
      "logps/chosen": -0.6244140863418579,
      "logps/rejected": -1.670312523841858,
      "loss": 0.699,
      "nll_loss": 0.6700195074081421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06246338039636612,
      "rewards/margins": 0.10474853217601776,
      "rewards/rejected": -0.16708984971046448,
      "step": 9940
    },
    {
      "epoch": 0.7550462892699954,
      "grad_norm": 2.775687366537809,
      "learning_rate": 8.020075313873367e-07,
      "log_odds_chosen": 1.4794921875,
      "log_odds_ratio": -0.4207519590854645,
      "logits/chosen": -1.0089843273162842,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.6498047113418579,
      "logps/rejected": -1.6681640148162842,
      "loss": 0.6929,
      "nll_loss": 0.6792968511581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06495361030101776,
      "rewards/margins": 0.101806640625,
      "rewards/rejected": -0.16684570908546448,
      "step": 9950
    },
    {
      "epoch": 0.7558051297617241,
      "grad_norm": 2.9999965310002845,
      "learning_rate": 8.016048160562023e-07,
      "log_odds_chosen": 1.5947265625,
      "log_odds_ratio": -0.3536621034145355,
      "logits/chosen": -0.968945324420929,
      "logits/rejected": -0.8851562738418579,
      "logps/chosen": -0.5938476324081421,
      "logps/rejected": -1.6232421398162842,
      "loss": 0.6704,
      "nll_loss": 0.606249988079071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.0594482421875,
      "rewards/margins": 0.10288085788488388,
      "rewards/rejected": -0.1622314453125,
      "step": 9960
    },
    {
      "epoch": 0.7565639702534527,
      "grad_norm": 3.0533768128561545,
      "learning_rate": 8.012027067677667e-07,
      "log_odds_chosen": 1.4152343273162842,
      "log_odds_ratio": -0.4334960877895355,
      "logits/chosen": -0.990039050579071,
      "logits/rejected": -0.8798828125,
      "logps/chosen": -0.645800769329071,
      "logps/rejected": -1.5476562976837158,
      "loss": 0.7097,
      "nll_loss": 0.670117199420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06455077975988388,
      "rewards/margins": 0.09015502780675888,
      "rewards/rejected": -0.15476074814796448,
      "step": 9970
    },
    {
      "epoch": 0.7573228107451814,
      "grad_norm": 3.131970361190211,
      "learning_rate": 8.008012020035062e-07,
      "log_odds_chosen": 1.5291016101837158,
      "log_odds_ratio": -0.41162109375,
      "logits/chosen": -1.0380859375,
      "logits/rejected": -0.9429687261581421,
      "logps/chosen": -0.6058593988418579,
      "logps/rejected": -1.608789086341858,
      "loss": 0.6859,
      "nll_loss": 0.6395508050918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06052245944738388,
      "rewards/margins": 0.10036621242761612,
      "rewards/rejected": -0.16091307997703552,
      "step": 9980
    },
    {
      "epoch": 0.75808165123691,
      "grad_norm": 2.890624157805145,
      "learning_rate": 8.004003002502188e-07,
      "log_odds_chosen": 1.6139647960662842,
      "log_odds_ratio": -0.35957032442092896,
      "logits/chosen": -1.037695288658142,
      "logits/rejected": -0.873828113079071,
      "logps/chosen": -0.583789050579071,
      "logps/rejected": -1.670507788658142,
      "loss": 0.6765,
      "nll_loss": 0.673632800579071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05837402492761612,
      "rewards/margins": 0.10859374701976776,
      "rewards/rejected": -0.16691894829273224,
      "step": 9990
    },
    {
      "epoch": 0.7588404917286387,
      "grad_norm": 3.811180333498817,
      "learning_rate": 8e-07,
      "log_odds_chosen": 1.330896019935608,
      "log_odds_ratio": -0.46684569120407104,
      "logits/chosen": -1.069726586341858,
      "logits/rejected": -0.952929675579071,
      "logps/chosen": -0.64404296875,
      "logps/rejected": -1.5496094226837158,
      "loss": 0.6774,
      "nll_loss": 0.5791991949081421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06447754055261612,
      "rewards/margins": 0.090631864964962,
      "rewards/rejected": -0.15488281846046448,
      "step": 10000
    },
    {
      "epoch": 0.7588404917286387,
      "eval_log_odds_chosen": 1.098879098892212,
      "eval_log_odds_ratio": -0.499722421169281,
      "eval_logits/chosen": -0.8501765131950378,
      "eval_logits/rejected": -0.7714591026306152,
      "eval_logps/chosen": -0.7597776055335999,
      "eval_logps/rejected": -1.5665394067764282,
      "eval_loss": 1.2043746709823608,
      "eval_nll_loss": 1.1630213260650635,
      "eval_rewards/accuracies": 0.7124233245849609,
      "eval_rewards/chosen": -0.07597849518060684,
      "eval_rewards/margins": 0.0806727185845375,
      "eval_rewards/rejected": -0.1566520780324936,
      "eval_runtime": 1310.1799,
      "eval_samples_per_second": 71.657,
      "eval_steps_per_second": 1.12,
      "step": 10000
    },
    {
      "epoch": 0.7595993322203672,
      "grad_norm": 2.4226719938293093,
      "learning_rate": 7.996002997502185e-07,
      "log_odds_chosen": 1.4033203125,
      "log_odds_ratio": -0.42500001192092896,
      "logits/chosen": -1.075781226158142,
      "logits/rejected": -1.009374976158142,
      "logps/chosen": -0.6817382574081421,
      "logps/rejected": -1.636328101158142,
      "loss": 0.704,
      "nll_loss": 0.6771484613418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06821288913488388,
      "rewards/margins": 0.09544678032398224,
      "rewards/rejected": -0.16362304985523224,
      "step": 10010
    },
    {
      "epoch": 0.7603581727120959,
      "grad_norm": 3.1382628262362493,
      "learning_rate": 7.992011980034937e-07,
      "log_odds_chosen": 1.384179711341858,
      "log_odds_ratio": -0.3919433653354645,
      "logits/chosen": -1.068359375,
      "logits/rejected": -0.9390624761581421,
      "logps/chosen": -0.672558605670929,
      "logps/rejected": -1.6046874523162842,
      "loss": 0.6829,
      "nll_loss": 0.64794921875,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06729736179113388,
      "rewards/margins": 0.09316406399011612,
      "rewards/rejected": -0.1602783203125,
      "step": 10020
    },
    {
      "epoch": 0.7611170132038245,
      "grad_norm": 2.6751466068113694,
      "learning_rate": 7.98802693267671e-07,
      "log_odds_chosen": 1.730078101158142,
      "log_odds_ratio": -0.36079102754592896,
      "logits/chosen": -1.009179711341858,
      "logits/rejected": -0.8900390863418579,
      "logps/chosen": -0.653027355670929,
      "logps/rejected": -1.8712890148162842,
      "loss": 0.6987,
      "nll_loss": 0.6446288824081421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06528320163488388,
      "rewards/margins": 0.1219383254647255,
      "rewards/rejected": -0.18723145127296448,
      "step": 10030
    },
    {
      "epoch": 0.7618758536955532,
      "grad_norm": 2.989307328368908,
      "learning_rate": 7.984047840557992e-07,
      "log_odds_chosen": 1.467187523841858,
      "log_odds_ratio": -0.4013671875,
      "logits/chosen": -0.9878906011581421,
      "logits/rejected": -0.890429675579071,
      "logps/chosen": -0.620312511920929,
      "logps/rejected": -1.615234375,
      "loss": 0.6937,
      "nll_loss": 0.6514648199081421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06207275390625,
      "rewards/margins": 0.09954833984375,
      "rewards/rejected": -0.16152343153953552,
      "step": 10040
    },
    {
      "epoch": 0.7626346941872818,
      "grad_norm": 5.154125822266643,
      "learning_rate": 7.980074688861063e-07,
      "log_odds_chosen": 1.3654296398162842,
      "log_odds_ratio": -0.4061523377895355,
      "logits/chosen": -1.0205078125,
      "logits/rejected": -0.888671875,
      "logps/chosen": -0.633593738079071,
      "logps/rejected": -1.5421874523162842,
      "loss": 0.6979,
      "nll_loss": 0.738476574420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06329345703125,
      "rewards/margins": 0.09093017876148224,
      "rewards/rejected": -0.15419921278953552,
      "step": 10050
    },
    {
      "epoch": 0.7633935346790105,
      "grad_norm": 4.012825994669084,
      "learning_rate": 7.976107462819775e-07,
      "log_odds_chosen": 1.548242211341858,
      "log_odds_ratio": -0.3906005918979645,
      "logits/chosen": -1.024999976158142,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.6646484136581421,
      "logps/rejected": -1.758203148841858,
      "loss": 0.7017,
      "nll_loss": 0.6932617425918579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06651611626148224,
      "rewards/margins": 0.10919189453125,
      "rewards/rejected": -0.17578125,
      "step": 10060
    },
    {
      "epoch": 0.7641523751707391,
      "grad_norm": 3.3296288665819134,
      "learning_rate": 7.97214614771931e-07,
      "log_odds_chosen": 1.6428711414337158,
      "log_odds_ratio": -0.34406739473342896,
      "logits/chosen": -1.0398437976837158,
      "logits/rejected": -0.876171886920929,
      "logps/chosen": -0.574902355670929,
      "logps/rejected": -1.681249976158142,
      "loss": 0.6887,
      "nll_loss": 0.647167980670929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05750732496380806,
      "rewards/margins": 0.110443115234375,
      "rewards/rejected": -0.16794434189796448,
      "step": 10070
    },
    {
      "epoch": 0.7649112156624678,
      "grad_norm": 2.7697040157598365,
      "learning_rate": 7.968190728895957e-07,
      "log_odds_chosen": 1.634765625,
      "log_odds_ratio": -0.37617188692092896,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.890429675579071,
      "logps/chosen": -0.6016601324081421,
      "logps/rejected": -1.729882836341858,
      "loss": 0.7182,
      "nll_loss": 0.662109375,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06016845628619194,
      "rewards/margins": 0.11286620795726776,
      "rewards/rejected": -0.17312011122703552,
      "step": 10080
    },
    {
      "epoch": 0.7656700561541964,
      "grad_norm": 2.590517647317679,
      "learning_rate": 7.964241191736886e-07,
      "log_odds_chosen": 1.5373046398162842,
      "log_odds_ratio": -0.373046875,
      "logits/chosen": -1.015039086341858,
      "logits/rejected": -0.9029296636581421,
      "logps/chosen": -0.630664050579071,
      "logps/rejected": -1.6794922351837158,
      "loss": 0.6863,
      "nll_loss": 0.633496105670929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.0631103515625,
      "rewards/margins": 0.10477294772863388,
      "rewards/rejected": -0.16787108778953552,
      "step": 10090
    },
    {
      "epoch": 0.7664288966459251,
      "grad_norm": 2.889063019887122,
      "learning_rate": 7.960297521679913e-07,
      "log_odds_chosen": 1.3564331531524658,
      "log_odds_ratio": -0.40478515625,
      "logits/chosen": -0.9580078125,
      "logits/rejected": -0.837109386920929,
      "logps/chosen": -0.650097668170929,
      "logps/rejected": -1.5216796398162842,
      "loss": 0.6949,
      "nll_loss": 0.65380859375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06508789211511612,
      "rewards/margins": 0.08707733452320099,
      "rewards/rejected": -0.15217284858226776,
      "step": 10100
    },
    {
      "epoch": 0.7671877371376536,
      "grad_norm": 2.804976762124133,
      "learning_rate": 7.956359704213283e-07,
      "log_odds_chosen": 1.5294921398162842,
      "log_odds_ratio": -0.4141601622104645,
      "logits/chosen": -1.0029296875,
      "logits/rejected": -0.901171863079071,
      "logps/chosen": -0.650683581829071,
      "logps/rejected": -1.66015625,
      "loss": 0.6875,
      "nll_loss": 0.6455078125,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06502685695886612,
      "rewards/margins": 0.10102691501379013,
      "rewards/rejected": -0.16611328721046448,
      "step": 10110
    },
    {
      "epoch": 0.7679465776293823,
      "grad_norm": 2.9243913708279665,
      "learning_rate": 7.95242772487544e-07,
      "log_odds_chosen": 1.719140648841858,
      "log_odds_ratio": -0.34589844942092896,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.9521484375,
      "logps/chosen": -0.6297851800918579,
      "logps/rejected": -1.8455078601837158,
      "loss": 0.6819,
      "nll_loss": 0.679980456829071,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06296386569738388,
      "rewards/margins": 0.12154541164636612,
      "rewards/rejected": -0.18454590439796448,
      "step": 10120
    },
    {
      "epoch": 0.7687054181211109,
      "grad_norm": 2.737786103925968,
      "learning_rate": 7.94850156925481e-07,
      "log_odds_chosen": 1.193017601966858,
      "log_odds_ratio": -0.4439941346645355,
      "logits/chosen": -0.9976562261581421,
      "logits/rejected": -0.9056640863418579,
      "logps/chosen": -0.6734374761581421,
      "logps/rejected": -1.473046898841858,
      "loss": 0.7262,
      "nll_loss": 0.763378918170929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0673828125,
      "rewards/margins": 0.07997741550207138,
      "rewards/rejected": -0.14729003608226776,
      "step": 10130
    },
    {
      "epoch": 0.7694642586128396,
      "grad_norm": 2.6542219235735125,
      "learning_rate": 7.944581222989574e-07,
      "log_odds_chosen": 1.5681641101837158,
      "log_odds_ratio": -0.3794921934604645,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.9140625,
      "logps/chosen": -0.632519543170929,
      "logps/rejected": -1.7082030773162842,
      "loss": 0.6957,
      "nll_loss": 0.6753906011581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.063232421875,
      "rewards/margins": 0.10761108249425888,
      "rewards/rejected": -0.1708984375,
      "step": 10140
    },
    {
      "epoch": 0.7702230991045682,
      "grad_norm": 2.934006755318837,
      "learning_rate": 7.940666671767441e-07,
      "log_odds_chosen": 1.9152343273162842,
      "log_odds_ratio": -0.31494140625,
      "logits/chosen": -1.048242211341858,
      "logits/rejected": -0.910351574420929,
      "logps/chosen": -0.585742175579071,
      "logps/rejected": -1.9140625,
      "loss": 0.697,
      "nll_loss": 0.602343738079071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05849609524011612,
      "rewards/margins": 0.13291016221046448,
      "rewards/rejected": -0.1915283203125,
      "step": 10150
    },
    {
      "epoch": 0.7709819395962969,
      "grad_norm": 3.0611759379035877,
      "learning_rate": 7.936757901325451e-07,
      "log_odds_chosen": 1.6902344226837158,
      "log_odds_ratio": -0.3733886778354645,
      "logits/chosen": -1.0603516101837158,
      "logits/rejected": -0.9271484613418579,
      "logps/chosen": -0.60693359375,
      "logps/rejected": -1.767187476158142,
      "loss": 0.6787,
      "nll_loss": 0.5677734613418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06070556491613388,
      "rewards/margins": 0.11605224758386612,
      "rewards/rejected": -0.17666015028953552,
      "step": 10160
    },
    {
      "epoch": 0.7717407800880255,
      "grad_norm": 2.831114281044147,
      "learning_rate": 7.932854897449727e-07,
      "log_odds_chosen": 1.3722655773162842,
      "log_odds_ratio": -0.383056640625,
      "logits/chosen": -0.9716796875,
      "logits/rejected": -0.8671875,
      "logps/chosen": -0.649121105670929,
      "logps/rejected": -1.567968726158142,
      "loss": 0.6732,
      "nll_loss": 0.6712890863418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06489257514476776,
      "rewards/margins": 0.09174804389476776,
      "rewards/rejected": -0.15664061903953552,
      "step": 10170
    },
    {
      "epoch": 0.7724996205797542,
      "grad_norm": 3.458788080195538,
      "learning_rate": 7.928957645975286e-07,
      "log_odds_chosen": 1.563085913658142,
      "log_odds_ratio": -0.3597412109375,
      "logits/chosen": -1.033593773841858,
      "logits/rejected": -0.92578125,
      "logps/chosen": -0.6097656488418579,
      "logps/rejected": -1.650390625,
      "loss": 0.7044,
      "nll_loss": 0.66259765625,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06097412109375,
      "rewards/margins": 0.10419921576976776,
      "rewards/rejected": -0.16513672471046448,
      "step": 10180
    },
    {
      "epoch": 0.7732584610714828,
      "grad_norm": 2.9175591936403036,
      "learning_rate": 7.925066132785799e-07,
      "log_odds_chosen": 1.3307616710662842,
      "log_odds_ratio": -0.4554199278354645,
      "logits/chosen": -1.0378906726837158,
      "logits/rejected": -0.923046886920929,
      "logps/chosen": -0.6590820550918579,
      "logps/rejected": -1.5546875,
      "loss": 0.6901,
      "nll_loss": 0.708203136920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06583251804113388,
      "rewards/margins": 0.08952637016773224,
      "rewards/rejected": -0.15546874701976776,
      "step": 10190
    },
    {
      "epoch": 0.7740173015632115,
      "grad_norm": 3.455275310817514,
      "learning_rate": 7.921180343813395e-07,
      "log_odds_chosen": 1.446679711341858,
      "log_odds_ratio": -0.39042967557907104,
      "logits/chosen": -1.0119140148162842,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -0.6405273675918579,
      "logps/rejected": -1.6183593273162842,
      "loss": 0.6807,
      "nll_loss": 0.618847668170929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06400146335363388,
      "rewards/margins": 0.09783630073070526,
      "rewards/rejected": -0.16181640326976776,
      "step": 10200
    },
    {
      "epoch": 0.77477614205494,
      "grad_norm": 3.7695760516870864,
      "learning_rate": 7.917300265038436e-07,
      "log_odds_chosen": 1.628515601158142,
      "log_odds_ratio": -0.34814453125,
      "logits/chosen": -1.0031249523162842,
      "logits/rejected": -0.877148449420929,
      "logps/chosen": -0.599609375,
      "logps/rejected": -1.7175781726837158,
      "loss": 0.6878,
      "nll_loss": 0.6944335699081421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05991210788488388,
      "rewards/margins": 0.11183471977710724,
      "rewards/rejected": -0.17167969048023224,
      "step": 10210
    },
    {
      "epoch": 0.7755349825466686,
      "grad_norm": 2.973202088725128,
      "learning_rate": 7.913425882489307e-07,
      "log_odds_chosen": 1.630859375,
      "log_odds_ratio": -0.36127930879592896,
      "logits/chosen": -1.0343749523162842,
      "logits/rejected": -0.931835949420929,
      "logps/chosen": -0.6068359613418579,
      "logps/rejected": -1.693750023841858,
      "loss": 0.6976,
      "nll_loss": 0.6380859613418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06064452975988388,
      "rewards/margins": 0.10864868015050888,
      "rewards/rejected": -0.16938476264476776,
      "step": 10220
    },
    {
      "epoch": 0.7762938230383973,
      "grad_norm": 3.6670366049631413,
      "learning_rate": 7.909557182242211e-07,
      "log_odds_chosen": 1.613671898841858,
      "log_odds_ratio": -0.35273438692092896,
      "logits/chosen": -1.018945336341858,
      "logits/rejected": -0.9203125238418579,
      "logps/chosen": -0.624218761920929,
      "logps/rejected": -1.687890648841858,
      "loss": 0.6924,
      "nll_loss": 0.6534179449081421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06246338039636612,
      "rewards/margins": 0.10626220703125,
      "rewards/rejected": -0.16862793266773224,
      "step": 10230
    },
    {
      "epoch": 0.7770526635301259,
      "grad_norm": 2.89567532338407,
      "learning_rate": 7.905694150420947e-07,
      "log_odds_chosen": 1.6171875,
      "log_odds_ratio": -0.35419923067092896,
      "logits/chosen": -1.0603516101837158,
      "logits/rejected": -0.9267578125,
      "logps/chosen": -0.644238293170929,
      "logps/rejected": -1.763671875,
      "loss": 0.6799,
      "nll_loss": 0.630859375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06448974460363388,
      "rewards/margins": 0.1116943359375,
      "rewards/rejected": -0.1761474609375,
      "step": 10240
    },
    {
      "epoch": 0.7778115040218546,
      "grad_norm": 2.6710109666188484,
      "learning_rate": 7.901836773196717e-07,
      "log_odds_chosen": 1.828125,
      "log_odds_ratio": -0.34614259004592896,
      "logits/chosen": -1.0457031726837158,
      "logits/rejected": -0.9556640386581421,
      "logps/chosen": -0.625292956829071,
      "logps/rejected": -1.943750023841858,
      "loss": 0.6674,
      "nll_loss": 0.6556640863418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06254883110523224,
      "rewards/margins": 0.13184814155101776,
      "rewards/rejected": -0.19436034560203552,
      "step": 10250
    },
    {
      "epoch": 0.7785703445135832,
      "grad_norm": 3.2640273917813425,
      "learning_rate": 7.897985036787898e-07,
      "log_odds_chosen": 1.8986327648162842,
      "log_odds_ratio": -0.32421875,
      "logits/chosen": -1.0261719226837158,
      "logits/rejected": -0.8980468511581421,
      "logps/chosen": -0.6161133050918579,
      "logps/rejected": -1.933203101158142,
      "loss": 0.6974,
      "nll_loss": 0.679492175579071,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06160888820886612,
      "rewards/margins": 0.13151855766773224,
      "rewards/rejected": -0.193115234375,
      "step": 10260
    },
    {
      "epoch": 0.7793291850053119,
      "grad_norm": 2.9366417247959404,
      "learning_rate": 7.894138927459853e-07,
      "log_odds_chosen": 1.665429711341858,
      "log_odds_ratio": -0.3697265684604645,
      "logits/chosen": -1.0388672351837158,
      "logits/rejected": -0.908007800579071,
      "logps/chosen": -0.6659179925918579,
      "logps/rejected": -1.8253905773162842,
      "loss": 0.7017,
      "nll_loss": 0.6714843511581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.066650390625,
      "rewards/margins": 0.11589355766773224,
      "rewards/rejected": -0.18251952528953552,
      "step": 10270
    },
    {
      "epoch": 0.7800880254970405,
      "grad_norm": 2.5126768378557043,
      "learning_rate": 7.890298431524716e-07,
      "log_odds_chosen": 1.4007079601287842,
      "log_odds_ratio": -0.42753905057907104,
      "logits/chosen": -1.0460937023162842,
      "logits/rejected": -0.9189453125,
      "logps/chosen": -0.6494140625,
      "logps/rejected": -1.6115233898162842,
      "loss": 0.6748,
      "nll_loss": 0.620312511920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06492920219898224,
      "rewards/margins": 0.0963134765625,
      "rewards/rejected": -0.16123047471046448,
      "step": 10280
    },
    {
      "epoch": 0.7808468659887692,
      "grad_norm": 3.480661630568177,
      "learning_rate": 7.88646353534119e-07,
      "log_odds_chosen": 1.5300781726837158,
      "log_odds_ratio": -0.38383787870407104,
      "logits/chosen": -1.0408203601837158,
      "logits/rejected": -0.939648449420929,
      "logps/chosen": -0.624804675579071,
      "logps/rejected": -1.701562523841858,
      "loss": 0.6927,
      "nll_loss": 0.676953136920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06248779222369194,
      "rewards/margins": 0.10773925483226776,
      "rewards/rejected": -0.17006835341453552,
      "step": 10290
    },
    {
      "epoch": 0.7816057064804978,
      "grad_norm": 3.0588677544299805,
      "learning_rate": 7.882634225314345e-07,
      "log_odds_chosen": 1.7869141101837158,
      "log_odds_ratio": -0.3658691346645355,
      "logits/chosen": -1.0183594226837158,
      "logits/rejected": -0.9027343988418579,
      "logps/chosen": -0.657519519329071,
      "logps/rejected": -1.9421875476837158,
      "loss": 0.6956,
      "nll_loss": 0.716601550579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06577148288488388,
      "rewards/margins": 0.12856444716453552,
      "rewards/rejected": -0.19428710639476776,
      "step": 10300
    },
    {
      "epoch": 0.7823645469722265,
      "grad_norm": 2.616246816113427,
      "learning_rate": 7.87881048789541e-07,
      "log_odds_chosen": 1.540380835533142,
      "log_odds_ratio": -0.3597656190395355,
      "logits/chosen": -1.0251953601837158,
      "logits/rejected": -0.925976574420929,
      "logps/chosen": -0.619335949420929,
      "logps/rejected": -1.664648413658142,
      "loss": 0.7113,
      "nll_loss": 0.661914050579071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06192626804113388,
      "rewards/margins": 0.10466613620519638,
      "rewards/rejected": -0.1666259765625,
      "step": 10310
    },
    {
      "epoch": 0.783123387463955,
      "grad_norm": 2.7555140409616024,
      "learning_rate": 7.874992309581578e-07,
      "log_odds_chosen": 1.4484374523162842,
      "log_odds_ratio": -0.4012695252895355,
      "logits/chosen": -1.0251953601837158,
      "logits/rejected": -0.9443359375,
      "logps/chosen": -0.614550769329071,
      "logps/rejected": -1.5984375476837158,
      "loss": 0.7032,
      "nll_loss": 0.646777331829071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06148681789636612,
      "rewards/margins": 0.09844360500574112,
      "rewards/rejected": -0.159912109375,
      "step": 10320
    },
    {
      "epoch": 0.7838822279556837,
      "grad_norm": 3.7576720172269034,
      "learning_rate": 7.871179676915801e-07,
      "log_odds_chosen": 1.568945288658142,
      "log_odds_ratio": -0.3597168028354645,
      "logits/chosen": -1.032812476158142,
      "logits/rejected": -0.9339843988418579,
      "logps/chosen": -0.6025390625,
      "logps/rejected": -1.640234351158142,
      "loss": 0.693,
      "nll_loss": 0.625683605670929,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06025390699505806,
      "rewards/margins": 0.103759765625,
      "rewards/rejected": -0.163818359375,
      "step": 10330
    },
    {
      "epoch": 0.7846410684474123,
      "grad_norm": 2.8219829226384188,
      "learning_rate": 7.867372576486597e-07,
      "log_odds_chosen": 1.905859351158142,
      "log_odds_ratio": -0.31743162870407104,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.8716796636581421,
      "logps/chosen": -0.623046875,
      "logps/rejected": -2.000781297683716,
      "loss": 0.6999,
      "nll_loss": 0.605664074420929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06224365159869194,
      "rewards/margins": 0.1378173828125,
      "rewards/rejected": -0.19987793266773224,
      "step": 10340
    },
    {
      "epoch": 0.785399908939141,
      "grad_norm": 3.625150449691406,
      "learning_rate": 7.863570994927847e-07,
      "log_odds_chosen": 1.752343773841858,
      "log_odds_ratio": -0.344970703125,
      "logits/chosen": -1.0222656726837158,
      "logits/rejected": -0.9146484136581421,
      "logps/chosen": -0.626757800579071,
      "logps/rejected": -1.887109398841858,
      "loss": 0.7058,
      "nll_loss": 0.6971679925918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06267090141773224,
      "rewards/margins": 0.12612304091453552,
      "rewards/rejected": -0.18876953423023224,
      "step": 10350
    },
    {
      "epoch": 0.7861587494308696,
      "grad_norm": 5.24895860587327,
      "learning_rate": 7.859774918918594e-07,
      "log_odds_chosen": 1.74365234375,
      "log_odds_ratio": -0.3735412657260895,
      "logits/chosen": -1.027734398841858,
      "logits/rejected": -0.925000011920929,
      "logps/chosen": -0.627636730670929,
      "logps/rejected": -1.875390648841858,
      "loss": 0.6903,
      "nll_loss": 0.696484386920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06280517578125,
      "rewards/margins": 0.12468872219324112,
      "rewards/rejected": -0.18752440810203552,
      "step": 10360
    },
    {
      "epoch": 0.7869175899225983,
      "grad_norm": 2.877844721057341,
      "learning_rate": 7.855984335182852e-07,
      "log_odds_chosen": 1.6216552257537842,
      "log_odds_ratio": -0.3968261778354645,
      "logits/chosen": -0.999804675579071,
      "logits/rejected": -0.90625,
      "logps/chosen": -0.6620117425918579,
      "logps/rejected": -1.7605469226837158,
      "loss": 0.6876,
      "nll_loss": 0.708984375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06618652492761612,
      "rewards/margins": 0.11000366508960724,
      "rewards/rejected": -0.17622070014476776,
      "step": 10370
    },
    {
      "epoch": 0.7876764304143269,
      "grad_norm": 3.0125370416159227,
      "learning_rate": 7.852199230489422e-07,
      "log_odds_chosen": 1.6015625,
      "log_odds_ratio": -0.3634277284145355,
      "logits/chosen": -1.013671875,
      "logits/rejected": -0.892578125,
      "logps/chosen": -0.6142578125,
      "logps/rejected": -1.6857421398162842,
      "loss": 0.6669,
      "nll_loss": 0.698046863079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0614013671875,
      "rewards/margins": 0.10732422024011612,
      "rewards/rejected": -0.16879883408546448,
      "step": 10380
    },
    {
      "epoch": 0.7884352709060556,
      "grad_norm": 3.500203976437138,
      "learning_rate": 7.848419591651668e-07,
      "log_odds_chosen": 1.5294921398162842,
      "log_odds_ratio": -0.380615234375,
      "logits/chosen": -1.068359375,
      "logits/rejected": -0.9359375238418579,
      "logps/chosen": -0.646484375,
      "logps/rejected": -1.6984374523162842,
      "loss": 0.6812,
      "nll_loss": 0.6226562261581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06463623046875,
      "rewards/margins": 0.105224609375,
      "rewards/rejected": -0.169921875,
      "step": 10390
    },
    {
      "epoch": 0.7891941113977842,
      "grad_norm": 2.8987018674354106,
      "learning_rate": 7.844645405527361e-07,
      "log_odds_chosen": 1.48828125,
      "log_odds_ratio": -0.40180665254592896,
      "logits/chosen": -1.0261719226837158,
      "logits/rejected": -0.893750011920929,
      "logps/chosen": -0.63671875,
      "logps/rejected": -1.6554687023162842,
      "loss": 0.7035,
      "nll_loss": 0.73876953125,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06365966796875,
      "rewards/margins": 0.10177002102136612,
      "rewards/rejected": -0.16557617485523224,
      "step": 10400
    },
    {
      "epoch": 0.7899529518895129,
      "grad_norm": 3.2006644603647145,
      "learning_rate": 7.840876659018457e-07,
      "log_odds_chosen": 1.418554663658142,
      "log_odds_ratio": -0.3944091796875,
      "logits/chosen": -1.033789038658142,
      "logits/rejected": -0.930468738079071,
      "logps/chosen": -0.6244140863418579,
      "logps/rejected": -1.578710913658142,
      "loss": 0.6882,
      "nll_loss": 0.660839855670929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06241454929113388,
      "rewards/margins": 0.09541626274585724,
      "rewards/rejected": -0.15793457627296448,
      "step": 10410
    },
    {
      "epoch": 0.7907117923812415,
      "grad_norm": 3.271789563613443,
      "learning_rate": 7.837113339070922e-07,
      "log_odds_chosen": 1.663476586341858,
      "log_odds_ratio": -0.35078126192092896,
      "logits/chosen": -1.033789038658142,
      "logits/rejected": -0.8951171636581421,
      "logps/chosen": -0.5811523199081421,
      "logps/rejected": -1.7021484375,
      "loss": 0.6982,
      "nll_loss": 0.6133788824081421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05809326097369194,
      "rewards/margins": 0.112091064453125,
      "rewards/rejected": -0.17011718451976776,
      "step": 10420
    },
    {
      "epoch": 0.7914706328729701,
      "grad_norm": 2.638641821698903,
      "learning_rate": 7.833355432674538e-07,
      "log_odds_chosen": 1.2238280773162842,
      "log_odds_ratio": -0.4422363340854645,
      "logits/chosen": -0.980664074420929,
      "logits/rejected": -0.869335949420929,
      "logps/chosen": -0.6771484613418579,
      "logps/rejected": -1.516015648841858,
      "loss": 0.6971,
      "nll_loss": 0.7525390386581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06773681938648224,
      "rewards/margins": 0.08385010063648224,
      "rewards/rejected": -0.15156249701976776,
      "step": 10430
    },
    {
      "epoch": 0.7922294733646987,
      "grad_norm": 3.3915758168243575,
      "learning_rate": 7.829602926862713e-07,
      "log_odds_chosen": 1.505957007408142,
      "log_odds_ratio": -0.4209960997104645,
      "logits/chosen": -1.037109375,
      "logits/rejected": -0.929492175579071,
      "logps/chosen": -0.6786133050918579,
      "logps/rejected": -1.730859398841858,
      "loss": 0.6809,
      "nll_loss": 0.6971679925918579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06781005859375,
      "rewards/margins": 0.10518798977136612,
      "rewards/rejected": -0.173095703125,
      "step": 10440
    },
    {
      "epoch": 0.7929883138564274,
      "grad_norm": 3.177062851435062,
      "learning_rate": 7.825855808712296e-07,
      "log_odds_chosen": 1.692773461341858,
      "log_odds_ratio": -0.39497071504592896,
      "logits/chosen": -1.024023413658142,
      "logits/rejected": -0.9105468988418579,
      "logps/chosen": -0.5873047113418579,
      "logps/rejected": -1.796875,
      "loss": 0.6961,
      "nll_loss": 0.605664074420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.05870361253619194,
      "rewards/margins": 0.1209716796875,
      "rewards/rejected": -0.17983397841453552,
      "step": 10450
    },
    {
      "epoch": 0.793747154348156,
      "grad_norm": 4.055077815203594,
      "learning_rate": 7.822114065343386e-07,
      "log_odds_chosen": 1.661523461341858,
      "log_odds_ratio": -0.3618408143520355,
      "logits/chosen": -1.0320312976837158,
      "logits/rejected": -0.910937488079071,
      "logps/chosen": -0.600390613079071,
      "logps/rejected": -1.755273461341858,
      "loss": 0.6876,
      "nll_loss": 0.6451171636581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06002197414636612,
      "rewards/margins": 0.11531982570886612,
      "rewards/rejected": -0.17536620795726776,
      "step": 10460
    },
    {
      "epoch": 0.7945059948398847,
      "grad_norm": 2.722963411034344,
      "learning_rate": 7.818377683919149e-07,
      "log_odds_chosen": 1.4983398914337158,
      "log_odds_ratio": -0.37346190214157104,
      "logits/chosen": -1.032812476158142,
      "logits/rejected": -0.9085937738418579,
      "logps/chosen": -0.6153320074081421,
      "logps/rejected": -1.6007812023162842,
      "loss": 0.6632,
      "nll_loss": 0.5763183832168579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06156005710363388,
      "rewards/margins": 0.09859924018383026,
      "rewards/rejected": -0.15993651747703552,
      "step": 10470
    },
    {
      "epoch": 0.7952648353316133,
      "grad_norm": 4.276289329814106,
      "learning_rate": 7.814646651645635e-07,
      "log_odds_chosen": 1.967382788658142,
      "log_odds_ratio": -0.3251953125,
      "logits/chosen": -1.00390625,
      "logits/rejected": -0.9097656011581421,
      "logps/chosen": -0.609667956829071,
      "logps/rejected": -1.9787108898162842,
      "loss": 0.6599,
      "nll_loss": 0.675976574420929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06103515625,
      "rewards/margins": 0.13691405951976776,
      "rewards/rejected": -0.19790038466453552,
      "step": 10480
    },
    {
      "epoch": 0.7960236758233419,
      "grad_norm": 2.8626855547428023,
      "learning_rate": 7.810920955771586e-07,
      "log_odds_chosen": 1.660742163658142,
      "log_odds_ratio": -0.37578123807907104,
      "logits/chosen": -1.046484351158142,
      "logits/rejected": -0.922070324420929,
      "logps/chosen": -0.5972656011581421,
      "logps/rejected": -1.7550780773162842,
      "loss": 0.667,
      "nll_loss": 0.66455078125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.05971679836511612,
      "rewards/margins": 0.11589355766773224,
      "rewards/rejected": -0.17548827826976776,
      "step": 10490
    },
    {
      "epoch": 0.7967825163150706,
      "grad_norm": 3.3009989381681706,
      "learning_rate": 7.807200583588266e-07,
      "log_odds_chosen": 1.472265601158142,
      "log_odds_ratio": -0.3922363221645355,
      "logits/chosen": -1.025390625,
      "logits/rejected": -0.9453125,
      "logps/chosen": -0.5921875238418579,
      "logps/rejected": -1.5568358898162842,
      "loss": 0.662,
      "nll_loss": 0.6484375,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.05916748195886612,
      "rewards/margins": 0.09642334282398224,
      "rewards/rejected": -0.15559081733226776,
      "step": 10500
    },
    {
      "epoch": 0.7975413568067992,
      "grad_norm": 3.0959046717459064,
      "learning_rate": 7.803485522429261e-07,
      "log_odds_chosen": 2.033007860183716,
      "log_odds_ratio": -0.3226074278354645,
      "logits/chosen": -1.075585961341858,
      "logits/rejected": -0.955859363079071,
      "logps/chosen": -0.6126953363418579,
      "logps/rejected": -2.0511717796325684,
      "loss": 0.6935,
      "nll_loss": 0.591503918170929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.061279296875,
      "rewards/margins": 0.14405517280101776,
      "rewards/rejected": -0.205322265625,
      "step": 10510
    },
    {
      "epoch": 0.7983001972985279,
      "grad_norm": 3.082003314894203,
      "learning_rate": 7.799775759670318e-07,
      "log_odds_chosen": 1.655859351158142,
      "log_odds_ratio": -0.3758789002895355,
      "logits/chosen": -0.985156238079071,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.6073242425918579,
      "logps/rejected": -1.774999976158142,
      "loss": 0.6678,
      "nll_loss": 0.610156238079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06074218824505806,
      "rewards/margins": 0.11672363430261612,
      "rewards/rejected": -0.17756347358226776,
      "step": 10520
    },
    {
      "epoch": 0.7990590377902564,
      "grad_norm": 2.7762354617200504,
      "learning_rate": 7.796071282729149e-07,
      "log_odds_chosen": 1.662500023841858,
      "log_odds_ratio": -0.3458496034145355,
      "logits/chosen": -1.0431640148162842,
      "logits/rejected": -0.894335925579071,
      "logps/chosen": -0.599804699420929,
      "logps/rejected": -1.720703125,
      "loss": 0.6784,
      "nll_loss": 0.6410156488418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05997314304113388,
      "rewards/margins": 0.11212158203125,
      "rewards/rejected": -0.17202147841453552,
      "step": 10530
    },
    {
      "epoch": 0.7998178782819851,
      "grad_norm": 2.5878751843031074,
      "learning_rate": 7.792372079065259e-07,
      "log_odds_chosen": 1.486083984375,
      "log_odds_ratio": -0.3973144590854645,
      "logits/chosen": -0.985546886920929,
      "logits/rejected": -0.9019531011581421,
      "logps/chosen": -0.6236327886581421,
      "logps/rejected": -1.6310546398162842,
      "loss": 0.6848,
      "nll_loss": 0.6357421875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0623779296875,
      "rewards/margins": 0.100738525390625,
      "rewards/rejected": -0.16302490234375,
      "step": 10540
    },
    {
      "epoch": 0.8005767187737137,
      "grad_norm": 2.8143631076062054,
      "learning_rate": 7.788678136179767e-07,
      "log_odds_chosen": 1.640039086341858,
      "log_odds_ratio": -0.385009765625,
      "logits/chosen": -1.0304687023162842,
      "logits/rejected": -0.9560546875,
      "logps/chosen": -0.6664062738418579,
      "logps/rejected": -1.808007836341858,
      "loss": 0.6594,
      "nll_loss": 0.60791015625,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06671142578125,
      "rewards/margins": 0.114044189453125,
      "rewards/rejected": -0.1807861328125,
      "step": 10550
    },
    {
      "epoch": 0.8013355592654424,
      "grad_norm": 2.639337373503303,
      "learning_rate": 7.78498944161523e-07,
      "log_odds_chosen": 1.4988281726837158,
      "log_odds_ratio": -0.39936524629592896,
      "logits/chosen": -1.027929663658142,
      "logits/rejected": -0.9330078363418579,
      "logps/chosen": -0.671093761920929,
      "logps/rejected": -1.712499976158142,
      "loss": 0.6762,
      "nll_loss": 0.6640625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06710205227136612,
      "rewards/margins": 0.10411377251148224,
      "rewards/rejected": -0.17128905653953552,
      "step": 10560
    },
    {
      "epoch": 0.802094399757171,
      "grad_norm": 3.3918458821674546,
      "learning_rate": 7.781305982955459e-07,
      "log_odds_chosen": 1.6994140148162842,
      "log_odds_ratio": -0.34062498807907104,
      "logits/chosen": -1.016992211341858,
      "logits/rejected": -0.8999999761581421,
      "logps/chosen": -0.618457019329071,
      "logps/rejected": -1.764062523841858,
      "loss": 0.7041,
      "nll_loss": 0.6913086175918579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06185302883386612,
      "rewards/margins": 0.11441650241613388,
      "rewards/rejected": -0.17626953125,
      "step": 10570
    },
    {
      "epoch": 0.8028532402488997,
      "grad_norm": 2.9449769083438473,
      "learning_rate": 7.777627747825355e-07,
      "log_odds_chosen": 1.641015648841858,
      "log_odds_ratio": -0.350830078125,
      "logits/chosen": -1.0486328601837158,
      "logits/rejected": -0.900195300579071,
      "logps/chosen": -0.6241210699081421,
      "logps/rejected": -1.766210913658142,
      "loss": 0.6812,
      "nll_loss": 0.6424804925918579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06243896484375,
      "rewards/margins": 0.11414184421300888,
      "rewards/rejected": -0.17656250298023224,
      "step": 10580
    },
    {
      "epoch": 0.8036120807406283,
      "grad_norm": 2.9580149516718817,
      "learning_rate": 7.773954723890725e-07,
      "log_odds_chosen": 1.7490723133087158,
      "log_odds_ratio": -0.34064942598342896,
      "logits/chosen": -0.987500011920929,
      "logits/rejected": -0.8984375,
      "logps/chosen": -0.598828136920929,
      "logps/rejected": -1.8154296875,
      "loss": 0.6614,
      "nll_loss": 0.635449230670929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05983886867761612,
      "rewards/margins": 0.12144164741039276,
      "rewards/rejected": -0.18137207627296448,
      "step": 10590
    },
    {
      "epoch": 0.804370921232357,
      "grad_norm": 3.2056892182318473,
      "learning_rate": 7.770286898858113e-07,
      "log_odds_chosen": 1.677148461341858,
      "log_odds_ratio": -0.36640626192092896,
      "logits/chosen": -1.031835913658142,
      "logits/rejected": -0.9195312261581421,
      "logps/chosen": -0.64453125,
      "logps/rejected": -1.816796898841858,
      "loss": 0.6849,
      "nll_loss": 0.636425793170929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06440429389476776,
      "rewards/margins": 0.1171875,
      "rewards/rejected": -0.18166503310203552,
      "step": 10600
    },
    {
      "epoch": 0.8051297617240856,
      "grad_norm": 3.0418956800252457,
      "learning_rate": 7.766624260474625e-07,
      "log_odds_chosen": 1.728906273841858,
      "log_odds_ratio": -0.3870605528354645,
      "logits/chosen": -1.0402343273162842,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -0.6019531488418579,
      "logps/rejected": -1.8193359375,
      "loss": 0.6803,
      "nll_loss": 0.625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06016845628619194,
      "rewards/margins": 0.12171630561351776,
      "rewards/rejected": -0.18198242783546448,
      "step": 10610
    },
    {
      "epoch": 0.8058886022158143,
      "grad_norm": 3.62784286301264,
      "learning_rate": 7.762966796527759e-07,
      "log_odds_chosen": 1.5466797351837158,
      "log_odds_ratio": -0.3858886659145355,
      "logits/chosen": -1.0197265148162842,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.615429699420929,
      "logps/rejected": -1.6505858898162842,
      "loss": 0.6919,
      "nll_loss": 0.626660168170929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.0615234375,
      "rewards/margins": 0.10355224460363388,
      "rewards/rejected": -0.1651611328125,
      "step": 10620
    },
    {
      "epoch": 0.8066474427075428,
      "grad_norm": 3.4566602272106044,
      "learning_rate": 7.759314494845234e-07,
      "log_odds_chosen": 1.3899414539337158,
      "log_odds_ratio": -0.39814454317092896,
      "logits/chosen": -1.004296898841858,
      "logits/rejected": -0.9228515625,
      "logps/chosen": -0.618945300579071,
      "logps/rejected": -1.533789038658142,
      "loss": 0.6816,
      "nll_loss": 0.62646484375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06186523288488388,
      "rewards/margins": 0.09145507961511612,
      "rewards/rejected": -0.15329590439796448,
      "step": 10630
    },
    {
      "epoch": 0.8074062831992715,
      "grad_norm": 3.3488237361482596,
      "learning_rate": 7.755667343294812e-07,
      "log_odds_chosen": 1.536718726158142,
      "log_odds_ratio": -0.4296875,
      "logits/chosen": -0.9365234375,
      "logits/rejected": -0.883007824420929,
      "logps/chosen": -0.6763671636581421,
      "logps/rejected": -1.7541015148162842,
      "loss": 0.6838,
      "nll_loss": 0.66943359375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06766357272863388,
      "rewards/margins": 0.10777588188648224,
      "rewards/rejected": -0.17558594048023224,
      "step": 10640
    },
    {
      "epoch": 0.8081651236910001,
      "grad_norm": 4.524240916001708,
      "learning_rate": 7.752025329784146e-07,
      "log_odds_chosen": 1.6884765625,
      "log_odds_ratio": -0.372314453125,
      "logits/chosen": -1.048828125,
      "logits/rejected": -0.9341796636581421,
      "logps/chosen": -0.63916015625,
      "logps/rejected": -1.8328125476837158,
      "loss": 0.6794,
      "nll_loss": 0.582812488079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06387939304113388,
      "rewards/margins": 0.11937256157398224,
      "rewards/rejected": -0.1832275390625,
      "step": 10650
    },
    {
      "epoch": 0.8089239641827288,
      "grad_norm": 2.690281010121218,
      "learning_rate": 7.748388442260596e-07,
      "log_odds_chosen": 1.4500000476837158,
      "log_odds_ratio": -0.3978027403354645,
      "logits/chosen": -0.977734386920929,
      "logits/rejected": -0.866406261920929,
      "logps/chosen": -0.588574230670929,
      "logps/rejected": -1.533203125,
      "loss": 0.6812,
      "nll_loss": 0.6766601800918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05885009840130806,
      "rewards/margins": 0.0943450927734375,
      "rewards/rejected": -0.15327148139476776,
      "step": 10660
    },
    {
      "epoch": 0.8096828046744574,
      "grad_norm": 2.696343207878343,
      "learning_rate": 7.744756668711065e-07,
      "log_odds_chosen": 1.5031249523162842,
      "log_odds_ratio": -0.4110351502895355,
      "logits/chosen": -0.977343738079071,
      "logits/rejected": -0.915234386920929,
      "logps/chosen": -0.639941394329071,
      "logps/rejected": -1.621679663658142,
      "loss": 0.6871,
      "nll_loss": 0.636523425579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06395263969898224,
      "rewards/margins": 0.09816284477710724,
      "rewards/rejected": -0.16218261420726776,
      "step": 10670
    },
    {
      "epoch": 0.8104416451661861,
      "grad_norm": 3.6865004962594776,
      "learning_rate": 7.741129997161835e-07,
      "log_odds_chosen": 1.658203125,
      "log_odds_ratio": -0.38496094942092896,
      "logits/chosen": -1.034570336341858,
      "logits/rejected": -0.910351574420929,
      "logps/chosen": -0.5663086175918579,
      "logps/rejected": -1.6960937976837158,
      "loss": 0.6695,
      "nll_loss": 0.59912109375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.05660400539636612,
      "rewards/margins": 0.11300048977136612,
      "rewards/rejected": -0.1695556640625,
      "step": 10680
    },
    {
      "epoch": 0.8112004856579147,
      "grad_norm": 5.746400763660962,
      "learning_rate": 7.737508415678403e-07,
      "log_odds_chosen": 1.962890625,
      "log_odds_ratio": -0.3206420838832855,
      "logits/chosen": -1.029296875,
      "logits/rejected": -0.9271484613418579,
      "logps/chosen": -0.6006835699081421,
      "logps/rejected": -1.9874999523162842,
      "loss": 0.6476,
      "nll_loss": 0.564160168170929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06009521335363388,
      "rewards/margins": 0.1387939453125,
      "rewards/rejected": -0.1988525390625,
      "step": 10690
    },
    {
      "epoch": 0.8119593261496434,
      "grad_norm": 3.7556638876534403,
      "learning_rate": 7.733891912365308e-07,
      "log_odds_chosen": 1.2936522960662842,
      "log_odds_ratio": -0.447021484375,
      "logits/chosen": -0.9544922113418579,
      "logits/rejected": -0.8666015863418579,
      "logps/chosen": -0.656445324420929,
      "logps/rejected": -1.5285155773162842,
      "loss": 0.6721,
      "nll_loss": 0.6884765625,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06564941257238388,
      "rewards/margins": 0.08723144233226776,
      "rewards/rejected": -0.15291747450828552,
      "step": 10700
    },
    {
      "epoch": 0.812718166641372,
      "grad_norm": 3.123795174517137,
      "learning_rate": 7.730280475365979e-07,
      "log_odds_chosen": 1.7101562023162842,
      "log_odds_ratio": -0.36674803495407104,
      "logits/chosen": -1.0085937976837158,
      "logits/rejected": -0.8931640386581421,
      "logps/chosen": -0.638378918170929,
      "logps/rejected": -1.837890625,
      "loss": 0.6927,
      "nll_loss": 0.604687511920929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06381835788488388,
      "rewards/margins": 0.11988525092601776,
      "rewards/rejected": -0.18364258110523224,
      "step": 10710
    },
    {
      "epoch": 0.8134770071331007,
      "grad_norm": 2.8808675530077132,
      "learning_rate": 7.726674092862557e-07,
      "log_odds_chosen": 1.718652367591858,
      "log_odds_ratio": -0.344970703125,
      "logits/chosen": -0.980273425579071,
      "logits/rejected": -0.8623046875,
      "logps/chosen": -0.6200195550918579,
      "logps/rejected": -1.8230469226837158,
      "loss": 0.6867,
      "nll_loss": 0.6470702886581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06199951097369194,
      "rewards/margins": 0.120391845703125,
      "rewards/rejected": -0.18244628608226776,
      "step": 10720
    },
    {
      "epoch": 0.8142358476248293,
      "grad_norm": 2.739352808902024,
      "learning_rate": 7.723072753075748e-07,
      "log_odds_chosen": 1.6259765625,
      "log_odds_ratio": -0.388916015625,
      "logits/chosen": -0.9404296875,
      "logits/rejected": -0.8529297113418579,
      "logps/chosen": -0.649121105670929,
      "logps/rejected": -1.746484398841858,
      "loss": 0.6762,
      "nll_loss": 0.6102539300918579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06489257514476776,
      "rewards/margins": 0.10992431640625,
      "rewards/rejected": -0.17487792670726776,
      "step": 10730
    },
    {
      "epoch": 0.814994688116558,
      "grad_norm": 2.6384055155557293,
      "learning_rate": 7.719476444264649e-07,
      "log_odds_chosen": 1.66796875,
      "log_odds_ratio": -0.3416503965854645,
      "logits/chosen": -1.0779297351837158,
      "logits/rejected": -0.982617199420929,
      "logps/chosen": -0.6180664300918579,
      "logps/rejected": -1.713281273841858,
      "loss": 0.6739,
      "nll_loss": 0.6319335699081421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06181640550494194,
      "rewards/margins": 0.10963745415210724,
      "rewards/rejected": -0.17143554985523224,
      "step": 10740
    },
    {
      "epoch": 0.8157535286082865,
      "grad_norm": 3.4227329558508695,
      "learning_rate": 7.715885154726593e-07,
      "log_odds_chosen": 1.935156226158142,
      "log_odds_ratio": -0.351318359375,
      "logits/chosen": -1.016015648841858,
      "logits/rejected": -0.8974609375,
      "logps/chosen": -0.595507800579071,
      "logps/rejected": -1.94921875,
      "loss": 0.6564,
      "nll_loss": 0.6114257574081421,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05949706956744194,
      "rewards/margins": 0.13527831435203552,
      "rewards/rejected": -0.19489745795726776,
      "step": 10750
    },
    {
      "epoch": 0.8165123691000151,
      "grad_norm": 3.185608126388568,
      "learning_rate": 7.71229887279699e-07,
      "log_odds_chosen": 1.72802734375,
      "log_odds_ratio": -0.3501953184604645,
      "logits/chosen": -0.999218761920929,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.559374988079071,
      "logps/rejected": -1.691796898841858,
      "loss": 0.6607,
      "nll_loss": 0.568652331829071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05592041090130806,
      "rewards/margins": 0.11335144191980362,
      "rewards/rejected": -0.16938476264476776,
      "step": 10760
    },
    {
      "epoch": 0.8172712095917438,
      "grad_norm": 2.819173267169037,
      "learning_rate": 7.708717586849164e-07,
      "log_odds_chosen": 1.7556641101837158,
      "log_odds_ratio": -0.34326171875,
      "logits/chosen": -1.003515601158142,
      "logits/rejected": -0.839062511920929,
      "logps/chosen": -0.6163085699081421,
      "logps/rejected": -1.8605468273162842,
      "loss": 0.6817,
      "nll_loss": 0.6783202886581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06163330003619194,
      "rewards/margins": 0.12451171875,
      "rewards/rejected": -0.18605956435203552,
      "step": 10770
    },
    {
      "epoch": 0.8180300500834724,
      "grad_norm": 3.072289514394148,
      "learning_rate": 7.705141285294196e-07,
      "log_odds_chosen": 1.6414062976837158,
      "log_odds_ratio": -0.37089842557907104,
      "logits/chosen": -1.0544922351837158,
      "logits/rejected": -0.9378906488418579,
      "logps/chosen": -0.574999988079071,
      "logps/rejected": -1.701171875,
      "loss": 0.6572,
      "nll_loss": 0.5518554449081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05744628980755806,
      "rewards/margins": 0.11256103217601776,
      "rewards/rejected": -0.17001953721046448,
      "step": 10780
    },
    {
      "epoch": 0.8187888905752011,
      "grad_norm": 3.275955954456824,
      "learning_rate": 7.701569956580767e-07,
      "log_odds_chosen": 1.749609351158142,
      "log_odds_ratio": -0.32524412870407104,
      "logits/chosen": -1.0427734851837158,
      "logits/rejected": -0.9193359613418579,
      "logps/chosen": -0.5684570074081421,
      "logps/rejected": -1.7703125476837158,
      "loss": 0.6767,
      "nll_loss": 0.562695324420929,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.05682373046875,
      "rewards/margins": 0.12010498344898224,
      "rewards/rejected": -0.17695312201976776,
      "step": 10790
    },
    {
      "epoch": 0.8195477310669297,
      "grad_norm": 3.046546998413145,
      "learning_rate": 7.69800358919501e-07,
      "log_odds_chosen": 1.631250023841858,
      "log_odds_ratio": -0.4154296815395355,
      "logits/chosen": -1.005859375,
      "logits/rejected": -0.89453125,
      "logps/chosen": -0.666308581829071,
      "logps/rejected": -1.834375023841858,
      "loss": 0.6613,
      "nll_loss": 0.663769543170929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06666259467601776,
      "rewards/margins": 0.11674804985523224,
      "rewards/rejected": -0.18349608778953552,
      "step": 10800
    },
    {
      "epoch": 0.8203065715586584,
      "grad_norm": 2.846255653409117,
      "learning_rate": 7.694442171660333e-07,
      "log_odds_chosen": 1.5017578601837158,
      "log_odds_ratio": -0.42387694120407104,
      "logits/chosen": -1.025390625,
      "logits/rejected": -0.910351574420929,
      "logps/chosen": -0.6181640625,
      "logps/rejected": -1.6173827648162842,
      "loss": 0.6846,
      "nll_loss": 0.6197265386581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06182861328125,
      "rewards/margins": 0.09968261420726776,
      "rewards/rejected": -0.16154785454273224,
      "step": 10810
    },
    {
      "epoch": 0.821065412050387,
      "grad_norm": 2.92987510870056,
      "learning_rate": 7.690885692537282e-07,
      "log_odds_chosen": 1.654296875,
      "log_odds_ratio": -0.3650756776332855,
      "logits/chosen": -1.0460937023162842,
      "logits/rejected": -0.9085937738418579,
      "logps/chosen": -0.6031249761581421,
      "logps/rejected": -1.712499976158142,
      "loss": 0.6724,
      "nll_loss": 0.586718738079071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06026611477136612,
      "rewards/margins": 0.11093749850988388,
      "rewards/rejected": -0.1712646484375,
      "step": 10820
    },
    {
      "epoch": 0.8218242525421157,
      "grad_norm": 3.025305174015617,
      "learning_rate": 7.687334140423383e-07,
      "log_odds_chosen": 1.852929711341858,
      "log_odds_ratio": -0.34326171875,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.8666015863418579,
      "logps/chosen": -0.6119140386581421,
      "logps/rejected": -1.8955078125,
      "loss": 0.6854,
      "nll_loss": 0.6749023199081421,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06119384616613388,
      "rewards/margins": 0.12813720107078552,
      "rewards/rejected": -0.189453125,
      "step": 10830
    },
    {
      "epoch": 0.8225830930338442,
      "grad_norm": 3.248790996301624,
      "learning_rate": 7.683787503952985e-07,
      "log_odds_chosen": 1.478613257408142,
      "log_odds_ratio": -0.37158203125,
      "logits/chosen": -1.041601538658142,
      "logits/rejected": -0.9140625,
      "logps/chosen": -0.6617187261581421,
      "logps/rejected": -1.663476586341858,
      "loss": 0.6878,
      "nll_loss": 0.599902331829071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06618652492761612,
      "rewards/margins": 0.10013427585363388,
      "rewards/rejected": -0.16635742783546448,
      "step": 10840
    },
    {
      "epoch": 0.8233419335255729,
      "grad_norm": 3.4614424912797914,
      "learning_rate": 7.680245771797108e-07,
      "log_odds_chosen": 1.705078125,
      "log_odds_ratio": -0.3819335997104645,
      "logits/chosen": -0.9976562261581421,
      "logits/rejected": -0.895703136920929,
      "logps/chosen": -0.6337890625,
      "logps/rejected": -1.794531226158142,
      "loss": 0.6798,
      "nll_loss": 0.6270507574081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06333007663488388,
      "rewards/margins": 0.11602783203125,
      "rewards/rejected": -0.17941895127296448,
      "step": 10850
    },
    {
      "epoch": 0.8241007740173015,
      "grad_norm": 2.7843658544893897,
      "learning_rate": 7.676708932663293e-07,
      "log_odds_chosen": 1.712499976158142,
      "log_odds_ratio": -0.3712524473667145,
      "logits/chosen": -1.0158202648162842,
      "logits/rejected": -0.88671875,
      "logps/chosen": -0.6410156488418579,
      "logps/rejected": -1.7822265625,
      "loss": 0.6577,
      "nll_loss": 0.637988269329071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06409911811351776,
      "rewards/margins": 0.11424560844898224,
      "rewards/rejected": -0.17836913466453552,
      "step": 10860
    },
    {
      "epoch": 0.8248596145090302,
      "grad_norm": 3.207446551034803,
      "learning_rate": 7.67317697529545e-07,
      "log_odds_chosen": 1.87890625,
      "log_odds_ratio": -0.3067871034145355,
      "logits/chosen": -1.030859351158142,
      "logits/rejected": -0.9232422113418579,
      "logps/chosen": -0.5962890386581421,
      "logps/rejected": -1.896093726158142,
      "loss": 0.6657,
      "nll_loss": 0.5938476324081421,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.05963134765625,
      "rewards/margins": 0.12996825575828552,
      "rewards/rejected": -0.18955078721046448,
      "step": 10870
    },
    {
      "epoch": 0.8256184550007588,
      "grad_norm": 2.6072097815594795,
      "learning_rate": 7.669649888473704e-07,
      "log_odds_chosen": 1.540624976158142,
      "log_odds_ratio": -0.43315428495407104,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.891796886920929,
      "logps/chosen": -0.694140613079071,
      "logps/rejected": -1.8152344226837158,
      "loss": 0.6889,
      "nll_loss": 0.68115234375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06937255710363388,
      "rewards/margins": 0.11213378608226776,
      "rewards/rejected": -0.1815185546875,
      "step": 10880
    },
    {
      "epoch": 0.8263772954924875,
      "grad_norm": 3.2328209425598127,
      "learning_rate": 7.666127661014253e-07,
      "log_odds_chosen": 1.627539038658142,
      "log_odds_ratio": -0.3424316346645355,
      "logits/chosen": -0.9574218988418579,
      "logits/rejected": -0.85546875,
      "logps/chosen": -0.62548828125,
      "logps/rejected": -1.6960937976837158,
      "loss": 0.6889,
      "nll_loss": 0.6361328363418579,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06256103515625,
      "rewards/margins": 0.10711669921875,
      "rewards/rejected": -0.16977539658546448,
      "step": 10890
    },
    {
      "epoch": 0.8271361359842161,
      "grad_norm": 2.8619658612645926,
      "learning_rate": 7.662610281769211e-07,
      "log_odds_chosen": 1.6306641101837158,
      "log_odds_ratio": -0.38740235567092896,
      "logits/chosen": -1.0359375476837158,
      "logits/rejected": -0.9359375238418579,
      "logps/chosen": -0.6517578363418579,
      "logps/rejected": -1.799218773841858,
      "loss": 0.6741,
      "nll_loss": 0.6617187261581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.065185546875,
      "rewards/margins": 0.1148681640625,
      "rewards/rejected": -0.18007811903953552,
      "step": 10900
    },
    {
      "epoch": 0.8278949764759448,
      "grad_norm": 3.4497463503204133,
      "learning_rate": 7.659097739626465e-07,
      "log_odds_chosen": 1.616796851158142,
      "log_odds_ratio": -0.3377929627895355,
      "logits/chosen": -1.0447266101837158,
      "logits/rejected": -0.9310547113418579,
      "logps/chosen": -0.61181640625,
      "logps/rejected": -1.6574218273162842,
      "loss": 0.6759,
      "nll_loss": 0.630810558795929,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06124267727136612,
      "rewards/margins": 0.10463257133960724,
      "rewards/rejected": -0.16572265326976776,
      "step": 10910
    },
    {
      "epoch": 0.8286538169676734,
      "grad_norm": 3.7967047337664015,
      "learning_rate": 7.655590023509527e-07,
      "log_odds_chosen": 1.7556641101837158,
      "log_odds_ratio": -0.36591798067092896,
      "logits/chosen": -1.0246093273162842,
      "logits/rejected": -0.882519543170929,
      "logps/chosen": -0.646289050579071,
      "logps/rejected": -1.9054687023162842,
      "loss": 0.6609,
      "nll_loss": 0.6537109613418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06463623046875,
      "rewards/margins": 0.12578125298023224,
      "rewards/rejected": -0.190673828125,
      "step": 10920
    },
    {
      "epoch": 0.8294126574594021,
      "grad_norm": 2.7173877382246685,
      "learning_rate": 7.652087122377384e-07,
      "log_odds_chosen": 1.8125,
      "log_odds_ratio": -0.3530517518520355,
      "logits/chosen": -1.0753905773162842,
      "logits/rejected": -0.9886718988418579,
      "logps/chosen": -0.685742199420929,
      "logps/rejected": -1.927343726158142,
      "loss": 0.6603,
      "nll_loss": 0.665820300579071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06856689602136612,
      "rewards/margins": 0.12407837063074112,
      "rewards/rejected": -0.19272461533546448,
      "step": 10930
    },
    {
      "epoch": 0.8301714979511307,
      "grad_norm": 3.180938641596581,
      "learning_rate": 7.648589025224355e-07,
      "log_odds_chosen": 1.641015648841858,
      "log_odds_ratio": -0.354248046875,
      "logits/chosen": -1.020117163658142,
      "logits/rejected": -0.885937511920929,
      "logps/chosen": -0.6034179925918579,
      "logps/rejected": -1.7179687023162842,
      "loss": 0.6753,
      "nll_loss": 0.6224609613418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06036376953125,
      "rewards/margins": 0.11148681491613388,
      "rewards/rejected": -0.17189940810203552,
      "step": 10940
    },
    {
      "epoch": 0.8309303384428594,
      "grad_norm": 2.85614261298617,
      "learning_rate": 7.645095721079945e-07,
      "log_odds_chosen": 1.3601562976837158,
      "log_odds_ratio": -0.42119139432907104,
      "logits/chosen": -1.0119140148162842,
      "logits/rejected": -0.9525390863418579,
      "logps/chosen": -0.622363269329071,
      "logps/rejected": -1.510156273841858,
      "loss": 0.671,
      "nll_loss": 0.608593761920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06221923977136612,
      "rewards/margins": 0.08867187798023224,
      "rewards/rejected": -0.15095214545726776,
      "step": 10950
    },
    {
      "epoch": 0.8316891789345879,
      "grad_norm": 3.48823108001633,
      "learning_rate": 7.641607199008701e-07,
      "log_odds_chosen": 1.6598632335662842,
      "log_odds_ratio": -0.3478027284145355,
      "logits/chosen": -0.9849609136581421,
      "logits/rejected": -0.926953136920929,
      "logps/chosen": -0.586621105670929,
      "logps/rejected": -1.7384765148162842,
      "loss": 0.6681,
      "nll_loss": 0.5791015625,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05866699293255806,
      "rewards/margins": 0.1151580810546875,
      "rewards/rejected": -0.17380371689796448,
      "step": 10960
    },
    {
      "epoch": 0.8324480194263166,
      "grad_norm": 3.448118491439069,
      "learning_rate": 7.638123448110066e-07,
      "log_odds_chosen": 1.595703125,
      "log_odds_ratio": -0.38160401582717896,
      "logits/chosen": -1.0263671875,
      "logits/rejected": -0.943359375,
      "logps/chosen": -0.693164050579071,
      "logps/rejected": -1.784765601158142,
      "loss": 0.6691,
      "nll_loss": 0.6646484136581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06934814155101776,
      "rewards/margins": 0.10926513373851776,
      "rewards/rejected": -0.17863769829273224,
      "step": 10970
    },
    {
      "epoch": 0.8332068599180452,
      "grad_norm": 3.139006252979104,
      "learning_rate": 7.634644457518242e-07,
      "log_odds_chosen": 1.65625,
      "log_odds_ratio": -0.3799804747104645,
      "logits/chosen": -1.019140601158142,
      "logits/rejected": -0.916796863079071,
      "logps/chosen": -0.620800793170929,
      "logps/rejected": -1.771875023841858,
      "loss": 0.6827,
      "nll_loss": 0.6434570550918579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06208496168255806,
      "rewards/margins": 0.11516113579273224,
      "rewards/rejected": -0.17729492485523224,
      "step": 10980
    },
    {
      "epoch": 0.8339657004097739,
      "grad_norm": 2.5603586631513173,
      "learning_rate": 7.631170216402039e-07,
      "log_odds_chosen": 1.5349609851837158,
      "log_odds_ratio": -0.376708984375,
      "logits/chosen": -1.023046851158142,
      "logits/rejected": -0.8998047113418579,
      "logps/chosen": -0.6375976800918579,
      "logps/rejected": -1.6652343273162842,
      "loss": 0.6772,
      "nll_loss": 0.7020508050918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06375732272863388,
      "rewards/margins": 0.10280761867761612,
      "rewards/rejected": -0.16655273735523224,
      "step": 10990
    },
    {
      "epoch": 0.8347245409015025,
      "grad_norm": 2.6542839406337455,
      "learning_rate": 7.627700713964739e-07,
      "log_odds_chosen": 1.482812523841858,
      "log_odds_ratio": -0.3738769590854645,
      "logits/chosen": -1.0011718273162842,
      "logits/rejected": -0.9248046875,
      "logps/chosen": -0.670703113079071,
      "logps/rejected": -1.708593726158142,
      "loss": 0.6563,
      "nll_loss": 0.6292968988418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06706543266773224,
      "rewards/margins": 0.10386963188648224,
      "rewards/rejected": -0.17099609971046448,
      "step": 11000
    },
    {
      "epoch": 0.8354833813932312,
      "grad_norm": 3.358638335595734,
      "learning_rate": 7.624235939443953e-07,
      "log_odds_chosen": 1.6583983898162842,
      "log_odds_ratio": -0.3785644471645355,
      "logits/chosen": -1.0498046875,
      "logits/rejected": -0.925976574420929,
      "logps/chosen": -0.6590820550918579,
      "logps/rejected": -1.8351562023162842,
      "loss": 0.6896,
      "nll_loss": 0.6463867425918579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06584472954273224,
      "rewards/margins": 0.11749267578125,
      "rewards/rejected": -0.18339844048023224,
      "step": 11010
    },
    {
      "epoch": 0.8362422218849598,
      "grad_norm": 3.0674833361226352,
      "learning_rate": 7.620775882111482e-07,
      "log_odds_chosen": 1.7878906726837158,
      "log_odds_ratio": -0.3297363221645355,
      "logits/chosen": -1.032617211341858,
      "logits/rejected": -0.932812511920929,
      "logps/chosen": -0.5757812261581421,
      "logps/rejected": -1.7667968273162842,
      "loss": 0.6694,
      "nll_loss": 0.5938476324081421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.05754394456744194,
      "rewards/margins": 0.11895751953125,
      "rewards/rejected": -0.17673340439796448,
      "step": 11020
    },
    {
      "epoch": 0.8370010623766884,
      "grad_norm": 3.3462995547157877,
      "learning_rate": 7.617320531273181e-07,
      "log_odds_chosen": 1.6171875,
      "log_odds_ratio": -0.38044434785842896,
      "logits/chosen": -1.0906250476837158,
      "logits/rejected": -0.950976550579071,
      "logps/chosen": -0.654101550579071,
      "logps/rejected": -1.779296875,
      "loss": 0.6516,
      "nll_loss": 0.589062511920929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06541748344898224,
      "rewards/margins": 0.11263427883386612,
      "rewards/rejected": -0.17814941704273224,
      "step": 11030
    },
    {
      "epoch": 0.8377599028684171,
      "grad_norm": 4.448526628127915,
      "learning_rate": 7.613869876268809e-07,
      "log_odds_chosen": 1.7587890625,
      "log_odds_ratio": -0.35795897245407104,
      "logits/chosen": -1.05859375,
      "logits/rejected": -0.9371093511581421,
      "logps/chosen": -0.6131836175918579,
      "logps/rejected": -1.869140625,
      "loss": 0.6674,
      "nll_loss": 0.5858398675918579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06130371242761612,
      "rewards/margins": 0.125518798828125,
      "rewards/rejected": -0.18691405653953552,
      "step": 11040
    },
    {
      "epoch": 0.8385187433601456,
      "grad_norm": 4.567743043824706,
      "learning_rate": 7.610423906471905e-07,
      "log_odds_chosen": 1.559960961341858,
      "log_odds_ratio": -0.42436522245407104,
      "logits/chosen": -0.9740234613418579,
      "logits/rejected": -0.8427734375,
      "logps/chosen": -0.6005859375,
      "logps/rejected": -1.6681640148162842,
      "loss": 0.7086,
      "nll_loss": 0.6507812738418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06002197414636612,
      "rewards/margins": 0.10684509575366974,
      "rewards/rejected": -0.16685791313648224,
      "step": 11050
    },
    {
      "epoch": 0.8392775838518743,
      "grad_norm": 2.822592601831677,
      "learning_rate": 7.606982611289639e-07,
      "log_odds_chosen": 1.840234398841858,
      "log_odds_ratio": -0.33623045682907104,
      "logits/chosen": -1.0146484375,
      "logits/rejected": -0.8970702886581421,
      "logps/chosen": -0.619433581829071,
      "logps/rejected": -1.900781273841858,
      "loss": 0.6764,
      "nll_loss": 0.5692383050918579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06193847581744194,
      "rewards/margins": 0.12790527939796448,
      "rewards/rejected": -0.19001464545726776,
      "step": 11060
    },
    {
      "epoch": 0.8400364243436029,
      "grad_norm": 2.6626504302534837,
      "learning_rate": 7.60354598016268e-07,
      "log_odds_chosen": 1.572656273841858,
      "log_odds_ratio": -0.40727537870407104,
      "logits/chosen": -1.0158202648162842,
      "logits/rejected": -0.8853515386581421,
      "logps/chosen": -0.623339831829071,
      "logps/rejected": -1.732031226158142,
      "loss": 0.6448,
      "nll_loss": 0.607714831829071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06232910230755806,
      "rewards/margins": 0.11085204780101776,
      "rewards/rejected": -0.17304687201976776,
      "step": 11070
    },
    {
      "epoch": 0.8407952648353316,
      "grad_norm": 2.7847857935042515,
      "learning_rate": 7.600114002565063e-07,
      "log_odds_chosen": 1.588281273841858,
      "log_odds_ratio": -0.32817381620407104,
      "logits/chosen": -1.0519530773162842,
      "logits/rejected": -0.9234374761581421,
      "logps/chosen": -0.624804675579071,
      "logps/rejected": -1.708593726158142,
      "loss": 0.6731,
      "nll_loss": 0.666210949420929,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06253661960363388,
      "rewards/margins": 0.10844726860523224,
      "rewards/rejected": -0.17099609971046448,
      "step": 11080
    },
    {
      "epoch": 0.8415541053270602,
      "grad_norm": 2.972171384247822,
      "learning_rate": 7.596686668004049e-07,
      "log_odds_chosen": 1.5183594226837158,
      "log_odds_ratio": -0.39384764432907104,
      "logits/chosen": -1.0007812976837158,
      "logits/rejected": -0.8988281488418579,
      "logps/chosen": -0.6517578363418579,
      "logps/rejected": -1.721093773841858,
      "loss": 0.6336,
      "nll_loss": 0.574511706829071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06517334282398224,
      "rewards/margins": 0.10710449516773224,
      "rewards/rejected": -0.17216797173023224,
      "step": 11090
    },
    {
      "epoch": 0.8423129458187889,
      "grad_norm": 3.5551923267918246,
      "learning_rate": 7.593263966019991e-07,
      "log_odds_chosen": 1.913476586341858,
      "log_odds_ratio": -0.30524903535842896,
      "logits/chosen": -1.0187499523162842,
      "logits/rejected": -0.884960949420929,
      "logps/chosen": -0.6099609136581421,
      "logps/rejected": -1.9474608898162842,
      "loss": 0.6721,
      "nll_loss": 0.6063476800918579,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06094970554113388,
      "rewards/margins": 0.13374634087085724,
      "rewards/rejected": -0.19467774033546448,
      "step": 11100
    },
    {
      "epoch": 0.8430717863105175,
      "grad_norm": 2.5484613829611242,
      "learning_rate": 7.589845886186201e-07,
      "log_odds_chosen": 1.728906273841858,
      "log_odds_ratio": -0.3409179747104645,
      "logits/chosen": -1.031640648841858,
      "logits/rejected": -0.920117199420929,
      "logps/chosen": -0.636523425579071,
      "logps/rejected": -1.8484375476837158,
      "loss": 0.6668,
      "nll_loss": 0.6659179925918579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06373290717601776,
      "rewards/margins": 0.12114258110523224,
      "rewards/rejected": -0.18488769233226776,
      "step": 11110
    },
    {
      "epoch": 0.8438306268022462,
      "grad_norm": 3.570504120600914,
      "learning_rate": 7.586432418108816e-07,
      "log_odds_chosen": 1.882421851158142,
      "log_odds_ratio": -0.34038084745407104,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.913867175579071,
      "logps/chosen": -0.622265636920929,
      "logps/rejected": -1.975000023841858,
      "loss": 0.6574,
      "nll_loss": 0.556835949420929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06223144382238388,
      "rewards/margins": 0.13518066704273224,
      "rewards/rejected": -0.19731445610523224,
      "step": 11120
    },
    {
      "epoch": 0.8445894672939748,
      "grad_norm": 2.932644760202933,
      "learning_rate": 7.583023551426664e-07,
      "log_odds_chosen": 1.3997070789337158,
      "log_odds_ratio": -0.41162109375,
      "logits/chosen": -1.015234351158142,
      "logits/rejected": -0.8851562738418579,
      "logps/chosen": -0.6962890625,
      "logps/rejected": -1.682226538658142,
      "loss": 0.6546,
      "nll_loss": 0.6351562738418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06962890923023224,
      "rewards/margins": 0.09854431450366974,
      "rewards/rejected": -0.16813965141773224,
      "step": 11130
    },
    {
      "epoch": 0.8453483077857035,
      "grad_norm": 3.19426016686964,
      "learning_rate": 7.579619275811138e-07,
      "log_odds_chosen": 1.570898413658142,
      "log_odds_ratio": -0.367431640625,
      "logits/chosen": -1.0216796398162842,
      "logits/rejected": -0.9007812738418579,
      "logps/chosen": -0.613476574420929,
      "logps/rejected": -1.7156250476837158,
      "loss": 0.6857,
      "nll_loss": 0.679003894329071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06132812425494194,
      "rewards/margins": 0.11018066108226776,
      "rewards/rejected": -0.1715087890625,
      "step": 11140
    },
    {
      "epoch": 0.846107148277432,
      "grad_norm": 2.8475060324923764,
      "learning_rate": 7.576219580966055e-07,
      "log_odds_chosen": 1.6173827648162842,
      "log_odds_ratio": -0.36962890625,
      "logits/chosen": -0.9634765386581421,
      "logits/rejected": -0.868945300579071,
      "logps/chosen": -0.638867199420929,
      "logps/rejected": -1.775976538658142,
      "loss": 0.6497,
      "nll_loss": 0.679882824420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06387939304113388,
      "rewards/margins": 0.11359252780675888,
      "rewards/rejected": -0.17751464247703552,
      "step": 11150
    },
    {
      "epoch": 0.8468659887691607,
      "grad_norm": 2.9997241952910634,
      "learning_rate": 7.57282445662753e-07,
      "log_odds_chosen": 1.4814453125,
      "log_odds_ratio": -0.38701170682907104,
      "logits/chosen": -1.029882788658142,
      "logits/rejected": -0.917773425579071,
      "logps/chosen": -0.6236327886581421,
      "logps/rejected": -1.5986328125,
      "loss": 0.6717,
      "nll_loss": 0.6053711175918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06242675706744194,
      "rewards/margins": 0.09744872897863388,
      "rewards/rejected": -0.15974120795726776,
      "step": 11160
    },
    {
      "epoch": 0.8476248292608893,
      "grad_norm": 2.609581574755947,
      "learning_rate": 7.569433892563852e-07,
      "log_odds_chosen": 1.816015601158142,
      "log_odds_ratio": -0.3173828125,
      "logits/chosen": -1.0203125476837158,
      "logits/rejected": -0.8775390386581421,
      "logps/chosen": -0.641796886920929,
      "logps/rejected": -1.8800780773162842,
      "loss": 0.6457,
      "nll_loss": 0.607421875,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.06417236477136612,
      "rewards/margins": 0.12376709282398224,
      "rewards/rejected": -0.18808594346046448,
      "step": 11170
    },
    {
      "epoch": 0.848383669752618,
      "grad_norm": 3.1627600325858434,
      "learning_rate": 7.566047878575343e-07,
      "log_odds_chosen": 1.47216796875,
      "log_odds_ratio": -0.38037109375,
      "logits/chosen": -1.016992211341858,
      "logits/rejected": -0.9154297113418579,
      "logps/chosen": -0.6424804925918579,
      "logps/rejected": -1.6298828125,
      "loss": 0.6492,
      "nll_loss": 0.63525390625,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06427001953125,
      "rewards/margins": 0.09873352199792862,
      "rewards/rejected": -0.16301269829273224,
      "step": 11180
    },
    {
      "epoch": 0.8491425102443466,
      "grad_norm": 3.113411090416283,
      "learning_rate": 7.562666404494236e-07,
      "log_odds_chosen": 1.706445336341858,
      "log_odds_ratio": -0.3707031309604645,
      "logits/chosen": -0.9925781488418579,
      "logits/rejected": -0.938671886920929,
      "logps/chosen": -0.6504882574081421,
      "logps/rejected": -1.835546851158142,
      "loss": 0.6527,
      "nll_loss": 0.616503894329071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06507568061351776,
      "rewards/margins": 0.11849365383386612,
      "rewards/rejected": -0.18361815810203552,
      "step": 11190
    },
    {
      "epoch": 0.8499013507360753,
      "grad_norm": 3.9619968784056345,
      "learning_rate": 7.559289460184543e-07,
      "log_odds_chosen": 1.7843749523162842,
      "log_odds_ratio": -0.36176759004592896,
      "logits/chosen": -1.020898461341858,
      "logits/rejected": -0.933789074420929,
      "logps/chosen": -0.58544921875,
      "logps/rejected": -1.8312499523162842,
      "loss": 0.6706,
      "nll_loss": 0.584179699420929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05854492262005806,
      "rewards/margins": 0.12456054985523224,
      "rewards/rejected": -0.18315429985523224,
      "step": 11200
    },
    {
      "epoch": 0.8506601912278039,
      "grad_norm": 2.3948748178797556,
      "learning_rate": 7.555917035541937e-07,
      "log_odds_chosen": 1.7861328125,
      "log_odds_ratio": -0.32526856660842896,
      "logits/chosen": -1.0205078125,
      "logits/rejected": -0.861132800579071,
      "logps/chosen": -0.5669921636581421,
      "logps/rejected": -1.777734398841858,
      "loss": 0.6416,
      "nll_loss": 0.577441394329071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05668945237994194,
      "rewards/margins": 0.1209716796875,
      "rewards/rejected": -0.17770996689796448,
      "step": 11210
    },
    {
      "epoch": 0.8514190317195326,
      "grad_norm": 3.7918336391494503,
      "learning_rate": 7.552549120493609e-07,
      "log_odds_chosen": 1.422949194908142,
      "log_odds_ratio": -0.40180665254592896,
      "logits/chosen": -0.973437488079071,
      "logits/rejected": -0.889843761920929,
      "logps/chosen": -0.595019519329071,
      "logps/rejected": -1.5363280773162842,
      "loss": 0.6602,
      "nll_loss": 0.57275390625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.05950927734375,
      "rewards/margins": 0.09407653659582138,
      "rewards/rejected": -0.15361328423023224,
      "step": 11220
    },
    {
      "epoch": 0.8521778722112612,
      "grad_norm": 4.056634229516305,
      "learning_rate": 7.549185704998158e-07,
      "log_odds_chosen": 1.447412133216858,
      "log_odds_ratio": -0.4168457090854645,
      "logits/chosen": -1.0027344226837158,
      "logits/rejected": -0.8740234375,
      "logps/chosen": -0.6929687261581421,
      "logps/rejected": -1.7121093273162842,
      "loss": 0.6431,
      "nll_loss": 0.61474609375,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06927490234375,
      "rewards/margins": 0.10191955417394638,
      "rewards/rejected": -0.17124024033546448,
      "step": 11230
    },
    {
      "epoch": 0.8529367127029899,
      "grad_norm": 3.252267854586561,
      "learning_rate": 7.545826779045449e-07,
      "log_odds_chosen": 1.7365233898162842,
      "log_odds_ratio": -0.3375488221645355,
      "logits/chosen": -1.065820336341858,
      "logits/rejected": -0.9365234375,
      "logps/chosen": -0.59423828125,
      "logps/rejected": -1.785546898841858,
      "loss": 0.6837,
      "nll_loss": 0.597851574420929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05941162258386612,
      "rewards/margins": 0.11906738579273224,
      "rewards/rejected": -0.17856445908546448,
      "step": 11240
    },
    {
      "epoch": 0.8536955531947185,
      "grad_norm": 3.1942720430042186,
      "learning_rate": 7.542472332656506e-07,
      "log_odds_chosen": 1.4568359851837158,
      "log_odds_ratio": -0.39448243379592896,
      "logits/chosen": -0.9814453125,
      "logits/rejected": -0.894335925579071,
      "logps/chosen": -0.616992175579071,
      "logps/rejected": -1.608984351158142,
      "loss": 0.6611,
      "nll_loss": 0.6343749761581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06163330003619194,
      "rewards/margins": 0.09923706203699112,
      "rewards/rejected": -0.16083984076976776,
      "step": 11250
    },
    {
      "epoch": 0.8544543936864472,
      "grad_norm": 3.3682777463353064,
      "learning_rate": 7.539122355883373e-07,
      "log_odds_chosen": 1.921289086341858,
      "log_odds_ratio": -0.338134765625,
      "logits/chosen": -1.027929663658142,
      "logits/rejected": -0.913281261920929,
      "logps/chosen": -0.643261730670929,
      "logps/rejected": -2.0367188453674316,
      "loss": 0.6872,
      "nll_loss": 0.67333984375,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06436767429113388,
      "rewards/margins": 0.13941650092601776,
      "rewards/rejected": -0.20383301377296448,
      "step": 11260
    },
    {
      "epoch": 0.8552132341781757,
      "grad_norm": 3.1115067039322444,
      "learning_rate": 7.535776838808995e-07,
      "log_odds_chosen": 1.650292992591858,
      "log_odds_ratio": -0.3653320372104645,
      "logits/chosen": -1.0085937976837158,
      "logits/rejected": -0.8958984613418579,
      "logps/chosen": -0.6229492425918579,
      "logps/rejected": -1.7800781726837158,
      "loss": 0.6705,
      "nll_loss": 0.6768554449081421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06230468675494194,
      "rewards/margins": 0.11549071967601776,
      "rewards/rejected": -0.1778564453125,
      "step": 11270
    },
    {
      "epoch": 0.8559720746699044,
      "grad_norm": 3.076210649134253,
      "learning_rate": 7.532435771547094e-07,
      "log_odds_chosen": 1.6580078601837158,
      "log_odds_ratio": -0.389892578125,
      "logits/chosen": -1.0439453125,
      "logits/rejected": -0.9554687738418579,
      "logps/chosen": -0.61083984375,
      "logps/rejected": -1.7781250476837158,
      "loss": 0.6555,
      "nll_loss": 0.617480456829071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06108398362994194,
      "rewards/margins": 0.11690673977136612,
      "rewards/rejected": -0.177978515625,
      "step": 11280
    },
    {
      "epoch": 0.856730915161633,
      "grad_norm": 3.2439195743043774,
      "learning_rate": 7.52909914424205e-07,
      "log_odds_chosen": 1.72900390625,
      "log_odds_ratio": -0.3655761778354645,
      "logits/chosen": -1.051367163658142,
      "logits/rejected": -0.921679675579071,
      "logps/chosen": -0.6348632574081421,
      "logps/rejected": -1.8396484851837158,
      "loss": 0.6707,
      "nll_loss": 0.609375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06342773139476776,
      "rewards/margins": 0.12058105319738388,
      "rewards/rejected": -0.18400879204273224,
      "step": 11290
    },
    {
      "epoch": 0.8574897556533617,
      "grad_norm": 3.0636769274752718,
      "learning_rate": 7.525766947068777e-07,
      "log_odds_chosen": 1.714453101158142,
      "log_odds_ratio": -0.36640626192092896,
      "logits/chosen": -1.051367163658142,
      "logits/rejected": -0.927734375,
      "logps/chosen": -0.6298828125,
      "logps/rejected": -1.873437523841858,
      "loss": 0.6682,
      "nll_loss": 0.6678711175918579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06296386569738388,
      "rewards/margins": 0.12434081733226776,
      "rewards/rejected": -0.18730469048023224,
      "step": 11300
    },
    {
      "epoch": 0.8582485961450903,
      "grad_norm": 3.3063845948164303,
      "learning_rate": 7.522439170232598e-07,
      "log_odds_chosen": 1.5341796875,
      "log_odds_ratio": -0.4236816465854645,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.8802734613418579,
      "logps/chosen": -0.669238269329071,
      "logps/rejected": -1.716796875,
      "loss": 0.6616,
      "nll_loss": 0.701953113079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06691894680261612,
      "rewards/margins": 0.10477294772863388,
      "rewards/rejected": -0.17165526747703552,
      "step": 11310
    },
    {
      "epoch": 0.8590074366368189,
      "grad_norm": 3.0819105492816607,
      "learning_rate": 7.519115803969124e-07,
      "log_odds_chosen": 1.594140648841858,
      "log_odds_ratio": -0.36821287870407104,
      "logits/chosen": -1.0017578601837158,
      "logits/rejected": -0.875781238079071,
      "logps/chosen": -0.57958984375,
      "logps/rejected": -1.641015648841858,
      "loss": 0.6637,
      "nll_loss": 0.6087890863418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.05793457105755806,
      "rewards/margins": 0.10629882663488388,
      "rewards/rejected": -0.16420897841453552,
      "step": 11320
    },
    {
      "epoch": 0.8597662771285476,
      "grad_norm": 3.2464121956934426,
      "learning_rate": 7.515796838544139e-07,
      "log_odds_chosen": 1.694921851158142,
      "log_odds_ratio": -0.35527342557907104,
      "logits/chosen": -1.027929663658142,
      "logits/rejected": -0.9267578125,
      "logps/chosen": -0.624316394329071,
      "logps/rejected": -1.76171875,
      "loss": 0.6857,
      "nll_loss": 0.646679699420929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06236572191119194,
      "rewards/margins": 0.11375579982995987,
      "rewards/rejected": -0.17631836235523224,
      "step": 11330
    },
    {
      "epoch": 0.8605251176202762,
      "grad_norm": 3.457457605797307,
      "learning_rate": 7.51248226425348e-07,
      "log_odds_chosen": 1.40673828125,
      "log_odds_ratio": -0.4073242247104645,
      "logits/chosen": -1.0294921398162842,
      "logits/rejected": -0.9400390386581421,
      "logps/chosen": -0.6036132574081421,
      "logps/rejected": -1.514062523841858,
      "loss": 0.6736,
      "nll_loss": 0.62109375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06036376953125,
      "rewards/margins": 0.09108276665210724,
      "rewards/rejected": -0.15151366591453552,
      "step": 11340
    },
    {
      "epoch": 0.8612839581120049,
      "grad_norm": 3.630449367713292,
      "learning_rate": 7.509172071422913e-07,
      "log_odds_chosen": 1.5783202648162842,
      "log_odds_ratio": -0.3897460997104645,
      "logits/chosen": -1.0207030773162842,
      "logits/rejected": -0.8759765625,
      "logps/chosen": -0.652539074420929,
      "logps/rejected": -1.787500023841858,
      "loss": 0.6368,
      "nll_loss": 0.61865234375,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06524658203125,
      "rewards/margins": 0.11354980617761612,
      "rewards/rejected": -0.17873534560203552,
      "step": 11350
    },
    {
      "epoch": 0.8620427986037335,
      "grad_norm": 3.2147911869397015,
      "learning_rate": 7.505866250408015e-07,
      "log_odds_chosen": 1.8308594226837158,
      "log_odds_ratio": -0.334716796875,
      "logits/chosen": -1.0652344226837158,
      "logits/rejected": -0.8990234136581421,
      "logps/chosen": -0.621289074420929,
      "logps/rejected": -1.91796875,
      "loss": 0.6529,
      "nll_loss": 0.650390625,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06208496168255806,
      "rewards/margins": 0.12971191108226776,
      "rewards/rejected": -0.1917724609375,
      "step": 11360
    },
    {
      "epoch": 0.8628016390954621,
      "grad_norm": 3.3463974347956964,
      "learning_rate": 7.50256479159406e-07,
      "log_odds_chosen": 1.707421898841858,
      "log_odds_ratio": -0.3255859315395355,
      "logits/chosen": -1.047265648841858,
      "logits/rejected": -0.913867175579071,
      "logps/chosen": -0.580761730670929,
      "logps/rejected": -1.7109375,
      "loss": 0.6445,
      "nll_loss": 0.5704101324081421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05810546875,
      "rewards/margins": 0.11301269382238388,
      "rewards/rejected": -0.17106933891773224,
      "step": 11370
    },
    {
      "epoch": 0.8635604795871907,
      "grad_norm": 3.0223671369937004,
      "learning_rate": 7.499267685395902e-07,
      "log_odds_chosen": 1.631445288658142,
      "log_odds_ratio": -0.361572265625,
      "logits/chosen": -1.0087890625,
      "logits/rejected": -0.9046875238418579,
      "logps/chosen": -0.6050781011581421,
      "logps/rejected": -1.7041015625,
      "loss": 0.6515,
      "nll_loss": 0.652050793170929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06046142429113388,
      "rewards/margins": 0.10994873195886612,
      "rewards/rejected": -0.17050781846046448,
      "step": 11380
    },
    {
      "epoch": 0.8643193200789194,
      "grad_norm": 3.294692250380363,
      "learning_rate": 7.495974922257845e-07,
      "log_odds_chosen": 1.4822266101837158,
      "log_odds_ratio": -0.40595704317092896,
      "logits/chosen": -1.0158202648162842,
      "logits/rejected": -0.900195300579071,
      "logps/chosen": -0.654003918170929,
      "logps/rejected": -1.685546875,
      "loss": 0.6598,
      "nll_loss": 0.6771484613418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06540527194738388,
      "rewards/margins": 0.10305175930261612,
      "rewards/rejected": -0.1683349609375,
      "step": 11390
    },
    {
      "epoch": 0.865078160570648,
      "grad_norm": 2.928469478542356,
      "learning_rate": 7.492686492653552e-07,
      "log_odds_chosen": 1.6989257335662842,
      "log_odds_ratio": -0.3568115234375,
      "logits/chosen": -1.014257788658142,
      "logits/rejected": -0.8919922113418579,
      "logps/chosen": -0.5938476324081421,
      "logps/rejected": -1.758203148841858,
      "loss": 0.658,
      "nll_loss": 0.5743163824081421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05936889722943306,
      "rewards/margins": 0.11652831733226776,
      "rewards/rejected": -0.176025390625,
      "step": 11400
    },
    {
      "epoch": 0.8658370010623767,
      "grad_norm": 3.197786038713574,
      "learning_rate": 7.489402387085902e-07,
      "log_odds_chosen": 1.751953125,
      "log_odds_ratio": -0.3661865293979645,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.8500000238418579,
      "logps/chosen": -0.6366211175918579,
      "logps/rejected": -1.8406250476837158,
      "loss": 0.6551,
      "nll_loss": 0.6631835699081421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06367187201976776,
      "rewards/margins": 0.120361328125,
      "rewards/rejected": -0.18403320014476776,
      "step": 11410
    },
    {
      "epoch": 0.8665958415541053,
      "grad_norm": 3.3149692624391824,
      "learning_rate": 7.486122596086891e-07,
      "log_odds_chosen": 1.7160155773162842,
      "log_odds_ratio": -0.3590331971645355,
      "logits/chosen": -1.0236327648162842,
      "logits/rejected": -0.8988281488418579,
      "logps/chosen": -0.623730480670929,
      "logps/rejected": -1.844140648841858,
      "loss": 0.6682,
      "nll_loss": 0.6151367425918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06241454929113388,
      "rewards/margins": 0.1221923828125,
      "rewards/rejected": -0.18442383408546448,
      "step": 11420
    },
    {
      "epoch": 0.867354682045834,
      "grad_norm": 3.1822616489445794,
      "learning_rate": 7.482847110217516e-07,
      "log_odds_chosen": 1.6416015625,
      "log_odds_ratio": -0.351318359375,
      "logits/chosen": -1.0275390148162842,
      "logits/rejected": -0.8765624761581421,
      "logps/chosen": -0.647265613079071,
      "logps/rejected": -1.8156249523162842,
      "loss": 0.678,
      "nll_loss": 0.615429699420929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.064697265625,
      "rewards/margins": 0.11687012016773224,
      "rewards/rejected": -0.18159179389476776,
      "step": 11430
    },
    {
      "epoch": 0.8681135225375626,
      "grad_norm": 2.599554196276931,
      "learning_rate": 7.479575920067657e-07,
      "log_odds_chosen": 1.6515624523162842,
      "log_odds_ratio": -0.370849609375,
      "logits/chosen": -0.990234375,
      "logits/rejected": -0.869335949420929,
      "logps/chosen": -0.60595703125,
      "logps/rejected": -1.71484375,
      "loss": 0.6737,
      "nll_loss": 0.653515636920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06064452975988388,
      "rewards/margins": 0.11082763969898224,
      "rewards/rejected": -0.17143554985523224,
      "step": 11440
    },
    {
      "epoch": 0.8688723630292913,
      "grad_norm": 3.4150041718147586,
      "learning_rate": 7.476309016255964e-07,
      "log_odds_chosen": 1.4673340320587158,
      "log_odds_ratio": -0.39653319120407104,
      "logits/chosen": -0.9990234375,
      "logits/rejected": -0.915234386920929,
      "logps/chosen": -0.636523425579071,
      "logps/rejected": -1.6486327648162842,
      "loss": 0.646,
      "nll_loss": 0.671093761920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06361083686351776,
      "rewards/margins": 0.10118408501148224,
      "rewards/rejected": -0.16486816108226776,
      "step": 11450
    },
    {
      "epoch": 0.8696312035210199,
      "grad_norm": 2.6993003639511755,
      "learning_rate": 7.473046389429744e-07,
      "log_odds_chosen": 1.7160155773162842,
      "log_odds_ratio": -0.33442384004592896,
      "logits/chosen": -1.027929663658142,
      "logits/rejected": -0.8794921636581421,
      "logps/chosen": -0.6201171875,
      "logps/rejected": -1.7820312976837158,
      "loss": 0.6689,
      "nll_loss": 0.6874023675918579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06203613430261612,
      "rewards/margins": 0.11628417670726776,
      "rewards/rejected": -0.17839355766773224,
      "step": 11460
    },
    {
      "epoch": 0.8703900440127486,
      "grad_norm": 2.6995439056656068,
      "learning_rate": 7.469788030264852e-07,
      "log_odds_chosen": 1.3898437023162842,
      "log_odds_ratio": -0.41865235567092896,
      "logits/chosen": -1.0185546875,
      "logits/rejected": -0.9136718511581421,
      "logps/chosen": -0.6224609613418579,
      "logps/rejected": -1.546484351158142,
      "loss": 0.6681,
      "nll_loss": 0.6353515386581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06223144382238388,
      "rewards/margins": 0.092529296875,
      "rewards/rejected": -0.15473632514476776,
      "step": 11470
    },
    {
      "epoch": 0.8711488845044771,
      "grad_norm": 3.2950188718887645,
      "learning_rate": 7.466533929465574e-07,
      "log_odds_chosen": 1.693750023841858,
      "log_odds_ratio": -0.3218750059604645,
      "logits/chosen": -1.029882788658142,
      "logits/rejected": -0.876953125,
      "logps/chosen": -0.600878894329071,
      "logps/rejected": -1.737890601158142,
      "loss": 0.6295,
      "nll_loss": 0.591796875,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06009521335363388,
      "rewards/margins": 0.11368408054113388,
      "rewards/rejected": -0.17360839247703552,
      "step": 11480
    },
    {
      "epoch": 0.8719077249962058,
      "grad_norm": 2.9967159336769553,
      "learning_rate": 7.463284077764519e-07,
      "log_odds_chosen": 1.375390648841858,
      "log_odds_ratio": -0.423828125,
      "logits/chosen": -1.0427734851837158,
      "logits/rejected": -0.9222656488418579,
      "logps/chosen": -0.6172851324081421,
      "logps/rejected": -1.52734375,
      "loss": 0.6684,
      "nll_loss": 0.645800769329071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06174316257238388,
      "rewards/margins": 0.09095458686351776,
      "rewards/rejected": -0.15273436903953552,
      "step": 11490
    },
    {
      "epoch": 0.8726665654879344,
      "grad_norm": 2.9233262804458584,
      "learning_rate": 7.460038465922511e-07,
      "log_odds_chosen": 1.725195288658142,
      "log_odds_ratio": -0.376220703125,
      "logits/chosen": -1.0041015148162842,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.5884765386581421,
      "logps/rejected": -1.7414062023162842,
      "loss": 0.6456,
      "nll_loss": 0.60595703125,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.05881347507238388,
      "rewards/margins": 0.1153564453125,
      "rewards/rejected": -0.17421874403953552,
      "step": 11500
    },
    {
      "epoch": 0.8734254059796631,
      "grad_norm": 3.7653169427507773,
      "learning_rate": 7.456797084728466e-07,
      "log_odds_chosen": 1.6687500476837158,
      "log_odds_ratio": -0.3714843690395355,
      "logits/chosen": -1.00390625,
      "logits/rejected": -0.9134765863418579,
      "logps/chosen": -0.6255859136581421,
      "logps/rejected": -1.7316405773162842,
      "loss": 0.6523,
      "nll_loss": 0.6363281011581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06253661960363388,
      "rewards/margins": 0.11057128757238388,
      "rewards/rejected": -0.17331543564796448,
      "step": 11510
    },
    {
      "epoch": 0.8741842464713917,
      "grad_norm": 3.2407013418993915,
      "learning_rate": 7.453559924999299e-07,
      "log_odds_chosen": 1.8445312976837158,
      "log_odds_ratio": -0.31547850370407104,
      "logits/chosen": -1.023046851158142,
      "logits/rejected": -0.9058593511581421,
      "logps/chosen": -0.6288086175918579,
      "logps/rejected": -1.9363281726837158,
      "loss": 0.6711,
      "nll_loss": 0.6357421875,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06287841498851776,
      "rewards/margins": 0.1307373046875,
      "rewards/rejected": -0.19365234673023224,
      "step": 11520
    },
    {
      "epoch": 0.8749430869631204,
      "grad_norm": 3.096734328810894,
      "learning_rate": 7.450326977579804e-07,
      "log_odds_chosen": 1.484765648841858,
      "log_odds_ratio": -0.3749023377895355,
      "logits/chosen": -1.0068359375,
      "logits/rejected": -0.87109375,
      "logps/chosen": -0.6025390625,
      "logps/rejected": -1.548437476158142,
      "loss": 0.6748,
      "nll_loss": 0.6392577886581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.060302734375,
      "rewards/margins": 0.09442138671875,
      "rewards/rejected": -0.15476074814796448,
      "step": 11530
    },
    {
      "epoch": 0.875701927454849,
      "grad_norm": 3.124124571388173,
      "learning_rate": 7.447098233342549e-07,
      "log_odds_chosen": 1.496679663658142,
      "log_odds_ratio": -0.37626951932907104,
      "logits/chosen": -0.9496093988418579,
      "logits/rejected": -0.9048827886581421,
      "logps/chosen": -0.576367199420929,
      "logps/rejected": -1.535546898841858,
      "loss": 0.6601,
      "nll_loss": 0.5765625238418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05762939527630806,
      "rewards/margins": 0.09595946967601776,
      "rewards/rejected": -0.15373535454273224,
      "step": 11540
    },
    {
      "epoch": 0.8764607679465777,
      "grad_norm": 3.7916592178987405,
      "learning_rate": 7.443873683187767e-07,
      "log_odds_chosen": 1.498046875,
      "log_odds_ratio": -0.4017578065395355,
      "logits/chosen": -0.949999988079071,
      "logits/rejected": -0.908398449420929,
      "logps/chosen": -0.6958984136581421,
      "logps/rejected": -1.7351562976837158,
      "loss": 0.6821,
      "nll_loss": 0.7275390625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.069580078125,
      "rewards/margins": 0.10383300483226776,
      "rewards/rejected": -0.1734619140625,
      "step": 11550
    },
    {
      "epoch": 0.8772196084383063,
      "grad_norm": 2.964465935207747,
      "learning_rate": 7.440653318043245e-07,
      "log_odds_chosen": 1.6256835460662842,
      "log_odds_ratio": -0.369873046875,
      "logits/chosen": -1.0138671398162842,
      "logits/rejected": -0.9009765386581421,
      "logps/chosen": -0.5791015625,
      "logps/rejected": -1.6662108898162842,
      "loss": 0.6418,
      "nll_loss": 0.575488269329071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05787963792681694,
      "rewards/margins": 0.10871734470129013,
      "rewards/rejected": -0.16657714545726776,
      "step": 11560
    },
    {
      "epoch": 0.877978448930035,
      "grad_norm": 2.7545977670397264,
      "learning_rate": 7.437437128864224e-07,
      "log_odds_chosen": 1.538476586341858,
      "log_odds_ratio": -0.39042967557907104,
      "logits/chosen": -1.0154297351837158,
      "logits/rejected": -0.9341796636581421,
      "logps/chosen": -0.60009765625,
      "logps/rejected": -1.625,
      "loss": 0.6814,
      "nll_loss": 0.60498046875,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06004638597369194,
      "rewards/margins": 0.10250244289636612,
      "rewards/rejected": -0.16262206435203552,
      "step": 11570
    },
    {
      "epoch": 0.8787372894217635,
      "grad_norm": 3.2627795713662855,
      "learning_rate": 7.434225106633287e-07,
      "log_odds_chosen": 1.373046875,
      "log_odds_ratio": -0.4639648497104645,
      "logits/chosen": -1.0595703125,
      "logits/rejected": -0.9468749761581421,
      "logps/chosen": -0.665234386920929,
      "logps/rejected": -1.670312523841858,
      "loss": 0.6746,
      "nll_loss": 0.6644531488418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06650390475988388,
      "rewards/margins": 0.10053710639476776,
      "rewards/rejected": -0.16701659560203552,
      "step": 11580
    },
    {
      "epoch": 0.8794961299134921,
      "grad_norm": 3.379080932960757,
      "learning_rate": 7.431017242360253e-07,
      "log_odds_chosen": 1.55908203125,
      "log_odds_ratio": -0.3919433653354645,
      "logits/chosen": -1.0060546398162842,
      "logits/rejected": -0.9166015386581421,
      "logps/chosen": -0.5948241949081421,
      "logps/rejected": -1.6853516101837158,
      "loss": 0.6514,
      "nll_loss": 0.5703125,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0594482421875,
      "rewards/margins": 0.10916747897863388,
      "rewards/rejected": -0.16865234076976776,
      "step": 11590
    },
    {
      "epoch": 0.8802549704052208,
      "grad_norm": 3.286178817462641,
      "learning_rate": 7.427813527082074e-07,
      "log_odds_chosen": 1.6437499523162842,
      "log_odds_ratio": -0.34272462129592896,
      "logits/chosen": -1.068359375,
      "logits/rejected": -0.9808593988418579,
      "logps/chosen": -0.5990234613418579,
      "logps/rejected": -1.7197265625,
      "loss": 0.6402,
      "nll_loss": 0.5623534917831421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05991210788488388,
      "rewards/margins": 0.1121826171875,
      "rewards/rejected": -0.17202147841453552,
      "step": 11600
    },
    {
      "epoch": 0.8810138108969494,
      "grad_norm": 3.3546226619536275,
      "learning_rate": 7.424613951862727e-07,
      "log_odds_chosen": 1.45849609375,
      "log_odds_ratio": -0.4569335877895355,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.875781238079071,
      "logps/chosen": -0.648144543170929,
      "logps/rejected": -1.6591796875,
      "loss": 0.6698,
      "nll_loss": 0.673632800579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0648193359375,
      "rewards/margins": 0.10117187350988388,
      "rewards/rejected": -0.16582031548023224,
      "step": 11610
    },
    {
      "epoch": 0.8817726513886781,
      "grad_norm": 3.9481495585188973,
      "learning_rate": 7.42141850779311e-07,
      "log_odds_chosen": 1.7605469226837158,
      "log_odds_ratio": -0.36015623807907104,
      "logits/chosen": -1.053320288658142,
      "logits/rejected": -0.926953136920929,
      "logps/chosen": -0.644824206829071,
      "logps/rejected": -1.8937499523162842,
      "loss": 0.6743,
      "nll_loss": 0.660839855670929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06448974460363388,
      "rewards/margins": 0.12491454929113388,
      "rewards/rejected": -0.18942871689796448,
      "step": 11620
    },
    {
      "epoch": 0.8825314918804067,
      "grad_norm": 3.559247853683837,
      "learning_rate": 7.418227185990941e-07,
      "log_odds_chosen": 1.796289086341858,
      "log_odds_ratio": -0.34809571504592896,
      "logits/chosen": -1.0232422351837158,
      "logits/rejected": -0.901562511920929,
      "logps/chosen": -0.585156261920929,
      "logps/rejected": -1.8312499523162842,
      "loss": 0.6733,
      "nll_loss": 0.6512695550918579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05849609524011612,
      "rewards/margins": 0.12471923977136612,
      "rewards/rejected": -0.1832275390625,
      "step": 11630
    },
    {
      "epoch": 0.8832903323721354,
      "grad_norm": 3.520156742524677,
      "learning_rate": 7.415039977600647e-07,
      "log_odds_chosen": 1.7458984851837158,
      "log_odds_ratio": -0.3470703065395355,
      "logits/chosen": -1.058007836341858,
      "logits/rejected": -0.9351562261581421,
      "logps/chosen": -0.5816406011581421,
      "logps/rejected": -1.7449219226837158,
      "loss": 0.6511,
      "nll_loss": 0.564453125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05814208835363388,
      "rewards/margins": 0.1163330078125,
      "rewards/rejected": -0.17441406846046448,
      "step": 11640
    },
    {
      "epoch": 0.884049172863864,
      "grad_norm": 3.410183072627819,
      "learning_rate": 7.411856873793271e-07,
      "log_odds_chosen": 1.668554663658142,
      "log_odds_ratio": -0.35161131620407104,
      "logits/chosen": -1.048828125,
      "logits/rejected": -0.947460949420929,
      "logps/chosen": -0.548632800579071,
      "logps/rejected": -1.6417968273162842,
      "loss": 0.6368,
      "nll_loss": 0.615234375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.054931640625,
      "rewards/margins": 0.10922851413488388,
      "rewards/rejected": -0.1640625,
      "step": 11650
    },
    {
      "epoch": 0.8848080133555927,
      "grad_norm": 2.872271722998988,
      "learning_rate": 7.408677865766361e-07,
      "log_odds_chosen": 1.628320336341858,
      "log_odds_ratio": -0.37993162870407104,
      "logits/chosen": -0.977343738079071,
      "logits/rejected": -0.8939453363418579,
      "logps/chosen": -0.5655273199081421,
      "logps/rejected": -1.634374976158142,
      "loss": 0.6482,
      "nll_loss": 0.6104491949081421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.05656738206744194,
      "rewards/margins": 0.10699462890625,
      "rewards/rejected": -0.16357421875,
      "step": 11660
    },
    {
      "epoch": 0.8855668538473213,
      "grad_norm": 2.8853969209815533,
      "learning_rate": 7.405502944743868e-07,
      "log_odds_chosen": 1.704687476158142,
      "log_odds_ratio": -0.3372558653354645,
      "logits/chosen": -1.013671875,
      "logits/rejected": -0.896289050579071,
      "logps/chosen": -0.6268554925918579,
      "logps/rejected": -1.804101586341858,
      "loss": 0.6388,
      "nll_loss": 0.618945300579071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.062744140625,
      "rewards/margins": 0.11781005561351776,
      "rewards/rejected": -0.18061523139476776,
      "step": 11670
    },
    {
      "epoch": 0.88632569433905,
      "grad_norm": 2.975009194252188,
      "learning_rate": 7.402332101976052e-07,
      "log_odds_chosen": 1.771875023841858,
      "log_odds_ratio": -0.333984375,
      "logits/chosen": -1.048828125,
      "logits/rejected": -0.913867175579071,
      "logps/chosen": -0.6502929925918579,
      "logps/rejected": -1.8996093273162842,
      "loss": 0.6537,
      "nll_loss": 0.58056640625,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06502685695886612,
      "rewards/margins": 0.12486572563648224,
      "rewards/rejected": -0.18989257514476776,
      "step": 11680
    },
    {
      "epoch": 0.8870845348307785,
      "grad_norm": 3.3480354488537647,
      "learning_rate": 7.399165328739372e-07,
      "log_odds_chosen": 1.801171898841858,
      "log_odds_ratio": -0.3548828065395355,
      "logits/chosen": -0.966992199420929,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.594433605670929,
      "logps/rejected": -1.841796875,
      "loss": 0.6577,
      "nll_loss": 0.6136718988418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05947265774011612,
      "rewards/margins": 0.12478027492761612,
      "rewards/rejected": -0.18413086235523224,
      "step": 11690
    },
    {
      "epoch": 0.8878433753225072,
      "grad_norm": 2.701217306438458,
      "learning_rate": 7.396002616336387e-07,
      "log_odds_chosen": 1.871679663658142,
      "log_odds_ratio": -0.358642578125,
      "logits/chosen": -1.022851586341858,
      "logits/rejected": -0.9111328125,
      "logps/chosen": -0.6219726800918579,
      "logps/rejected": -1.941015601158142,
      "loss": 0.6572,
      "nll_loss": 0.6273437738418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06220703199505806,
      "rewards/margins": 0.13192138075828552,
      "rewards/rejected": -0.19399413466453552,
      "step": 11700
    },
    {
      "epoch": 0.8886022158142358,
      "grad_norm": 2.9381660653089536,
      "learning_rate": 7.392843956095663e-07,
      "log_odds_chosen": 1.5520508289337158,
      "log_odds_ratio": -0.4093261659145355,
      "logits/chosen": -1.037109375,
      "logits/rejected": -0.9125000238418579,
      "logps/chosen": -0.6387695074081421,
      "logps/rejected": -1.698828101158142,
      "loss": 0.6477,
      "nll_loss": 0.5956054925918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.0638427734375,
      "rewards/margins": 0.106048583984375,
      "rewards/rejected": -0.16989746689796448,
      "step": 11710
    },
    {
      "epoch": 0.8893610563059645,
      "grad_norm": 3.2779025355957194,
      "learning_rate": 7.389689339371664e-07,
      "log_odds_chosen": 1.8591797351837158,
      "log_odds_ratio": -0.28886717557907104,
      "logits/chosen": -1.0978515148162842,
      "logits/rejected": -0.939648449420929,
      "logps/chosen": -0.591113269329071,
      "logps/rejected": -1.891015648841858,
      "loss": 0.6451,
      "nll_loss": 0.5689452886581421,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05914306640625,
      "rewards/margins": 0.12994384765625,
      "rewards/rejected": -0.18901367485523224,
      "step": 11720
    },
    {
      "epoch": 0.8901198967976931,
      "grad_norm": 3.0969955708145394,
      "learning_rate": 7.386538757544653e-07,
      "log_odds_chosen": 1.5403320789337158,
      "log_odds_ratio": -0.4004882872104645,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.94921875,
      "logps/chosen": -0.68603515625,
      "logps/rejected": -1.7785155773162842,
      "loss": 0.647,
      "nll_loss": 0.6703125238418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.068603515625,
      "rewards/margins": 0.10924682766199112,
      "rewards/rejected": -0.17788085341453552,
      "step": 11730
    },
    {
      "epoch": 0.8908787372894218,
      "grad_norm": 3.948526007772392,
      "learning_rate": 7.3833922020206e-07,
      "log_odds_chosen": 1.8898437023162842,
      "log_odds_ratio": -0.328125,
      "logits/chosen": -1.040429711341858,
      "logits/rejected": -0.940625011920929,
      "logps/chosen": -0.59375,
      "logps/rejected": -1.9462890625,
      "loss": 0.6336,
      "nll_loss": 0.552929699420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05938720703125,
      "rewards/margins": 0.13508300483226776,
      "rewards/rejected": -0.19467774033546448,
      "step": 11740
    },
    {
      "epoch": 0.8916375777811504,
      "grad_norm": 3.6352863021528106,
      "learning_rate": 7.38024966423108e-07,
      "log_odds_chosen": 1.842187523841858,
      "log_odds_ratio": -0.35107421875,
      "logits/chosen": -1.0392577648162842,
      "logits/rejected": -0.9583984613418579,
      "logps/chosen": -0.627734363079071,
      "logps/rejected": -1.951171875,
      "loss": 0.6495,
      "nll_loss": 0.603808581829071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06280517578125,
      "rewards/margins": 0.13227538764476776,
      "rewards/rejected": -0.19501952826976776,
      "step": 11750
    },
    {
      "epoch": 0.8923964182728791,
      "grad_norm": 3.106633949271223,
      "learning_rate": 7.377111135633174e-07,
      "log_odds_chosen": 1.441796898841858,
      "log_odds_ratio": -0.41743165254592896,
      "logits/chosen": -1.0185546875,
      "logits/rejected": -0.9332031011581421,
      "logps/chosen": -0.6416015625,
      "logps/rejected": -1.643945336341858,
      "loss": 0.6647,
      "nll_loss": 0.611328125,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06418456882238388,
      "rewards/margins": 0.10022582858800888,
      "rewards/rejected": -0.16435547173023224,
      "step": 11760
    },
    {
      "epoch": 0.8931552587646077,
      "grad_norm": 2.78889207137325,
      "learning_rate": 7.373976607709372e-07,
      "log_odds_chosen": 1.8173828125,
      "log_odds_ratio": -0.3038574159145355,
      "logits/chosen": -1.0476562976837158,
      "logits/rejected": -0.944531261920929,
      "logps/chosen": -0.621777355670929,
      "logps/rejected": -1.857421875,
      "loss": 0.6751,
      "nll_loss": 0.6304687261581421,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.06218872219324112,
      "rewards/margins": 0.12353515625,
      "rewards/rejected": -0.18571777641773224,
      "step": 11770
    },
    {
      "epoch": 0.8939140992563364,
      "grad_norm": 3.1401449269415216,
      "learning_rate": 7.370846071967476e-07,
      "log_odds_chosen": 1.5535156726837158,
      "log_odds_ratio": -0.40288084745407104,
      "logits/chosen": -1.057226538658142,
      "logits/rejected": -0.91796875,
      "logps/chosen": -0.69482421875,
      "logps/rejected": -1.80859375,
      "loss": 0.6468,
      "nll_loss": 0.6424804925918579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06955566257238388,
      "rewards/margins": 0.11126708984375,
      "rewards/rejected": -0.18051758408546448,
      "step": 11780
    },
    {
      "epoch": 0.8946729397480649,
      "grad_norm": 2.5156113724366014,
      "learning_rate": 7.367719519940501e-07,
      "log_odds_chosen": 1.763281226158142,
      "log_odds_ratio": -0.3629394471645355,
      "logits/chosen": -1.053125023841858,
      "logits/rejected": -0.928906261920929,
      "logps/chosen": -0.5873047113418579,
      "logps/rejected": -1.798828125,
      "loss": 0.6458,
      "nll_loss": 0.5731445550918579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05874023586511612,
      "rewards/margins": 0.12115478515625,
      "rewards/rejected": -0.17978516221046448,
      "step": 11790
    },
    {
      "epoch": 0.8954317802397936,
      "grad_norm": 2.7310902019931573,
      "learning_rate": 7.364596943186587e-07,
      "log_odds_chosen": 1.7121093273162842,
      "log_odds_ratio": -0.3478027284145355,
      "logits/chosen": -1.0556640625,
      "logits/rejected": -0.9140625,
      "logps/chosen": -0.609082043170929,
      "logps/rejected": -1.775390625,
      "loss": 0.6567,
      "nll_loss": 0.607128918170929,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06093750149011612,
      "rewards/margins": 0.11660156399011612,
      "rewards/rejected": -0.17756347358226776,
      "step": 11800
    },
    {
      "epoch": 0.8961906207315222,
      "grad_norm": 2.990131064777284,
      "learning_rate": 7.36147833328889e-07,
      "log_odds_chosen": 1.8049437999725342,
      "log_odds_ratio": -0.37421876192092896,
      "logits/chosen": -0.990429699420929,
      "logits/rejected": -0.890625,
      "logps/chosen": -0.6767578125,
      "logps/rejected": -1.9705078601837158,
      "loss": 0.6452,
      "nll_loss": 0.667773425579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06766357272863388,
      "rewards/margins": 0.12948302924633026,
      "rewards/rejected": -0.19707031548023224,
      "step": 11810
    },
    {
      "epoch": 0.8969494612232509,
      "grad_norm": 3.1128098462279983,
      "learning_rate": 7.358363681855503e-07,
      "log_odds_chosen": 1.850683569908142,
      "log_odds_ratio": -0.33417969942092896,
      "logits/chosen": -0.9830077886581421,
      "logits/rejected": -0.845703125,
      "logps/chosen": -0.64990234375,
      "logps/rejected": -1.8855469226837158,
      "loss": 0.6571,
      "nll_loss": 0.685253918170929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06494140625,
      "rewards/margins": 0.12360839545726776,
      "rewards/rejected": -0.18850097060203552,
      "step": 11820
    },
    {
      "epoch": 0.8977083017149795,
      "grad_norm": 3.3083430914983323,
      "learning_rate": 7.355252980519345e-07,
      "log_odds_chosen": 1.837304711341858,
      "log_odds_ratio": -0.33037108182907104,
      "logits/chosen": -1.1103515625,
      "logits/rejected": -0.9554687738418579,
      "logps/chosen": -0.620800793170929,
      "logps/rejected": -1.889062523841858,
      "loss": 0.6184,
      "nll_loss": 0.557910144329071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06210937350988388,
      "rewards/margins": 0.12678222358226776,
      "rewards/rejected": -0.18876953423023224,
      "step": 11830
    },
    {
      "epoch": 0.8984671422067082,
      "grad_norm": 3.9041359774410465,
      "learning_rate": 7.352146220938078e-07,
      "log_odds_chosen": 1.4796874523162842,
      "log_odds_ratio": -0.4404296875,
      "logits/chosen": -1.0222656726837158,
      "logits/rejected": -0.906054675579071,
      "logps/chosen": -0.6332031488418579,
      "logps/rejected": -1.697265625,
      "loss": 0.665,
      "nll_loss": 0.61328125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06333007663488388,
      "rewards/margins": 0.10637207329273224,
      "rewards/rejected": -0.16958007216453552,
      "step": 11840
    },
    {
      "epoch": 0.8992259826984368,
      "grad_norm": 3.219427227677807,
      "learning_rate": 7.349043394794005e-07,
      "log_odds_chosen": 1.675195336341858,
      "log_odds_ratio": -0.3504394590854645,
      "logits/chosen": -1.04296875,
      "logits/rejected": -0.929882824420929,
      "logps/chosen": -0.6142578125,
      "logps/rejected": -1.758398413658142,
      "loss": 0.6403,
      "nll_loss": 0.6097656488418579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06148681789636612,
      "rewards/margins": 0.1143798828125,
      "rewards/rejected": -0.17583008110523224,
      "step": 11850
    },
    {
      "epoch": 0.8999848231901654,
      "grad_norm": 3.4223339136611974,
      "learning_rate": 7.345944493793987e-07,
      "log_odds_chosen": 1.617517113685608,
      "log_odds_ratio": -0.37646484375,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.935546875,
      "logps/chosen": -0.647265613079071,
      "logps/rejected": -1.7333984375,
      "loss": 0.6328,
      "nll_loss": 0.585644543170929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06477050483226776,
      "rewards/margins": 0.10868225246667862,
      "rewards/rejected": -0.17341308295726776,
      "step": 11860
    },
    {
      "epoch": 0.9007436636818941,
      "grad_norm": 3.252667403890425,
      "learning_rate": 7.342849509669337e-07,
      "log_odds_chosen": 1.5021483898162842,
      "log_odds_ratio": -0.4127441346645355,
      "logits/chosen": -1.0320312976837158,
      "logits/rejected": -0.903515636920929,
      "logps/chosen": -0.6084960699081421,
      "logps/rejected": -1.574609398841858,
      "loss": 0.6368,
      "nll_loss": 0.6416015625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06081543117761612,
      "rewards/margins": 0.09659423679113388,
      "rewards/rejected": -0.15739746391773224,
      "step": 11870
    },
    {
      "epoch": 0.9015025041736227,
      "grad_norm": 3.152637118780761,
      "learning_rate": 7.339758434175737e-07,
      "log_odds_chosen": 1.40185546875,
      "log_odds_ratio": -0.4181152284145355,
      "logits/chosen": -1.000585913658142,
      "logits/rejected": -0.876953125,
      "logps/chosen": -0.610546886920929,
      "logps/rejected": -1.524804711341858,
      "loss": 0.6466,
      "nll_loss": 0.594921886920929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06107177585363388,
      "rewards/margins": 0.09139404445886612,
      "rewards/rejected": -0.15253905951976776,
      "step": 11880
    },
    {
      "epoch": 0.9022613446653514,
      "grad_norm": 4.109447528502229,
      "learning_rate": 7.336671259093143e-07,
      "log_odds_chosen": 1.796972632408142,
      "log_odds_ratio": -0.35895997285842896,
      "logits/chosen": -1.0476562976837158,
      "logits/rejected": -0.9232422113418579,
      "logps/chosen": -0.641894519329071,
      "logps/rejected": -1.955468773841858,
      "loss": 0.6417,
      "nll_loss": 0.6031249761581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06418456882238388,
      "rewards/margins": 0.13120117783546448,
      "rewards/rejected": -0.19553223252296448,
      "step": 11890
    },
    {
      "epoch": 0.9030201851570799,
      "grad_norm": 3.268739879699178,
      "learning_rate": 7.33358797622569e-07,
      "log_odds_chosen": 1.83349609375,
      "log_odds_ratio": -0.39238280057907104,
      "logits/chosen": -1.060546875,
      "logits/rejected": -0.935351550579071,
      "logps/chosen": -0.65380859375,
      "logps/rejected": -1.8937499523162842,
      "loss": 0.6458,
      "nll_loss": 0.6128906011581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06533203274011612,
      "rewards/margins": 0.12408447265625,
      "rewards/rejected": -0.18952636420726776,
      "step": 11900
    },
    {
      "epoch": 0.9037790256488086,
      "grad_norm": 3.110762349757057,
      "learning_rate": 7.330508577401606e-07,
      "log_odds_chosen": 1.583593726158142,
      "log_odds_ratio": -0.38134765625,
      "logits/chosen": -1.0412108898162842,
      "logits/rejected": -0.9476562738418579,
      "logps/chosen": -0.5772460699081421,
      "logps/rejected": -1.60546875,
      "loss": 0.656,
      "nll_loss": 0.624218761920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.05781250074505806,
      "rewards/margins": 0.10285644233226776,
      "rewards/rejected": -0.1605224609375,
      "step": 11910
    },
    {
      "epoch": 0.9045378661405372,
      "grad_norm": 3.483227300877117,
      "learning_rate": 7.327433054473117e-07,
      "log_odds_chosen": 1.701171875,
      "log_odds_ratio": -0.34648436307907104,
      "logits/chosen": -0.9917968511581421,
      "logits/rejected": -0.9066406488418579,
      "logps/chosen": -0.597363293170929,
      "logps/rejected": -1.714257836341858,
      "loss": 0.6429,
      "nll_loss": 0.602734386920929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05980224534869194,
      "rewards/margins": 0.11165771633386612,
      "rewards/rejected": -0.17133788764476776,
      "step": 11920
    },
    {
      "epoch": 0.9052967066322659,
      "grad_norm": 3.672418790207694,
      "learning_rate": 7.324361399316357e-07,
      "log_odds_chosen": 1.5207030773162842,
      "log_odds_ratio": -0.39140623807907104,
      "logits/chosen": -1.0216796398162842,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.6280273199081421,
      "logps/rejected": -1.6398437023162842,
      "loss": 0.6363,
      "nll_loss": 0.6446288824081421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06278076022863388,
      "rewards/margins": 0.10123901069164276,
      "rewards/rejected": -0.16403809189796448,
      "step": 11930
    },
    {
      "epoch": 0.9060555471239945,
      "grad_norm": 4.053285182268707,
      "learning_rate": 7.321293603831281e-07,
      "log_odds_chosen": 1.557275414466858,
      "log_odds_ratio": -0.40693360567092896,
      "logits/chosen": -1.018164038658142,
      "logits/rejected": -0.9136718511581421,
      "logps/chosen": -0.697558581829071,
      "logps/rejected": -1.7976562976837158,
      "loss": 0.6613,
      "nll_loss": 0.715039074420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06973876804113388,
      "rewards/margins": 0.11004028469324112,
      "rewards/rejected": -0.17976073920726776,
      "step": 11940
    },
    {
      "epoch": 0.9068143876157232,
      "grad_norm": 2.9692824257321697,
      "learning_rate": 7.318229659941572e-07,
      "log_odds_chosen": 1.3681640625,
      "log_odds_ratio": -0.461181640625,
      "logits/chosen": -1.015039086341858,
      "logits/rejected": -0.9398437738418579,
      "logps/chosen": -0.6263672113418579,
      "logps/rejected": -1.5652344226837158,
      "loss": 0.6335,
      "nll_loss": 0.6200195550918579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06259765475988388,
      "rewards/margins": 0.09382323920726776,
      "rewards/rejected": -0.1563720703125,
      "step": 11950
    },
    {
      "epoch": 0.9075732281074518,
      "grad_norm": 3.1165967050358323,
      "learning_rate": 7.315169559594551e-07,
      "log_odds_chosen": 1.676171898841858,
      "log_odds_ratio": -0.36870115995407104,
      "logits/chosen": -1.058203101158142,
      "logits/rejected": -0.938281238079071,
      "logps/chosen": -0.6133788824081421,
      "logps/rejected": -1.739843726158142,
      "loss": 0.6295,
      "nll_loss": 0.651562511920929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06137695163488388,
      "rewards/margins": 0.112548828125,
      "rewards/rejected": -0.17390136420726776,
      "step": 11960
    },
    {
      "epoch": 0.9083320685991805,
      "grad_norm": 2.934125999161286,
      "learning_rate": 7.31211329476109e-07,
      "log_odds_chosen": 2.051562547683716,
      "log_odds_ratio": -0.33470457792282104,
      "logits/chosen": -1.0138671398162842,
      "logits/rejected": -0.9078124761581421,
      "logps/chosen": -0.6385742425918579,
      "logps/rejected": -2.1015625,
      "loss": 0.6454,
      "nll_loss": 0.654589831829071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06387939304113388,
      "rewards/margins": 0.14645996689796448,
      "rewards/rejected": -0.21035155653953552,
      "step": 11970
    },
    {
      "epoch": 0.9090909090909091,
      "grad_norm": 3.1697267537755933,
      "learning_rate": 7.309060857435526e-07,
      "log_odds_chosen": 1.506250023841858,
      "log_odds_ratio": -0.3849121034145355,
      "logits/chosen": -1.022070288658142,
      "logits/rejected": -0.9033203125,
      "logps/chosen": -0.6166015863418579,
      "logps/rejected": -1.626367211341858,
      "loss": 0.6467,
      "nll_loss": 0.60595703125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06166992336511612,
      "rewards/margins": 0.10086669772863388,
      "rewards/rejected": -0.16242675483226776,
      "step": 11980
    },
    {
      "epoch": 0.9098497495826378,
      "grad_norm": 4.3764883304732445,
      "learning_rate": 7.30601223963557e-07,
      "log_odds_chosen": 1.8020508289337158,
      "log_odds_ratio": -0.33598631620407104,
      "logits/chosen": -1.0837891101837158,
      "logits/rejected": -0.970703125,
      "logps/chosen": -0.606738269329071,
      "logps/rejected": -1.8447265625,
      "loss": 0.6607,
      "nll_loss": 0.5601562261581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06064452975988388,
      "rewards/margins": 0.12388153374195099,
      "rewards/rejected": -0.18461914360523224,
      "step": 11990
    },
    {
      "epoch": 0.9106085900743663,
      "grad_norm": 3.77292271609659,
      "learning_rate": 7.302967433402214e-07,
      "log_odds_chosen": 1.874609351158142,
      "log_odds_ratio": -0.3087402284145355,
      "logits/chosen": -1.0701172351837158,
      "logits/rejected": -0.9224609136581421,
      "logps/chosen": -0.58544921875,
      "logps/rejected": -1.8429687023162842,
      "loss": 0.6273,
      "nll_loss": 0.561328113079071,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05852050706744194,
      "rewards/margins": 0.12587890028953552,
      "rewards/rejected": -0.18437500298023224,
      "step": 12000
    },
    {
      "epoch": 0.911367430566095,
      "grad_norm": 3.2148758992314956,
      "learning_rate": 7.299926430799657e-07,
      "log_odds_chosen": 1.6477539539337158,
      "log_odds_ratio": -0.3829589784145355,
      "logits/chosen": -1.0353515148162842,
      "logits/rejected": -0.9566406011581421,
      "logps/chosen": -0.621289074420929,
      "logps/rejected": -1.7351562976837158,
      "loss": 0.6428,
      "nll_loss": 0.61474609375,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06210937350988388,
      "rewards/margins": 0.111419677734375,
      "rewards/rejected": -0.1737060546875,
      "step": 12010
    },
    {
      "epoch": 0.9121262710578236,
      "grad_norm": 2.8590537125114257,
      "learning_rate": 7.296889223915205e-07,
      "log_odds_chosen": 1.6876952648162842,
      "log_odds_ratio": -0.3500732481479645,
      "logits/chosen": -1.037109375,
      "logits/rejected": -0.931445300579071,
      "logps/chosen": -0.5863281488418579,
      "logps/rejected": -1.7238280773162842,
      "loss": 0.6592,
      "nll_loss": 0.6029297113418579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05863036960363388,
      "rewards/margins": 0.11378173530101776,
      "rewards/rejected": -0.1722412109375,
      "step": 12020
    },
    {
      "epoch": 0.9128851115495523,
      "grad_norm": 3.0626633163543278,
      "learning_rate": 7.293855804859192e-07,
      "log_odds_chosen": 1.8337891101837158,
      "log_odds_ratio": -0.328857421875,
      "logits/chosen": -1.019921898841858,
      "logits/rejected": -0.9369140863418579,
      "logps/chosen": -0.624316394329071,
      "logps/rejected": -1.916406273841858,
      "loss": 0.6664,
      "nll_loss": 0.6142578125,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06236572191119194,
      "rewards/margins": 0.12929686903953552,
      "rewards/rejected": -0.19172362983226776,
      "step": 12030
    },
    {
      "epoch": 0.9136439520412809,
      "grad_norm": 4.375425529249199,
      "learning_rate": 7.290826165764892e-07,
      "log_odds_chosen": 1.483984351158142,
      "log_odds_ratio": -0.4235595762729645,
      "logits/chosen": -1.0615234375,
      "logits/rejected": -0.962109386920929,
      "logps/chosen": -0.6617187261581421,
      "logps/rejected": -1.708984375,
      "loss": 0.6532,
      "nll_loss": 0.622851550579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06614990532398224,
      "rewards/margins": 0.10487060248851776,
      "rewards/rejected": -0.171142578125,
      "step": 12040
    },
    {
      "epoch": 0.9144027925330096,
      "grad_norm": 2.837357685916429,
      "learning_rate": 7.28780029878843e-07,
      "log_odds_chosen": 1.6157715320587158,
      "log_odds_ratio": -0.3611816465854645,
      "logits/chosen": -1.0791015625,
      "logits/rejected": -0.967968761920929,
      "logps/chosen": -0.5692383050918579,
      "logps/rejected": -1.6376953125,
      "loss": 0.6518,
      "nll_loss": 0.56298828125,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05697021633386612,
      "rewards/margins": 0.10677184909582138,
      "rewards/rejected": -0.16359862685203552,
      "step": 12050
    },
    {
      "epoch": 0.9151616330247382,
      "grad_norm": 2.7648230116639474,
      "learning_rate": 7.284778196108706e-07,
      "log_odds_chosen": 1.979101538658142,
      "log_odds_ratio": -0.32368165254592896,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.9378906488418579,
      "logps/chosen": -0.586132824420929,
      "logps/rejected": -2.0042967796325684,
      "loss": 0.6476,
      "nll_loss": 0.578320324420929,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.05860595777630806,
      "rewards/margins": 0.14190062880516052,
      "rewards/rejected": -0.20053711533546448,
      "step": 12060
    },
    {
      "epoch": 0.9159204735164669,
      "grad_norm": 4.821555118625723,
      "learning_rate": 7.281759849927299e-07,
      "log_odds_chosen": 1.9773437976837158,
      "log_odds_ratio": -0.30439454317092896,
      "logits/chosen": -1.052148461341858,
      "logits/rejected": -0.966992199420929,
      "logps/chosen": -0.601367175579071,
      "logps/rejected": -1.9285156726837158,
      "loss": 0.6362,
      "nll_loss": 0.584179699420929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06016845628619194,
      "rewards/margins": 0.13276366889476776,
      "rewards/rejected": -0.1929931640625,
      "step": 12070
    },
    {
      "epoch": 0.9166793140081955,
      "grad_norm": 3.489407992794823,
      "learning_rate": 7.278745252468389e-07,
      "log_odds_chosen": 1.8035156726837158,
      "log_odds_ratio": -0.3304199278354645,
      "logits/chosen": -1.0261719226837158,
      "logits/rejected": -0.911328136920929,
      "logps/chosen": -0.5986328125,
      "logps/rejected": -1.8390624523162842,
      "loss": 0.6476,
      "nll_loss": 0.604199230670929,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.05989990383386612,
      "rewards/margins": 0.12399902194738388,
      "rewards/rejected": -0.18391112983226776,
      "step": 12080
    },
    {
      "epoch": 0.9174381544999242,
      "grad_norm": 3.3245536042064097,
      "learning_rate": 7.275734395978672e-07,
      "log_odds_chosen": 1.732421875,
      "log_odds_ratio": -0.34760743379592896,
      "logits/chosen": -1.0421874523162842,
      "logits/rejected": -0.949414074420929,
      "logps/chosen": -0.615917980670929,
      "logps/rejected": -1.8154296875,
      "loss": 0.6563,
      "nll_loss": 0.588574230670929,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06154785305261612,
      "rewards/margins": 0.11998291313648224,
      "rewards/rejected": -0.18154296278953552,
      "step": 12090
    },
    {
      "epoch": 0.9181969949916527,
      "grad_norm": 3.470604117993712,
      "learning_rate": 7.272727272727272e-07,
      "log_odds_chosen": 1.907812476158142,
      "log_odds_ratio": -0.3082031309604645,
      "logits/chosen": -1.0,
      "logits/rejected": -0.9146484136581421,
      "logps/chosen": -0.59716796875,
      "logps/rejected": -1.9171874523162842,
      "loss": 0.6368,
      "nll_loss": 0.613964855670929,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.05974121019244194,
      "rewards/margins": 0.13205567002296448,
      "rewards/rejected": -0.19179686903953552,
      "step": 12100
    },
    {
      "epoch": 0.9189558354833814,
      "grad_norm": 3.1331674358914454,
      "learning_rate": 7.269723875005668e-07,
      "log_odds_chosen": 1.705078125,
      "log_odds_ratio": -0.3971923887729645,
      "logits/chosen": -1.057226538658142,
      "logits/rejected": -0.9673827886581421,
      "logps/chosen": -0.6181640625,
      "logps/rejected": -1.8035156726837158,
      "loss": 0.6542,
      "nll_loss": 0.5884765386581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06185302883386612,
      "rewards/margins": 0.11842651665210724,
      "rewards/rejected": -0.18034668266773224,
      "step": 12110
    },
    {
      "epoch": 0.91971467597511,
      "grad_norm": 3.137744097256631,
      "learning_rate": 7.266724195127595e-07,
      "log_odds_chosen": 1.870703101158142,
      "log_odds_ratio": -0.33393555879592896,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.9087890386581421,
      "logps/chosen": -0.633105456829071,
      "logps/rejected": -1.966406226158142,
      "loss": 0.6477,
      "nll_loss": 0.5933593511581421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0633544921875,
      "rewards/margins": 0.13327637314796448,
      "rewards/rejected": -0.19667968153953552,
      "step": 12120
    },
    {
      "epoch": 0.9204735164668387,
      "grad_norm": 3.4355040771623178,
      "learning_rate": 7.26372822542898e-07,
      "log_odds_chosen": 1.8292968273162842,
      "log_odds_ratio": -0.32958984375,
      "logits/chosen": -1.08203125,
      "logits/rejected": -0.952343761920929,
      "logps/chosen": -0.6612304449081421,
      "logps/rejected": -1.939062476158142,
      "loss": 0.6491,
      "nll_loss": 0.627636730670929,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06612548977136612,
      "rewards/margins": 0.12783202528953552,
      "rewards/rejected": -0.19384765625,
      "step": 12130
    },
    {
      "epoch": 0.9212323569585673,
      "grad_norm": 3.839175569081039,
      "learning_rate": 7.260735958267845e-07,
      "log_odds_chosen": 1.4794921875,
      "log_odds_ratio": -0.43115234375,
      "logits/chosen": -1.007226586341858,
      "logits/rejected": -0.925976574420929,
      "logps/chosen": -0.667187511920929,
      "logps/rejected": -1.682031273841858,
      "loss": 0.6289,
      "nll_loss": 0.6341797113418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06671142578125,
      "rewards/margins": 0.1014404296875,
      "rewards/rejected": -0.168212890625,
      "step": 12140
    },
    {
      "epoch": 0.9219911974502959,
      "grad_norm": 3.369657652050872,
      "learning_rate": 7.257747386024231e-07,
      "log_odds_chosen": 1.568945288658142,
      "log_odds_ratio": -0.38090819120407104,
      "logits/chosen": -1.0070312023162842,
      "logits/rejected": -0.936328113079071,
      "logps/chosen": -0.654492199420929,
      "logps/rejected": -1.7726562023162842,
      "loss": 0.642,
      "nll_loss": 0.663281261920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06540527194738388,
      "rewards/margins": 0.11182098090648651,
      "rewards/rejected": -0.1773681640625,
      "step": 12150
    },
    {
      "epoch": 0.9227500379420246,
      "grad_norm": 2.743233203456519,
      "learning_rate": 7.254762501100117e-07,
      "log_odds_chosen": 1.690039038658142,
      "log_odds_ratio": -0.3551269471645355,
      "logits/chosen": -1.0771484375,
      "logits/rejected": -0.9818359613418579,
      "logps/chosen": -0.6436523199081421,
      "logps/rejected": -1.862890601158142,
      "loss": 0.6277,
      "nll_loss": 0.5995117425918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06427001953125,
      "rewards/margins": 0.12185058742761612,
      "rewards/rejected": -0.18625488877296448,
      "step": 12160
    },
    {
      "epoch": 0.9235088784337532,
      "grad_norm": 3.3073753395229937,
      "learning_rate": 7.251781295919335e-07,
      "log_odds_chosen": 1.6251952648162842,
      "log_odds_ratio": -0.3792968690395355,
      "logits/chosen": -1.0089843273162842,
      "logits/rejected": -0.840039074420929,
      "logps/chosen": -0.636425793170929,
      "logps/rejected": -1.781640648841858,
      "loss": 0.6391,
      "nll_loss": 0.6201171875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06367187201976776,
      "rewards/margins": 0.11439208686351776,
      "rewards/rejected": -0.17805175483226776,
      "step": 12170
    },
    {
      "epoch": 0.9242677189254819,
      "grad_norm": 3.828945441764687,
      "learning_rate": 7.248803762927498e-07,
      "log_odds_chosen": 1.8732421398162842,
      "log_odds_ratio": -0.36555176973342896,
      "logits/chosen": -1.0574219226837158,
      "logits/rejected": -0.9371093511581421,
      "logps/chosen": -0.6937500238418579,
      "logps/rejected": -2.0699219703674316,
      "loss": 0.6435,
      "nll_loss": 0.6416015625,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.0693359375,
      "rewards/margins": 0.13764648139476776,
      "rewards/rejected": -0.20693358778953552,
      "step": 12180
    },
    {
      "epoch": 0.9250265594172105,
      "grad_norm": 3.600732320008552,
      "learning_rate": 7.245829894591907e-07,
      "log_odds_chosen": 1.61474609375,
      "log_odds_ratio": -0.39606934785842896,
      "logits/chosen": -1.049414038658142,
      "logits/rejected": -0.9521484375,
      "logps/chosen": -0.635449230670929,
      "logps/rejected": -1.7380859851837158,
      "loss": 0.6416,
      "nll_loss": 0.624804675579071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06354980170726776,
      "rewards/margins": 0.11020507663488388,
      "rewards/rejected": -0.1737060546875,
      "step": 12190
    },
    {
      "epoch": 0.9257853999089392,
      "grad_norm": 4.048667225515555,
      "learning_rate": 7.242859683401482e-07,
      "log_odds_chosen": 1.822265625,
      "log_odds_ratio": -0.33735352754592896,
      "logits/chosen": -1.0388672351837158,
      "logits/rejected": -0.9361327886581421,
      "logps/chosen": -0.567675769329071,
      "logps/rejected": -1.8097655773162842,
      "loss": 0.6196,
      "nll_loss": 0.5791015625,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05675048753619194,
      "rewards/margins": 0.124267578125,
      "rewards/rejected": -0.18105468153953552,
      "step": 12200
    },
    {
      "epoch": 0.9265442404006677,
      "grad_norm": 4.82889094863894,
      "learning_rate": 7.239893121866677e-07,
      "log_odds_chosen": 1.472314476966858,
      "log_odds_ratio": -0.4139160215854645,
      "logits/chosen": -1.0119140148162842,
      "logits/rejected": -0.884960949420929,
      "logps/chosen": -0.6122070550918579,
      "logps/rejected": -1.571679711341858,
      "loss": 0.6364,
      "nll_loss": 0.698925793170929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06123046949505806,
      "rewards/margins": 0.09595108032226562,
      "rewards/rejected": -0.15720215439796448,
      "step": 12210
    },
    {
      "epoch": 0.9273030808923964,
      "grad_norm": 3.9274164295917435,
      "learning_rate": 7.236930202519399e-07,
      "log_odds_chosen": 1.537695288658142,
      "log_odds_ratio": -0.36577147245407104,
      "logits/chosen": -1.011328101158142,
      "logits/rejected": -0.890625,
      "logps/chosen": -0.6512695550918579,
      "logps/rejected": -1.694726586341858,
      "loss": 0.6379,
      "nll_loss": 0.651660144329071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06514892727136612,
      "rewards/margins": 0.10443725436925888,
      "rewards/rejected": -0.16948242485523224,
      "step": 12220
    },
    {
      "epoch": 0.928061921384125,
      "grad_norm": 3.0056770824234467,
      "learning_rate": 7.233970917912936e-07,
      "log_odds_chosen": 1.583984375,
      "log_odds_ratio": -0.352294921875,
      "logits/chosen": -1.057226538658142,
      "logits/rejected": -0.9439452886581421,
      "logps/chosen": -0.5870116949081421,
      "logps/rejected": -1.6357421875,
      "loss": 0.6409,
      "nll_loss": 0.566699206829071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05872802808880806,
      "rewards/margins": 0.10491943359375,
      "rewards/rejected": -0.1634521484375,
      "step": 12230
    },
    {
      "epoch": 0.9288207618758537,
      "grad_norm": 2.9185258487553227,
      "learning_rate": 7.231015260621871e-07,
      "log_odds_chosen": 1.661718726158142,
      "log_odds_ratio": -0.39226073026657104,
      "logits/chosen": -1.0675780773162842,
      "logits/rejected": -0.9779297113418579,
      "logps/chosen": -0.6009765863418579,
      "logps/rejected": -1.7687499523162842,
      "loss": 0.6684,
      "nll_loss": 0.6177734136581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06008300930261612,
      "rewards/margins": 0.11693115532398224,
      "rewards/rejected": -0.17695312201976776,
      "step": 12240
    },
    {
      "epoch": 0.9295796023675823,
      "grad_norm": 3.0158817855282845,
      "learning_rate": 7.228063223242011e-07,
      "log_odds_chosen": 1.6481444835662842,
      "log_odds_ratio": -0.36430662870407104,
      "logits/chosen": -1.037500023841858,
      "logits/rejected": -0.955859363079071,
      "logps/chosen": -0.610058605670929,
      "logps/rejected": -1.7267577648162842,
      "loss": 0.6475,
      "nll_loss": 0.584179699420929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06101074069738388,
      "rewards/margins": 0.11177368462085724,
      "rewards/rejected": -0.17277832329273224,
      "step": 12250
    },
    {
      "epoch": 0.930338442859311,
      "grad_norm": 3.0590306457823684,
      "learning_rate": 7.225114798390295e-07,
      "log_odds_chosen": 1.6525390148162842,
      "log_odds_ratio": -0.3733886778354645,
      "logits/chosen": -1.037500023841858,
      "logits/rejected": -0.9478515386581421,
      "logps/chosen": -0.625781238079071,
      "logps/rejected": -1.7751953601837158,
      "loss": 0.6635,
      "nll_loss": 0.580859363079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06258545070886612,
      "rewards/margins": 0.11477050930261612,
      "rewards/rejected": -0.1773681640625,
      "step": 12260
    },
    {
      "epoch": 0.9310972833510396,
      "grad_norm": 3.4151676723929385,
      "learning_rate": 7.222169978704737e-07,
      "log_odds_chosen": 1.878320336341858,
      "log_odds_ratio": -0.33588868379592896,
      "logits/chosen": -1.0771484375,
      "logits/rejected": -0.9521484375,
      "logps/chosen": -0.6270507574081421,
      "logps/rejected": -1.908593773841858,
      "loss": 0.6445,
      "nll_loss": 0.528515636920929,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06270752102136612,
      "rewards/margins": 0.12819823622703552,
      "rewards/rejected": -0.19091796875,
      "step": 12270
    },
    {
      "epoch": 0.9318561238427683,
      "grad_norm": 2.8939614201951045,
      "learning_rate": 7.219228756844335e-07,
      "log_odds_chosen": 1.5955078601837158,
      "log_odds_ratio": -0.38037109375,
      "logits/chosen": -1.0060546398162842,
      "logits/rejected": -0.925585925579071,
      "logps/chosen": -0.6421874761581421,
      "logps/rejected": -1.7390625476837158,
      "loss": 0.6473,
      "nll_loss": 0.637499988079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06423339992761612,
      "rewards/margins": 0.10979004204273224,
      "rewards/rejected": -0.17390136420726776,
      "step": 12280
    },
    {
      "epoch": 0.9326149643344969,
      "grad_norm": 3.084649131014509,
      "learning_rate": 7.216291125488994e-07,
      "log_odds_chosen": 1.8837890625,
      "log_odds_ratio": -0.3538574278354645,
      "logits/chosen": -1.0535156726837158,
      "logits/rejected": -0.9541015625,
      "logps/chosen": -0.6044921875,
      "logps/rejected": -1.9677734375,
      "loss": 0.6501,
      "nll_loss": 0.6669921875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06044922024011612,
      "rewards/margins": 0.13640137016773224,
      "rewards/rejected": -0.1966552734375,
      "step": 12290
    },
    {
      "epoch": 0.9333738048262256,
      "grad_norm": 3.2040275331095307,
      "learning_rate": 7.213357077339458e-07,
      "log_odds_chosen": 1.709570288658142,
      "log_odds_ratio": -0.3448730409145355,
      "logits/chosen": -1.040624976158142,
      "logits/rejected": -0.886914074420929,
      "logps/chosen": -0.5938476324081421,
      "logps/rejected": -1.761328101158142,
      "loss": 0.622,
      "nll_loss": 0.634765625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05937499925494194,
      "rewards/margins": 0.11688232421875,
      "rewards/rejected": -0.17624512314796448,
      "step": 12300
    },
    {
      "epoch": 0.9341326453179541,
      "grad_norm": 3.421632731847008,
      "learning_rate": 7.210426605117224e-07,
      "log_odds_chosen": 1.904687523841858,
      "log_odds_ratio": -0.293212890625,
      "logits/chosen": -1.0615234375,
      "logits/rejected": -0.918164074420929,
      "logps/chosen": -0.55908203125,
      "logps/rejected": -1.8425781726837158,
      "loss": 0.6589,
      "nll_loss": 0.6141601800918579,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.05587158352136612,
      "rewards/margins": 0.12843017280101776,
      "rewards/rejected": -0.18437500298023224,
      "step": 12310
    },
    {
      "epoch": 0.9348914858096828,
      "grad_norm": 3.349177154240887,
      "learning_rate": 7.207499701564471e-07,
      "log_odds_chosen": 1.623632788658142,
      "log_odds_ratio": -0.3895019590854645,
      "logits/chosen": -1.0402343273162842,
      "logits/rejected": -0.926953136920929,
      "logps/chosen": -0.628710925579071,
      "logps/rejected": -1.7732422351837158,
      "loss": 0.6489,
      "nll_loss": 0.5831054449081421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06290283054113388,
      "rewards/margins": 0.11437378078699112,
      "rewards/rejected": -0.17724609375,
      "step": 12320
    },
    {
      "epoch": 0.9356503263014114,
      "grad_norm": 2.755592474423754,
      "learning_rate": 7.204576359443989e-07,
      "log_odds_chosen": 1.365234375,
      "log_odds_ratio": -0.44609373807907104,
      "logits/chosen": -1.0490233898162842,
      "logits/rejected": -0.953320324420929,
      "logps/chosen": -0.6825195550918579,
      "logps/rejected": -1.5929687023162842,
      "loss": 0.6571,
      "nll_loss": 0.6201171875,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06822510063648224,
      "rewards/margins": 0.09102783352136612,
      "rewards/rejected": -0.15925292670726776,
      "step": 12330
    },
    {
      "epoch": 0.9364091667931401,
      "grad_norm": 3.0588118021815687,
      "learning_rate": 7.201656571539094e-07,
      "log_odds_chosen": 1.4456055164337158,
      "log_odds_ratio": -0.4183105528354645,
      "logits/chosen": -1.0246093273162842,
      "logits/rejected": -0.893750011920929,
      "logps/chosen": -0.611523449420929,
      "logps/rejected": -1.5958983898162842,
      "loss": 0.652,
      "nll_loss": 0.61962890625,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06109619140625,
      "rewards/margins": 0.09849853813648224,
      "rewards/rejected": -0.15959472954273224,
      "step": 12340
    },
    {
      "epoch": 0.9371680072848687,
      "grad_norm": 3.316956844787408,
      "learning_rate": 7.19874033065356e-07,
      "log_odds_chosen": 1.565820336341858,
      "log_odds_ratio": -0.36396485567092896,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.8978515863418579,
      "logps/chosen": -0.6015625,
      "logps/rejected": -1.6423828601837158,
      "loss": 0.6466,
      "nll_loss": 0.544628918170929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06016845628619194,
      "rewards/margins": 0.10418701171875,
      "rewards/rejected": -0.16440430283546448,
      "step": 12350
    },
    {
      "epoch": 0.9379268477765974,
      "grad_norm": 3.0417847742785393,
      "learning_rate": 7.195827629611545e-07,
      "log_odds_chosen": 1.6003906726837158,
      "log_odds_ratio": -0.40253907442092896,
      "logits/chosen": -1.009179711341858,
      "logits/rejected": -0.916015625,
      "logps/chosen": -0.6239258050918579,
      "logps/rejected": -1.73046875,
      "loss": 0.6357,
      "nll_loss": 0.601269543170929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06235351413488388,
      "rewards/margins": 0.11062011867761612,
      "rewards/rejected": -0.1729736328125,
      "step": 12360
    },
    {
      "epoch": 0.938685688268326,
      "grad_norm": 2.7070954378987815,
      "learning_rate": 7.19291846125751e-07,
      "log_odds_chosen": 1.637304663658142,
      "log_odds_ratio": -0.398193359375,
      "logits/chosen": -0.9927734136581421,
      "logits/rejected": -0.8453124761581421,
      "logps/chosen": -0.639355480670929,
      "logps/rejected": -1.834375023841858,
      "loss": 0.6262,
      "nll_loss": 0.5997070074081421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06390380859375,
      "rewards/margins": 0.11960449069738388,
      "rewards/rejected": -0.18364258110523224,
      "step": 12370
    },
    {
      "epoch": 0.9394445287600547,
      "grad_norm": 3.3842486250438517,
      "learning_rate": 7.190012818456154e-07,
      "log_odds_chosen": 1.4533202648162842,
      "log_odds_ratio": -0.41077882051467896,
      "logits/chosen": -1.0154297351837158,
      "logits/rejected": -0.9126952886581421,
      "logps/chosen": -0.638964831829071,
      "logps/rejected": -1.638281226158142,
      "loss": 0.6323,
      "nll_loss": 0.5953124761581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06389160454273224,
      "rewards/margins": 0.10004882514476776,
      "rewards/rejected": -0.16389159858226776,
      "step": 12380
    },
    {
      "epoch": 0.9402033692517833,
      "grad_norm": 3.247624406422012,
      "learning_rate": 7.187110694092334e-07,
      "log_odds_chosen": 1.7313964366912842,
      "log_odds_ratio": -0.34013670682907104,
      "logits/chosen": -1.0458984375,
      "logits/rejected": -0.929882824420929,
      "logps/chosen": -0.5977538824081421,
      "logps/rejected": -1.7849609851837158,
      "loss": 0.6436,
      "nll_loss": 0.6103515625,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05982666090130806,
      "rewards/margins": 0.11870269477367401,
      "rewards/rejected": -0.17844238877296448,
      "step": 12390
    },
    {
      "epoch": 0.940962209743512,
      "grad_norm": 3.246113907051256,
      "learning_rate": 7.184212081070996e-07,
      "log_odds_chosen": 1.34814453125,
      "log_odds_ratio": -0.43510740995407104,
      "logits/chosen": -1.0496094226837158,
      "logits/rejected": -0.9525390863418579,
      "logps/chosen": -0.603515625,
      "logps/rejected": -1.4753906726837158,
      "loss": 0.6186,
      "nll_loss": 0.576367199420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06027831882238388,
      "rewards/margins": 0.08726195991039276,
      "rewards/rejected": -0.14763183891773224,
      "step": 12400
    },
    {
      "epoch": 0.9417210502352406,
      "grad_norm": 4.048972329571119,
      "learning_rate": 7.181316972317097e-07,
      "log_odds_chosen": 1.87890625,
      "log_odds_ratio": -0.33056640625,
      "logits/chosen": -1.0419921875,
      "logits/rejected": -0.9527343511581421,
      "logps/chosen": -0.575878918170929,
      "logps/rejected": -1.8738281726837158,
      "loss": 0.6638,
      "nll_loss": 0.5716797113418579,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.05759277194738388,
      "rewards/margins": 0.12979736924171448,
      "rewards/rejected": -0.18740233778953552,
      "step": 12410
    },
    {
      "epoch": 0.9424798907269691,
      "grad_norm": 3.6177661336809477,
      "learning_rate": 7.17842536077554e-07,
      "log_odds_chosen": 1.876367211341858,
      "log_odds_ratio": -0.3365722596645355,
      "logits/chosen": -1.053125023841858,
      "logits/rejected": -0.933398425579071,
      "logps/chosen": -0.5751953125,
      "logps/rejected": -1.8367187976837158,
      "loss": 0.6336,
      "nll_loss": 0.559765636920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05759277194738388,
      "rewards/margins": 0.12614746391773224,
      "rewards/rejected": -0.18366698920726776,
      "step": 12420
    },
    {
      "epoch": 0.9432387312186978,
      "grad_norm": 2.59817206030697,
      "learning_rate": 7.175537239411094e-07,
      "log_odds_chosen": 1.7004883289337158,
      "log_odds_ratio": -0.3498291075229645,
      "logits/chosen": -1.053320288658142,
      "logits/rejected": -0.9443359375,
      "logps/chosen": -0.592968761920929,
      "logps/rejected": -1.7265625,
      "loss": 0.6119,
      "nll_loss": 0.577832043170929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05931396409869194,
      "rewards/margins": 0.1133880615234375,
      "rewards/rejected": -0.1727294921875,
      "step": 12430
    },
    {
      "epoch": 0.9439975717104264,
      "grad_norm": 3.6121715541606556,
      "learning_rate": 7.172652601208325e-07,
      "log_odds_chosen": 1.802832007408142,
      "log_odds_ratio": -0.37578123807907104,
      "logits/chosen": -1.056249976158142,
      "logits/rejected": -0.9556640386581421,
      "logps/chosen": -0.6756836175918579,
      "logps/rejected": -1.9833984375,
      "loss": 0.6509,
      "nll_loss": 0.601269543170929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06754150241613388,
      "rewards/margins": 0.130706787109375,
      "rewards/rejected": -0.19826659560203552,
      "step": 12440
    },
    {
      "epoch": 0.9447564122021551,
      "grad_norm": 3.717976009255026,
      "learning_rate": 7.169771439171534e-07,
      "log_odds_chosen": 1.471777319908142,
      "log_odds_ratio": -0.3936767578125,
      "logits/chosen": -0.9996093511581421,
      "logits/rejected": -0.868359386920929,
      "logps/chosen": -0.6167968511581421,
      "logps/rejected": -1.6105468273162842,
      "loss": 0.6501,
      "nll_loss": 0.631054699420929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06166992336511612,
      "rewards/margins": 0.09941406548023224,
      "rewards/rejected": -0.16105957329273224,
      "step": 12450
    },
    {
      "epoch": 0.9455152526938837,
      "grad_norm": 3.495830131576265,
      "learning_rate": 7.166893746324661e-07,
      "log_odds_chosen": 1.7121093273162842,
      "log_odds_ratio": -0.3720703125,
      "logits/chosen": -1.009765625,
      "logits/rejected": -0.8804687261581421,
      "logps/chosen": -0.5855468511581421,
      "logps/rejected": -1.744531273841858,
      "loss": 0.6228,
      "nll_loss": 0.57568359375,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05856933444738388,
      "rewards/margins": 0.11600341647863388,
      "rewards/rejected": -0.17438964545726776,
      "step": 12460
    },
    {
      "epoch": 0.9462740931856124,
      "grad_norm": 3.9367320397083607,
      "learning_rate": 7.164019515711245e-07,
      "log_odds_chosen": 1.6604492664337158,
      "log_odds_ratio": -0.37309569120407104,
      "logits/chosen": -1.0617187023162842,
      "logits/rejected": -0.9439452886581421,
      "logps/chosen": -0.5782226324081421,
      "logps/rejected": -1.703125,
      "loss": 0.6314,
      "nll_loss": 0.5536133050918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.05784912034869194,
      "rewards/margins": 0.11236266791820526,
      "rewards/rejected": -0.17014160752296448,
      "step": 12470
    },
    {
      "epoch": 0.947032933677341,
      "grad_norm": 3.8433577340601324,
      "learning_rate": 7.161148740394328e-07,
      "log_odds_chosen": 1.419531226158142,
      "log_odds_ratio": -0.3892578184604645,
      "logits/chosen": -1.0380859375,
      "logits/rejected": -0.9107421636581421,
      "logps/chosen": -0.666210949420929,
      "logps/rejected": -1.6414062976837158,
      "loss": 0.6591,
      "nll_loss": 0.643847644329071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06662597507238388,
      "rewards/margins": 0.09763183444738388,
      "rewards/rejected": -0.16433104872703552,
      "step": 12480
    },
    {
      "epoch": 0.9477917741690697,
      "grad_norm": 3.0629979222446195,
      "learning_rate": 7.158281413456402e-07,
      "log_odds_chosen": 1.6453125476837158,
      "log_odds_ratio": -0.3726562559604645,
      "logits/chosen": -1.056640625,
      "logits/rejected": -0.9263671636581421,
      "logps/chosen": -0.545703113079071,
      "logps/rejected": -1.662109375,
      "loss": 0.5954,
      "nll_loss": 0.523632824420929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05454101413488388,
      "rewards/margins": 0.11160888522863388,
      "rewards/rejected": -0.16618652641773224,
      "step": 12490
    },
    {
      "epoch": 0.9485506146607983,
      "grad_norm": 3.6397188991976095,
      "learning_rate": 7.155417527999326e-07,
      "log_odds_chosen": 1.6749999523162842,
      "log_odds_ratio": -0.370361328125,
      "logits/chosen": -1.0832030773162842,
      "logits/rejected": -0.9525390863418579,
      "logps/chosen": -0.609667956829071,
      "logps/rejected": -1.7580077648162842,
      "loss": 0.6552,
      "nll_loss": 0.5980468988418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06104736402630806,
      "rewards/margins": 0.11485596001148224,
      "rewards/rejected": -0.17585448920726776,
      "step": 12500
    },
    {
      "epoch": 0.949309455152527,
      "grad_norm": 3.6513096878281033,
      "learning_rate": 7.152557077144268e-07,
      "log_odds_chosen": 1.675195336341858,
      "log_odds_ratio": -0.33369141817092896,
      "logits/chosen": -1.0361328125,
      "logits/rejected": -0.9271484613418579,
      "logps/chosen": -0.6219726800918579,
      "logps/rejected": -1.7156250476837158,
      "loss": 0.6441,
      "nll_loss": 0.6400390863418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06223144382238388,
      "rewards/margins": 0.10930176079273224,
      "rewards/rejected": -0.17160645127296448,
      "step": 12510
    },
    {
      "epoch": 0.9500682956442555,
      "grad_norm": 3.125478610872391,
      "learning_rate": 7.149700054031623e-07,
      "log_odds_chosen": 1.7900390625,
      "log_odds_ratio": -0.3603515625,
      "logits/chosen": -1.001367211341858,
      "logits/rejected": -0.886914074420929,
      "logps/chosen": -0.643750011920929,
      "logps/rejected": -1.91796875,
      "loss": 0.6333,
      "nll_loss": 0.609179675579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06439208984375,
      "rewards/margins": 0.12733153998851776,
      "rewards/rejected": -0.19182129204273224,
      "step": 12520
    },
    {
      "epoch": 0.9508271361359842,
      "grad_norm": 3.0330832107339405,
      "learning_rate": 7.146846451820958e-07,
      "log_odds_chosen": 1.714453101158142,
      "log_odds_ratio": -0.3607177734375,
      "logits/chosen": -1.0412108898162842,
      "logits/rejected": -0.938281238079071,
      "logps/chosen": -0.585644543170929,
      "logps/rejected": -1.7277343273162842,
      "loss": 0.6456,
      "nll_loss": 0.560253918170929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.05850829929113388,
      "rewards/margins": 0.11420898139476776,
      "rewards/rejected": -0.1729736328125,
      "step": 12530
    },
    {
      "epoch": 0.9515859766277128,
      "grad_norm": 3.1876236428815004,
      "learning_rate": 7.14399626369093e-07,
      "log_odds_chosen": 1.5498046875,
      "log_odds_ratio": -0.38383787870407104,
      "logits/chosen": -0.999804675579071,
      "logits/rejected": -0.937304675579071,
      "logps/chosen": -0.5762695074081421,
      "logps/rejected": -1.5822265148162842,
      "loss": 0.6082,
      "nll_loss": 0.5865234136581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.05762939527630806,
      "rewards/margins": 0.10060425102710724,
      "rewards/rejected": -0.158203125,
      "step": 12540
    },
    {
      "epoch": 0.9523448171194415,
      "grad_norm": 3.2582191095058626,
      "learning_rate": 7.141149482839228e-07,
      "log_odds_chosen": 1.8603515625,
      "log_odds_ratio": -0.33293455839157104,
      "logits/chosen": -1.047265648841858,
      "logits/rejected": -0.919140636920929,
      "logps/chosen": -0.588574230670929,
      "logps/rejected": -1.8552734851837158,
      "loss": 0.6233,
      "nll_loss": 0.576367199420929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05885009840130806,
      "rewards/margins": 0.12664794921875,
      "rewards/rejected": -0.18559570610523224,
      "step": 12550
    },
    {
      "epoch": 0.9531036576111701,
      "grad_norm": 3.492901299683247,
      "learning_rate": 7.138306102482496e-07,
      "log_odds_chosen": 1.890234351158142,
      "log_odds_ratio": -0.31562501192092896,
      "logits/chosen": -1.0476562976837158,
      "logits/rejected": -0.928906261920929,
      "logps/chosen": -0.618945300579071,
      "logps/rejected": -1.9191405773162842,
      "loss": 0.6463,
      "nll_loss": 0.6036132574081421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06193847581744194,
      "rewards/margins": 0.13007812201976776,
      "rewards/rejected": -0.19196777045726776,
      "step": 12560
    },
    {
      "epoch": 0.9538624981028988,
      "grad_norm": 4.042901158238742,
      "learning_rate": 7.135466115856274e-07,
      "log_odds_chosen": 1.5910155773162842,
      "log_odds_ratio": -0.39326173067092896,
      "logits/chosen": -1.049414038658142,
      "logits/rejected": -0.9486328363418579,
      "logps/chosen": -0.608691394329071,
      "logps/rejected": -1.690039038658142,
      "loss": 0.6517,
      "nll_loss": 0.622265636920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06086425855755806,
      "rewards/margins": 0.10828857123851776,
      "rewards/rejected": -0.16904297471046448,
      "step": 12570
    },
    {
      "epoch": 0.9546213385946274,
      "grad_norm": 3.8719896555992124,
      "learning_rate": 7.13262951621492e-07,
      "log_odds_chosen": 1.6142578125,
      "log_odds_ratio": -0.4276367127895355,
      "logits/chosen": -1.046484351158142,
      "logits/rejected": -0.9189453125,
      "logps/chosen": -0.6373046636581421,
      "logps/rejected": -1.756445288658142,
      "loss": 0.6286,
      "nll_loss": 0.609375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06368408352136612,
      "rewards/margins": 0.11215820163488388,
      "rewards/rejected": -0.17573241889476776,
      "step": 12580
    },
    {
      "epoch": 0.9553801790863561,
      "grad_norm": 3.376624648732554,
      "learning_rate": 7.129796296831554e-07,
      "log_odds_chosen": 1.8406250476837158,
      "log_odds_ratio": -0.32275390625,
      "logits/chosen": -1.058203101158142,
      "logits/rejected": -0.9273437261581421,
      "logps/chosen": -0.607226550579071,
      "logps/rejected": -1.8527343273162842,
      "loss": 0.6143,
      "nll_loss": 0.551953136920929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06070556491613388,
      "rewards/margins": 0.12469482421875,
      "rewards/rejected": -0.18540039658546448,
      "step": 12590
    },
    {
      "epoch": 0.9561390195780847,
      "grad_norm": 3.1393370223414276,
      "learning_rate": 7.126966450997984e-07,
      "log_odds_chosen": 1.7294921875,
      "log_odds_ratio": -0.34199219942092896,
      "logits/chosen": -1.033203125,
      "logits/rejected": -0.9107421636581421,
      "logps/chosen": -0.617382824420929,
      "logps/rejected": -1.7724609375,
      "loss": 0.6286,
      "nll_loss": 0.6195312738418579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06173095852136612,
      "rewards/margins": 0.11542968451976776,
      "rewards/rejected": -0.17722168564796448,
      "step": 12600
    },
    {
      "epoch": 0.9568978600698134,
      "grad_norm": 2.4183098671920353,
      "learning_rate": 7.124139972024637e-07,
      "log_odds_chosen": 1.376562476158142,
      "log_odds_ratio": -0.4270996153354645,
      "logits/chosen": -1.073632836341858,
      "logits/rejected": -0.9212890863418579,
      "logps/chosen": -0.650683581829071,
      "logps/rejected": -1.584570288658142,
      "loss": 0.6358,
      "nll_loss": 0.6358398199081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06505127251148224,
      "rewards/margins": 0.09333495795726776,
      "rewards/rejected": -0.1583251953125,
      "step": 12610
    },
    {
      "epoch": 0.957656700561542,
      "grad_norm": 3.013487270430202,
      "learning_rate": 7.121316853240503e-07,
      "log_odds_chosen": 1.812890648841858,
      "log_odds_ratio": -0.33403319120407104,
      "logits/chosen": -1.0705077648162842,
      "logits/rejected": -0.9624999761581421,
      "logps/chosen": -0.6278320550918579,
      "logps/rejected": -1.87109375,
      "loss": 0.6229,
      "nll_loss": 0.5811523199081421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06275634467601776,
      "rewards/margins": 0.12427978217601776,
      "rewards/rejected": -0.1871337890625,
      "step": 12620
    },
    {
      "epoch": 0.9584155410532706,
      "grad_norm": 2.665079604025996,
      "learning_rate": 7.118497087993057e-07,
      "log_odds_chosen": 1.7042968273162842,
      "log_odds_ratio": -0.391845703125,
      "logits/chosen": -1.0740234851837158,
      "logits/rejected": -0.950976550579071,
      "logps/chosen": -0.6070312261581421,
      "logps/rejected": -1.7986328601837158,
      "loss": 0.6465,
      "nll_loss": 0.56005859375,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06069336086511612,
      "rewards/margins": 0.11931152641773224,
      "rewards/rejected": -0.18000487983226776,
      "step": 12630
    },
    {
      "epoch": 0.9591743815449992,
      "grad_norm": 3.1879801817622235,
      "learning_rate": 7.1156806696482e-07,
      "log_odds_chosen": 1.6044921875,
      "log_odds_ratio": -0.38374024629592896,
      "logits/chosen": -1.059960961341858,
      "logits/rejected": -0.9486328363418579,
      "logps/chosen": -0.5947265625,
      "logps/rejected": -1.685546875,
      "loss": 0.6492,
      "nll_loss": 0.5693359375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.0594482421875,
      "rewards/margins": 0.10908202826976776,
      "rewards/rejected": -0.16855469346046448,
      "step": 12640
    },
    {
      "epoch": 0.9599332220367279,
      "grad_norm": 3.171329183474717,
      "learning_rate": 7.112867591590192e-07,
      "log_odds_chosen": 1.768945336341858,
      "log_odds_ratio": -0.377685546875,
      "logits/chosen": -1.059960961341858,
      "logits/rejected": -0.9173828363418579,
      "logps/chosen": -0.593066394329071,
      "logps/rejected": -1.847265601158142,
      "loss": 0.6132,
      "nll_loss": 0.5785156488418579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.059326171875,
      "rewards/margins": 0.1254730224609375,
      "rewards/rejected": -0.1846923828125,
      "step": 12650
    },
    {
      "epoch": 0.9606920625284565,
      "grad_norm": 3.3976326336219236,
      "learning_rate": 7.110057847221588e-07,
      "log_odds_chosen": 1.43310546875,
      "log_odds_ratio": -0.41801756620407104,
      "logits/chosen": -1.020898461341858,
      "logits/rejected": -0.9189453125,
      "logps/chosen": -0.628125011920929,
      "logps/rejected": -1.6003906726837158,
      "loss": 0.6288,
      "nll_loss": 0.609375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06275634467601776,
      "rewards/margins": 0.09724731743335724,
      "rewards/rejected": -0.15993651747703552,
      "step": 12660
    },
    {
      "epoch": 0.9614509030201852,
      "grad_norm": 3.330384139746968,
      "learning_rate": 7.107251429963166e-07,
      "log_odds_chosen": 1.638281226158142,
      "log_odds_ratio": -0.3970703184604645,
      "logits/chosen": -1.066796898841858,
      "logits/rejected": -0.920117199420929,
      "logps/chosen": -0.6805664300918579,
      "logps/rejected": -1.8523437976837158,
      "loss": 0.6249,
      "nll_loss": 0.6075195074081421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06806640326976776,
      "rewards/margins": 0.11710815131664276,
      "rewards/rejected": -0.18503418564796448,
      "step": 12670
    },
    {
      "epoch": 0.9622097435119138,
      "grad_norm": 3.4897907824927974,
      "learning_rate": 7.104448333253878e-07,
      "log_odds_chosen": 1.691992163658142,
      "log_odds_ratio": -0.359375,
      "logits/chosen": -0.997265636920929,
      "logits/rejected": -0.873828113079071,
      "logps/chosen": -0.618945300579071,
      "logps/rejected": -1.778710961341858,
      "loss": 0.6254,
      "nll_loss": 0.610156238079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06193847581744194,
      "rewards/margins": 0.11593017727136612,
      "rewards/rejected": -0.17792968451976776,
      "step": 12680
    },
    {
      "epoch": 0.9629685840036424,
      "grad_norm": 2.932276283781953,
      "learning_rate": 7.101648550550766e-07,
      "log_odds_chosen": 1.6384766101837158,
      "log_odds_ratio": -0.37604981660842896,
      "logits/chosen": -1.0031249523162842,
      "logits/rejected": -0.921093761920929,
      "logps/chosen": -0.584179699420929,
      "logps/rejected": -1.735937476158142,
      "loss": 0.6337,
      "nll_loss": 0.5811523199081421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0584716796875,
      "rewards/margins": 0.11516113579273224,
      "rewards/rejected": -0.17365722358226776,
      "step": 12690
    },
    {
      "epoch": 0.9637274244953711,
      "grad_norm": 3.219556865503694,
      "learning_rate": 7.098852075328911e-07,
      "log_odds_chosen": 1.5888671875,
      "log_odds_ratio": -0.3916992247104645,
      "logits/chosen": -1.0730469226837158,
      "logits/rejected": -0.9488281011581421,
      "logps/chosen": -0.605175793170929,
      "logps/rejected": -1.6923828125,
      "loss": 0.6532,
      "nll_loss": 0.55419921875,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06048583984375,
      "rewards/margins": 0.10877685248851776,
      "rewards/rejected": -0.16933593153953552,
      "step": 12700
    },
    {
      "epoch": 0.9644862649870997,
      "grad_norm": 3.575114214459302,
      "learning_rate": 7.096058901081364e-07,
      "log_odds_chosen": 1.5470702648162842,
      "log_odds_ratio": -0.36474609375,
      "logits/chosen": -1.0478515625,
      "logits/rejected": -0.9048827886581421,
      "logps/chosen": -0.57421875,
      "logps/rejected": -1.595703125,
      "loss": 0.6361,
      "nll_loss": 0.5521484613418579,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.05744628980755806,
      "rewards/margins": 0.10216064751148224,
      "rewards/rejected": -0.1595458984375,
      "step": 12710
    },
    {
      "epoch": 0.9652451054788284,
      "grad_norm": 3.3574876320317535,
      "learning_rate": 7.093269021319087e-07,
      "log_odds_chosen": 1.637304663658142,
      "log_odds_ratio": -0.36796873807907104,
      "logits/chosen": -1.055078148841858,
      "logits/rejected": -0.9781249761581421,
      "logps/chosen": -0.6250976324081421,
      "logps/rejected": -1.7527344226837158,
      "loss": 0.645,
      "nll_loss": 0.6136718988418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06254883110523224,
      "rewards/margins": 0.11264648288488388,
      "rewards/rejected": -0.17531737685203552,
      "step": 12720
    },
    {
      "epoch": 0.9660039459705569,
      "grad_norm": 3.312555047032111,
      "learning_rate": 7.090482429570884e-07,
      "log_odds_chosen": 1.6420409679412842,
      "log_odds_ratio": -0.3838867247104645,
      "logits/chosen": -1.0509765148162842,
      "logits/rejected": -0.9302734136581421,
      "logps/chosen": -0.612988293170929,
      "logps/rejected": -1.7755858898162842,
      "loss": 0.6524,
      "nll_loss": 0.59521484375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06131591647863388,
      "rewards/margins": 0.11627807468175888,
      "rewards/rejected": -0.17756347358226776,
      "step": 12730
    },
    {
      "epoch": 0.9667627864622856,
      "grad_norm": 3.060907308831973,
      "learning_rate": 7.087699119383339e-07,
      "log_odds_chosen": 1.7607421875,
      "log_odds_ratio": -0.35075682401657104,
      "logits/chosen": -1.051171898841858,
      "logits/rejected": -0.9488281011581421,
      "logps/chosen": -0.6083984375,
      "logps/rejected": -1.799414038658142,
      "loss": 0.6482,
      "nll_loss": 0.662548840045929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06082763522863388,
      "rewards/margins": 0.11907958984375,
      "rewards/rejected": -0.1800537109375,
      "step": 12740
    },
    {
      "epoch": 0.9675216269540142,
      "grad_norm": 4.046453781976323,
      "learning_rate": 7.084919084320762e-07,
      "log_odds_chosen": 1.7817871570587158,
      "log_odds_ratio": -0.362548828125,
      "logits/chosen": -1.0076172351837158,
      "logits/rejected": -0.926562488079071,
      "logps/chosen": -0.667773425579071,
      "logps/rejected": -1.951171875,
      "loss": 0.6263,
      "nll_loss": 0.591015636920929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06679687649011612,
      "rewards/margins": 0.12822875380516052,
      "rewards/rejected": -0.19504395127296448,
      "step": 12750
    },
    {
      "epoch": 0.9682804674457429,
      "grad_norm": 3.5875128037508732,
      "learning_rate": 7.08214231796511e-07,
      "log_odds_chosen": 1.6591796875,
      "log_odds_ratio": -0.3875488340854645,
      "logits/chosen": -1.0656249523162842,
      "logits/rejected": -0.913281261920929,
      "logps/chosen": -0.62890625,
      "logps/rejected": -1.7726562023162842,
      "loss": 0.6334,
      "nll_loss": 0.580273449420929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06291504204273224,
      "rewards/margins": 0.11444701999425888,
      "rewards/rejected": -0.17746582627296448,
      "step": 12760
    },
    {
      "epoch": 0.9690393079374715,
      "grad_norm": 3.4594420118239886,
      "learning_rate": 7.079368813915939e-07,
      "log_odds_chosen": 1.7955322265625,
      "log_odds_ratio": -0.3617919981479645,
      "logits/chosen": -0.975390613079071,
      "logits/rejected": -0.8919922113418579,
      "logps/chosen": -0.629101574420929,
      "logps/rejected": -1.9152343273162842,
      "loss": 0.6251,
      "nll_loss": 0.5972656011581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06289062649011612,
      "rewards/margins": 0.1285400390625,
      "rewards/rejected": -0.19135741889476776,
      "step": 12770
    },
    {
      "epoch": 0.9697981484292002,
      "grad_norm": 3.034686454329317,
      "learning_rate": 7.076598565790337e-07,
      "log_odds_chosen": 1.849609375,
      "log_odds_ratio": -0.35517579317092896,
      "logits/chosen": -1.022070288658142,
      "logits/rejected": -0.888476550579071,
      "logps/chosen": -0.5931640863418579,
      "logps/rejected": -1.8878905773162842,
      "loss": 0.6593,
      "nll_loss": 0.6000000238418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05930175632238388,
      "rewards/margins": 0.12947997450828552,
      "rewards/rejected": -0.18881836533546448,
      "step": 12780
    },
    {
      "epoch": 0.9705569889209288,
      "grad_norm": 3.4566434305704163,
      "learning_rate": 7.073831567222859e-07,
      "log_odds_chosen": 1.5056641101837158,
      "log_odds_ratio": -0.4079833924770355,
      "logits/chosen": -1.022070288658142,
      "logits/rejected": -0.8890625238418579,
      "logps/chosen": -0.603320300579071,
      "logps/rejected": -1.660546898841858,
      "loss": 0.6455,
      "nll_loss": 0.6016601324081421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06033935397863388,
      "rewards/margins": 0.10577392578125,
      "rewards/rejected": -0.16606445610523224,
      "step": 12790
    },
    {
      "epoch": 0.9713158294126575,
      "grad_norm": 3.4928388276263362,
      "learning_rate": 7.071067811865475e-07,
      "log_odds_chosen": 1.5998046398162842,
      "log_odds_ratio": -0.37187498807907104,
      "logits/chosen": -1.058984398841858,
      "logits/rejected": -0.9351562261581421,
      "logps/chosen": -0.631640613079071,
      "logps/rejected": -1.730859398841858,
      "loss": 0.6459,
      "nll_loss": 0.600781261920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06315918266773224,
      "rewards/margins": 0.11007080227136612,
      "rewards/rejected": -0.1732177734375,
      "step": 12800
    },
    {
      "epoch": 0.9720746699043861,
      "grad_norm": 3.3917707572539384,
      "learning_rate": 7.068307293387497e-07,
      "log_odds_chosen": 1.456445336341858,
      "log_odds_ratio": -0.3985839784145355,
      "logits/chosen": -0.9765625,
      "logits/rejected": -0.889453113079071,
      "logps/chosen": -0.6226562261581421,
      "logps/rejected": -1.583984375,
      "loss": 0.6239,
      "nll_loss": 0.652539074420929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.062255859375,
      "rewards/margins": 0.09609375149011612,
      "rewards/rejected": -0.15839843451976776,
      "step": 12810
    },
    {
      "epoch": 0.9728335103961148,
      "grad_norm": 2.986133463304473,
      "learning_rate": 7.065550005475526e-07,
      "log_odds_chosen": 1.383886694908142,
      "log_odds_ratio": -0.4205078184604645,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.9027343988418579,
      "logps/chosen": -0.6148437261581421,
      "logps/rejected": -1.529882788658142,
      "loss": 0.6238,
      "nll_loss": 0.597949206829071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06143798679113388,
      "rewards/margins": 0.09144286811351776,
      "rewards/rejected": -0.15300293266773224,
      "step": 12820
    },
    {
      "epoch": 0.9735923508878433,
      "grad_norm": 3.3935372940810926,
      "learning_rate": 7.062795941833388e-07,
      "log_odds_chosen": 1.7761719226837158,
      "log_odds_ratio": -0.3666015565395355,
      "logits/chosen": -1.0333983898162842,
      "logits/rejected": -0.9154297113418579,
      "logps/chosen": -0.578906238079071,
      "logps/rejected": -1.8390624523162842,
      "loss": 0.6112,
      "nll_loss": 0.6499999761581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05789794772863388,
      "rewards/margins": 0.12608642876148224,
      "rewards/rejected": -0.18391112983226776,
      "step": 12830
    },
    {
      "epoch": 0.974351191379572,
      "grad_norm": 3.62030884081484,
      "learning_rate": 7.060045096182077e-07,
      "log_odds_chosen": 2.1451172828674316,
      "log_odds_ratio": -0.3006835877895355,
      "logits/chosen": -1.030664086341858,
      "logits/rejected": -0.9140625,
      "logps/chosen": -0.560546875,
      "logps/rejected": -2.0699219703674316,
      "loss": 0.6237,
      "nll_loss": 0.5516601800918579,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.05604248121380806,
      "rewards/margins": 0.15092773735523224,
      "rewards/rejected": -0.20698241889476776,
      "step": 12840
    },
    {
      "epoch": 0.9751100318713006,
      "grad_norm": 3.630275234867889,
      "learning_rate": 7.057297462259693e-07,
      "log_odds_chosen": 1.8611328601837158,
      "log_odds_ratio": -0.30903321504592896,
      "logits/chosen": -1.035742163658142,
      "logits/rejected": -0.8843749761581421,
      "logps/chosen": -0.567187488079071,
      "logps/rejected": -1.828125,
      "loss": 0.6252,
      "nll_loss": 0.597949206829071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05670166015625,
      "rewards/margins": 0.12598876655101776,
      "rewards/rejected": -0.18276366591453552,
      "step": 12850
    },
    {
      "epoch": 0.9758688723630293,
      "grad_norm": 3.222582164881772,
      "learning_rate": 7.05455303382138e-07,
      "log_odds_chosen": 1.5167968273162842,
      "log_odds_ratio": -0.3951171934604645,
      "logits/chosen": -1.039453148841858,
      "logits/rejected": -0.890429675579071,
      "logps/chosen": -0.5840820074081421,
      "logps/rejected": -1.568750023841858,
      "loss": 0.6308,
      "nll_loss": 0.5855468511581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.05838622897863388,
      "rewards/margins": 0.09852294623851776,
      "rewards/rejected": -0.15690918266773224,
      "step": 12860
    },
    {
      "epoch": 0.9766277128547579,
      "grad_norm": 3.29232142208366,
      "learning_rate": 7.051811804639268e-07,
      "log_odds_chosen": 1.406640648841858,
      "log_odds_ratio": -0.35649412870407104,
      "logits/chosen": -1.034570336341858,
      "logits/rejected": -0.920703113079071,
      "logps/chosen": -0.60498046875,
      "logps/rejected": -1.515234351158142,
      "loss": 0.6249,
      "nll_loss": 0.5967773199081421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06049804762005806,
      "rewards/margins": 0.09101562201976776,
      "rewards/rejected": -0.15158692002296448,
      "step": 12870
    },
    {
      "epoch": 0.9773865533464866,
      "grad_norm": 3.174326368632193,
      "learning_rate": 7.049073768502414e-07,
      "log_odds_chosen": 1.55078125,
      "log_odds_ratio": -0.3504882752895355,
      "logits/chosen": -1.030664086341858,
      "logits/rejected": -0.929492175579071,
      "logps/chosen": -0.619824230670929,
      "logps/rejected": -1.6691405773162842,
      "loss": 0.6252,
      "nll_loss": 0.556640625,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06197509914636612,
      "rewards/margins": 0.10479736328125,
      "rewards/rejected": -0.16660156846046448,
      "step": 12880
    },
    {
      "epoch": 0.9781453938382152,
      "grad_norm": 3.820521169030311,
      "learning_rate": 7.046338919216742e-07,
      "log_odds_chosen": 1.7875487804412842,
      "log_odds_ratio": -0.3712402284145355,
      "logits/chosen": -1.017578125,
      "logits/rejected": -0.915820300579071,
      "logps/chosen": -0.614453136920929,
      "logps/rejected": -1.8503906726837158,
      "loss": 0.63,
      "nll_loss": 0.611328125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06146240234375,
      "rewards/margins": 0.12366943061351776,
      "rewards/rejected": -0.18498535454273224,
      "step": 12890
    },
    {
      "epoch": 0.9789042343299439,
      "grad_norm": 2.6757624111378435,
      "learning_rate": 7.04360725060499e-07,
      "log_odds_chosen": 1.685693383216858,
      "log_odds_ratio": -0.39165037870407104,
      "logits/chosen": -1.0398437976837158,
      "logits/rejected": -0.918749988079071,
      "logps/chosen": -0.6107422113418579,
      "logps/rejected": -1.774804711341858,
      "loss": 0.6241,
      "nll_loss": 0.6080077886581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06109619140625,
      "rewards/margins": 0.11649169772863388,
      "rewards/rejected": -0.1776123046875,
      "step": 12900
    },
    {
      "epoch": 0.9796630748216725,
      "grad_norm": 3.4979767642612245,
      "learning_rate": 7.040878756506639e-07,
      "log_odds_chosen": 1.7971680164337158,
      "log_odds_ratio": -0.37060546875,
      "logits/chosen": -1.004492163658142,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.5665038824081421,
      "logps/rejected": -1.7763671875,
      "loss": 0.616,
      "nll_loss": 0.581347644329071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05662841722369194,
      "rewards/margins": 0.12106933444738388,
      "rewards/rejected": -0.17766113579273224,
      "step": 12910
    },
    {
      "epoch": 0.9804219153134012,
      "grad_norm": 3.514718202615962,
      "learning_rate": 7.038153430777867e-07,
      "log_odds_chosen": 1.7880859375,
      "log_odds_ratio": -0.35624998807907104,
      "logits/chosen": -0.9964843988418579,
      "logits/rejected": -0.9007812738418579,
      "logps/chosen": -0.628710925579071,
      "logps/rejected": -1.898046851158142,
      "loss": 0.6642,
      "nll_loss": 0.616992175579071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06290283054113388,
      "rewards/margins": 0.12700195610523224,
      "rewards/rejected": -0.18984374403953552,
      "step": 12920
    },
    {
      "epoch": 0.9811807558051298,
      "grad_norm": 2.7280370781919223,
      "learning_rate": 7.035431267291484e-07,
      "log_odds_chosen": 1.6765625476837158,
      "log_odds_ratio": -0.355712890625,
      "logits/chosen": -1.0744140148162842,
      "logits/rejected": -0.91015625,
      "logps/chosen": -0.668749988079071,
      "logps/rejected": -1.8523437976837158,
      "loss": 0.6469,
      "nll_loss": 0.6620117425918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06694336235523224,
      "rewards/margins": 0.1182861328125,
      "rewards/rejected": -0.18522949516773224,
      "step": 12930
    },
    {
      "epoch": 0.9819395962968585,
      "grad_norm": 2.6313667054090657,
      "learning_rate": 7.032712259936877e-07,
      "log_odds_chosen": 1.470361351966858,
      "log_odds_ratio": -0.37910157442092896,
      "logits/chosen": -1.0505859851837158,
      "logits/rejected": -0.9576171636581421,
      "logps/chosen": -0.6532226800918579,
      "logps/rejected": -1.615625023841858,
      "loss": 0.6199,
      "nll_loss": 0.6015625,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06534423679113388,
      "rewards/margins": 0.09639587253332138,
      "rewards/rejected": -0.16166992485523224,
      "step": 12940
    },
    {
      "epoch": 0.982698436788587,
      "grad_norm": 3.268672696324829,
      "learning_rate": 7.029996402619949e-07,
      "log_odds_chosen": 1.6294434070587158,
      "log_odds_ratio": -0.36479490995407104,
      "logits/chosen": -1.031835913658142,
      "logits/rejected": -0.9107421636581421,
      "logps/chosen": -0.604785144329071,
      "logps/rejected": -1.6843750476837158,
      "loss": 0.6197,
      "nll_loss": 0.549853503704071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06047363206744194,
      "rewards/margins": 0.10810241848230362,
      "rewards/rejected": -0.16860350966453552,
      "step": 12950
    },
    {
      "epoch": 0.9834572772803156,
      "grad_norm": 3.681391253393462,
      "learning_rate": 7.027283689263065e-07,
      "log_odds_chosen": 1.6418945789337158,
      "log_odds_ratio": -0.3367675840854645,
      "logits/chosen": -1.036523461341858,
      "logits/rejected": -0.903124988079071,
      "logps/chosen": -0.551562488079071,
      "logps/rejected": -1.642578125,
      "loss": 0.6194,
      "nll_loss": 0.540332019329071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05516357347369194,
      "rewards/margins": 0.10923461616039276,
      "rewards/rejected": -0.16437987983226776,
      "step": 12960
    },
    {
      "epoch": 0.9842161177720443,
      "grad_norm": 4.419637947488852,
      "learning_rate": 7.024574113804996e-07,
      "log_odds_chosen": 1.9646484851837158,
      "log_odds_ratio": -0.2902587950229645,
      "logits/chosen": -1.0701172351837158,
      "logits/rejected": -0.9623047113418579,
      "logps/chosen": -0.545605480670929,
      "logps/rejected": -1.8640625476837158,
      "loss": 0.6345,
      "nll_loss": 0.538769543170929,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.05452881008386612,
      "rewards/margins": 0.13179931044578552,
      "rewards/rejected": -0.18618163466453552,
      "step": 12970
    },
    {
      "epoch": 0.9849749582637729,
      "grad_norm": 3.322057394879621,
      "learning_rate": 7.021867670200857e-07,
      "log_odds_chosen": 1.814355492591858,
      "log_odds_ratio": -0.3724365234375,
      "logits/chosen": -1.028906226158142,
      "logits/rejected": -0.923046886920929,
      "logps/chosen": -0.6904296875,
      "logps/rejected": -1.9656250476837158,
      "loss": 0.6516,
      "nll_loss": 0.669238269329071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06898193061351776,
      "rewards/margins": 0.127410888671875,
      "rewards/rejected": -0.19658203423023224,
      "step": 12980
    },
    {
      "epoch": 0.9857337987555016,
      "grad_norm": 3.720809647983929,
      "learning_rate": 7.019164352422057e-07,
      "log_odds_chosen": 1.9021484851837158,
      "log_odds_ratio": -0.3316406309604645,
      "logits/chosen": -1.0167968273162842,
      "logits/rejected": -0.8978515863418579,
      "logps/chosen": -0.611035168170929,
      "logps/rejected": -1.944921851158142,
      "loss": 0.6208,
      "nll_loss": 0.584179699420929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06112060695886612,
      "rewards/margins": 0.13327637314796448,
      "rewards/rejected": -0.19443359971046448,
      "step": 12990
    },
    {
      "epoch": 0.9864926392472302,
      "grad_norm": 3.3484972531023782,
      "learning_rate": 7.016464154456234e-07,
      "log_odds_chosen": 1.789648413658142,
      "log_odds_ratio": -0.3425048887729645,
      "logits/chosen": -1.0537109375,
      "logits/rejected": -0.916796863079071,
      "logps/chosen": -0.6241210699081421,
      "logps/rejected": -1.919921875,
      "loss": 0.6349,
      "nll_loss": 0.6309570074081421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06241454929113388,
      "rewards/margins": 0.12967529892921448,
      "rewards/rejected": -0.19211426377296448,
      "step": 13000
    },
    {
      "epoch": 0.9872514797389589,
      "grad_norm": 3.303160604548869,
      "learning_rate": 7.013767070307207e-07,
      "log_odds_chosen": 1.567480444908142,
      "log_odds_ratio": -0.357421875,
      "logits/chosen": -0.9927734136581421,
      "logits/rejected": -0.873828113079071,
      "logps/chosen": -0.623828113079071,
      "logps/rejected": -1.6541016101837158,
      "loss": 0.613,
      "nll_loss": 0.5560547113418579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06232910230755806,
      "rewards/margins": 0.10316924750804901,
      "rewards/rejected": -0.16550293564796448,
      "step": 13010
    },
    {
      "epoch": 0.9880103202306875,
      "grad_norm": 2.835851509546799,
      "learning_rate": 7.011073093994919e-07,
      "log_odds_chosen": 1.4097168445587158,
      "log_odds_ratio": -0.4185546934604645,
      "logits/chosen": -1.014062523841858,
      "logits/rejected": -0.913281261920929,
      "logps/chosen": -0.671875,
      "logps/rejected": -1.614648461341858,
      "loss": 0.6459,
      "nll_loss": 0.6396484375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06719970703125,
      "rewards/margins": 0.09435119479894638,
      "rewards/rejected": -0.16142578423023224,
      "step": 13020
    },
    {
      "epoch": 0.9887691607224162,
      "grad_norm": 3.298303870961653,
      "learning_rate": 7.008382219555372e-07,
      "log_odds_chosen": 1.715429663658142,
      "log_odds_ratio": -0.34321290254592896,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.9404296875,
      "logps/chosen": -0.5772460699081421,
      "logps/rejected": -1.7578125,
      "loss": 0.6275,
      "nll_loss": 0.562207043170929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05772705003619194,
      "rewards/margins": 0.11793212592601776,
      "rewards/rejected": -0.17575684189796448,
      "step": 13030
    },
    {
      "epoch": 0.9895280012141447,
      "grad_norm": 3.1250931986858244,
      "learning_rate": 7.005694441040588e-07,
      "log_odds_chosen": 1.50341796875,
      "log_odds_ratio": -0.36894530057907104,
      "logits/chosen": -1.01171875,
      "logits/rejected": -0.9437500238418579,
      "logps/chosen": -0.6527343988418579,
      "logps/rejected": -1.652734398841858,
      "loss": 0.6217,
      "nll_loss": 0.5606445074081421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06523437798023224,
      "rewards/margins": 0.10012207180261612,
      "rewards/rejected": -0.16542968153953552,
      "step": 13040
    },
    {
      "epoch": 0.9902868417058734,
      "grad_norm": 3.054884737922653,
      "learning_rate": 7.003009752518536e-07,
      "log_odds_chosen": 1.65625,
      "log_odds_ratio": -0.3519043028354645,
      "logits/chosen": -1.0693359375,
      "logits/rejected": -0.9462890625,
      "logps/chosen": -0.5951172113418579,
      "logps/rejected": -1.7332031726837158,
      "loss": 0.6309,
      "nll_loss": 0.6143554449081421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.05950927734375,
      "rewards/margins": 0.11381836235523224,
      "rewards/rejected": -0.17336425185203552,
      "step": 13050
    },
    {
      "epoch": 0.991045682197602,
      "grad_norm": 2.712613708969208,
      "learning_rate": 7.000328148073091e-07,
      "log_odds_chosen": 1.4972655773162842,
      "log_odds_ratio": -0.3872314393520355,
      "logits/chosen": -1.055273413658142,
      "logits/rejected": -0.941601574420929,
      "logps/chosen": -0.6236327886581421,
      "logps/rejected": -1.623046875,
      "loss": 0.6188,
      "nll_loss": 0.5624023675918579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0623779296875,
      "rewards/margins": 0.09989318996667862,
      "rewards/rejected": -0.16237792372703552,
      "step": 13060
    },
    {
      "epoch": 0.9918045226893307,
      "grad_norm": 3.5714710422754203,
      "learning_rate": 6.997649621803973e-07,
      "log_odds_chosen": 1.694921851158142,
      "log_odds_ratio": -0.37993162870407104,
      "logits/chosen": -1.058984398841858,
      "logits/rejected": -0.958984375,
      "logps/chosen": -0.629199206829071,
      "logps/rejected": -1.7947266101837158,
      "loss": 0.6212,
      "nll_loss": 0.56298828125,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06287841498851776,
      "rewards/margins": 0.11655273288488388,
      "rewards/rejected": -0.17939452826976776,
      "step": 13070
    },
    {
      "epoch": 0.9925633631810593,
      "grad_norm": 3.4679366483493426,
      "learning_rate": 6.994974167826689e-07,
      "log_odds_chosen": 1.654687523841858,
      "log_odds_ratio": -0.3541503846645355,
      "logits/chosen": -1.0283203125,
      "logits/rejected": -0.909960925579071,
      "logps/chosen": -0.59375,
      "logps/rejected": -1.7136719226837158,
      "loss": 0.5992,
      "nll_loss": 0.5967773199081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05935058742761612,
      "rewards/margins": 0.11179199069738388,
      "rewards/rejected": -0.17128905653953552,
      "step": 13080
    },
    {
      "epoch": 0.993322203672788,
      "grad_norm": 3.735491275957325,
      "learning_rate": 6.99230178027249e-07,
      "log_odds_chosen": 1.818750023841858,
      "log_odds_ratio": -0.31303709745407104,
      "logits/chosen": -1.0486328601837158,
      "logits/rejected": -0.916210949420929,
      "logps/chosen": -0.58642578125,
      "logps/rejected": -1.801171898841858,
      "loss": 0.6359,
      "nll_loss": 0.5850585699081421,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.05867920070886612,
      "rewards/margins": 0.121490478515625,
      "rewards/rejected": -0.18010254204273224,
      "step": 13090
    },
    {
      "epoch": 0.9940810441645166,
      "grad_norm": 2.7480392621030076,
      "learning_rate": 6.989632453288303e-07,
      "log_odds_chosen": 1.792382836341858,
      "log_odds_ratio": -0.3248535096645355,
      "logits/chosen": -1.0363280773162842,
      "logits/rejected": -0.8902343511581421,
      "logps/chosen": -0.5838867425918579,
      "logps/rejected": -1.7839844226837158,
      "loss": 0.6259,
      "nll_loss": 0.585253894329071,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.05838622897863388,
      "rewards/margins": 0.12003173679113388,
      "rewards/rejected": -0.17839355766773224,
      "step": 13100
    },
    {
      "epoch": 0.9948398846562453,
      "grad_norm": 5.315338921131002,
      "learning_rate": 6.98696618103669e-07,
      "log_odds_chosen": 1.841796875,
      "log_odds_ratio": -0.32097166776657104,
      "logits/chosen": -1.0498046875,
      "logits/rejected": -0.919140636920929,
      "logps/chosen": -0.5809570550918579,
      "logps/rejected": -1.8525390625,
      "loss": 0.6258,
      "nll_loss": 0.600390613079071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05806884914636612,
      "rewards/margins": 0.12735596299171448,
      "rewards/rejected": -0.18535156548023224,
      "step": 13110
    },
    {
      "epoch": 0.9955987251479739,
      "grad_norm": 3.0285925402700675,
      "learning_rate": 6.984302957695782e-07,
      "log_odds_chosen": 1.8556640148162842,
      "log_odds_ratio": -0.33942872285842896,
      "logits/chosen": -1.048242211341858,
      "logits/rejected": -0.921875,
      "logps/chosen": -0.6363281011581421,
      "logps/rejected": -1.9675781726837158,
      "loss": 0.618,
      "nll_loss": 0.60693359375,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06368408352136612,
      "rewards/margins": 0.13309326767921448,
      "rewards/rejected": -0.19670410454273224,
      "step": 13120
    },
    {
      "epoch": 0.9963575656397026,
      "grad_norm": 2.8953548933062208,
      "learning_rate": 6.981642777459237e-07,
      "log_odds_chosen": 1.6671874523162842,
      "log_odds_ratio": -0.38591307401657104,
      "logits/chosen": -1.01953125,
      "logits/rejected": -0.916210949420929,
      "logps/chosen": -0.626660168170929,
      "logps/rejected": -1.7316405773162842,
      "loss": 0.6213,
      "nll_loss": 0.54833984375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0626220703125,
      "rewards/margins": 0.11055908352136612,
      "rewards/rejected": -0.173095703125,
      "step": 13130
    },
    {
      "epoch": 0.9971164061314312,
      "grad_norm": 3.177774014945304,
      "learning_rate": 6.978985634536182e-07,
      "log_odds_chosen": 1.3240234851837158,
      "log_odds_ratio": -0.4302734434604645,
      "logits/chosen": -1.0750000476837158,
      "logits/rejected": -0.9458984136581421,
      "logps/chosen": -0.705859363079071,
      "logps/rejected": -1.640234351158142,
      "loss": 0.6217,
      "nll_loss": 0.66357421875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07062987983226776,
      "rewards/margins": 0.09353027492761612,
      "rewards/rejected": -0.16413573920726776,
      "step": 13140
    },
    {
      "epoch": 0.9978752466231599,
      "grad_norm": 3.4225198631548306,
      "learning_rate": 6.976331523151157e-07,
      "log_odds_chosen": 1.6640625,
      "log_odds_ratio": -0.3455566465854645,
      "logits/chosen": -1.0119140148162842,
      "logits/rejected": -0.9013671875,
      "logps/chosen": -0.607617199420929,
      "logps/rejected": -1.762109398841858,
      "loss": 0.6317,
      "nll_loss": 0.581738293170929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06080322340130806,
      "rewards/margins": 0.11546631157398224,
      "rewards/rejected": -0.17624512314796448,
      "step": 13150
    },
    {
      "epoch": 0.9986340871148884,
      "grad_norm": 3.1568635113445698,
      "learning_rate": 6.973680437544066e-07,
      "log_odds_chosen": 1.628515601158142,
      "log_odds_ratio": -0.3516601622104645,
      "logits/chosen": -1.0027344226837158,
      "logits/rejected": -0.875195324420929,
      "logps/chosen": -0.6075195074081421,
      "logps/rejected": -1.7000000476837158,
      "loss": 0.616,
      "nll_loss": 0.612011730670929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06074218824505806,
      "rewards/margins": 0.1092529296875,
      "rewards/rejected": -0.17011718451976776,
      "step": 13160
    },
    {
      "epoch": 0.9993929276066171,
      "grad_norm": 3.1479439895484593,
      "learning_rate": 6.971032371970126e-07,
      "log_odds_chosen": 1.873046875,
      "log_odds_ratio": -0.32988280057907104,
      "logits/chosen": -1.062890648841858,
      "logits/rejected": -0.9652343988418579,
      "logps/chosen": -0.6163085699081421,
      "logps/rejected": -1.875390648841858,
      "loss": 0.633,
      "nll_loss": 0.6041015386581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06163330003619194,
      "rewards/margins": 0.12604980170726776,
      "rewards/rejected": -0.18784180283546448,
      "step": 13170
    },
    {
      "epoch": 1.0,
      "step": 13178,
      "total_flos": 0.0,
      "train_loss": 0.0,
      "train_runtime": 21.0695,
      "train_samples_per_second": 80055.918,
      "train_steps_per_second": 625.455
    }
  ],
  "logging_steps": 10,
  "max_steps": 13178,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 5000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}