{
  "best_metric": 1.1802747249603271,
  "best_model_checkpoint": "models/llama3.2-3b-orpo-coarse/checkpoint-5000",
  "epoch": 0.3794202458643193,
  "eval_steps": 5000,
  "global_step": 5000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0007588404917286386,
      "grad_norm": 16.28905678970287,
      "learning_rate": 8e-07,
      "log_odds_chosen": 0.6555420160293579,
      "log_odds_ratio": -0.68408203125,
      "logits/chosen": -1.0910155773162842,
      "logits/rejected": -1.1103515625,
      "logps/chosen": -2.21875,
      "logps/rejected": -2.825000047683716,
      "loss": 2.518,
      "nll_loss": 2.332812547683716,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.22182616591453552,
      "rewards/margins": 0.060699462890625,
      "rewards/rejected": -0.28266602754592896,
      "step": 10
    },
    {
      "epoch": 0.0015176809834572772,
      "grad_norm": 8.977044251578562,
      "learning_rate": 1.6e-06,
      "log_odds_chosen": 0.5552002191543579,
      "log_odds_ratio": -0.7049804925918579,
      "logits/chosen": -1.138085961341858,
      "logits/rejected": -1.131445288658142,
      "logps/chosen": -2.3125,
      "logps/rejected": -2.8218750953674316,
      "loss": 2.3322,
      "nll_loss": 2.2734375,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.23154297471046448,
      "rewards/margins": 0.0509796142578125,
      "rewards/rejected": -0.28271484375,
      "step": 20
    },
    {
      "epoch": 0.002276521475185916,
      "grad_norm": 3.4584431033537384,
      "learning_rate": 2.4e-06,
      "log_odds_chosen": 0.6031738519668579,
      "log_odds_ratio": -0.614941418170929,
      "logits/chosen": -1.235742211341858,
      "logits/rejected": -1.194726586341858,
      "logps/chosen": -1.784765601158142,
      "logps/rejected": -2.30078125,
      "loss": 1.9747,
      "nll_loss": 1.8878905773162842,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.178466796875,
      "rewards/margins": 0.05157470703125,
      "rewards/rejected": -0.22998046875,
      "step": 30
    },
    {
      "epoch": 0.0030353619669145544,
      "grad_norm": 2.779230061008798,
      "learning_rate": 3.2e-06,
      "log_odds_chosen": 0.37614136934280396,
      "log_odds_ratio": -0.7298828363418579,
      "logits/chosen": -1.0929687023162842,
      "logits/rejected": -1.0183594226837158,
      "logps/chosen": -1.552734375,
      "logps/rejected": -1.8781249523162842,
      "loss": 1.795,
      "nll_loss": 1.745703101158142,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15510253608226776,
      "rewards/margins": 0.03264465183019638,
      "rewards/rejected": -0.187744140625,
      "step": 40
    },
    {
      "epoch": 0.0037942024586431933,
      "grad_norm": 2.5592134050477653,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.559399425983429,
      "log_odds_ratio": -0.5757812261581421,
      "logits/chosen": -0.9652343988418579,
      "logits/rejected": -0.919726550579071,
      "logps/chosen": -1.413671851158142,
      "logps/rejected": -1.8718750476837158,
      "loss": 1.7396,
      "nll_loss": 1.596093773841858,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.141357421875,
      "rewards/margins": 0.04580993577837944,
      "rewards/rejected": -0.18730469048023224,
      "step": 50
    },
    {
      "epoch": 0.004553042950371832,
      "grad_norm": 2.8450488945476566,
      "learning_rate": 4.8e-06,
      "log_odds_chosen": 0.5027710199356079,
      "log_odds_ratio": -0.5972656011581421,
      "logits/chosen": -1.0324218273162842,
      "logits/rejected": -0.975390613079071,
      "logps/chosen": -1.277929663658142,
      "logps/rejected": -1.692968726158142,
      "loss": 1.6585,
      "nll_loss": 1.4763672351837158,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.12788085639476776,
      "rewards/margins": 0.04137573391199112,
      "rewards/rejected": -0.16914062201976776,
      "step": 60
    },
    {
      "epoch": 0.005311883442100471,
      "grad_norm": 2.4981511166293764,
      "learning_rate": 5.6e-06,
      "log_odds_chosen": 0.5220092535018921,
      "log_odds_ratio": -0.5845702886581421,
      "logits/chosen": -1.106054663658142,
      "logits/rejected": -1.019140601158142,
      "logps/chosen": -1.224218726158142,
      "logps/rejected": -1.671484351158142,
      "loss": 1.6213,
      "nll_loss": 1.501562476158142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.12246093899011612,
      "rewards/margins": 0.0447998046875,
      "rewards/rejected": -0.167236328125,
      "step": 70
    },
    {
      "epoch": 0.006070723933829109,
      "grad_norm": 2.6645809181218567,
      "learning_rate": 6.4e-06,
      "log_odds_chosen": 0.555920422077179,
      "log_odds_ratio": -0.568652331829071,
      "logits/chosen": -1.048437476158142,
      "logits/rejected": -0.991992175579071,
      "logps/chosen": -1.229882836341858,
      "logps/rejected": -1.6925780773162842,
      "loss": 1.5998,
      "nll_loss": 1.505273461341858,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.12290038913488388,
      "rewards/margins": 0.04631652683019638,
      "rewards/rejected": -0.16923828423023224,
      "step": 80
    },
    {
      "epoch": 0.006829564425557748,
      "grad_norm": 2.467324514034339,
      "learning_rate": 7.2e-06,
      "log_odds_chosen": 0.3549560606479645,
      "log_odds_ratio": -0.6615234613418579,
      "logits/chosen": -1.115625023841858,
      "logits/rejected": -1.0144531726837158,
      "logps/chosen": -1.228906273841858,
      "logps/rejected": -1.5417969226837158,
      "loss": 1.51,
      "nll_loss": 1.5164062976837158,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.1229248046875,
      "rewards/margins": 0.031341552734375,
      "rewards/rejected": -0.154296875,
      "step": 90
    },
    {
      "epoch": 0.007588404917286387,
      "grad_norm": 2.47471355596972,
      "learning_rate": 8e-06,
      "log_odds_chosen": 0.39606934785842896,
      "log_odds_ratio": -0.6527343988418579,
      "logits/chosen": -1.009179711341858,
      "logits/rejected": -0.9609375,
      "logps/chosen": -1.2267577648162842,
      "logps/rejected": -1.552734375,
      "loss": 1.5495,
      "nll_loss": 1.5304687023162842,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.12258300930261612,
      "rewards/margins": 0.03266448900103569,
      "rewards/rejected": -0.15522460639476776,
      "step": 100
    },
    {
      "epoch": 0.008347245409015025,
      "grad_norm": 2.02338598823769,
      "learning_rate": 7.627700713964738e-06,
      "log_odds_chosen": 0.5322631597518921,
      "log_odds_ratio": -0.566601574420929,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.888867199420929,
      "logps/chosen": -1.1130859851837158,
      "logps/rejected": -1.5285155773162842,
      "loss": 1.4784,
      "nll_loss": 1.435937523841858,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11125488579273224,
      "rewards/margins": 0.04149017482995987,
      "rewards/rejected": -0.15275879204273224,
      "step": 110
    },
    {
      "epoch": 0.009106085900743664,
      "grad_norm": 1.7600136940465556,
      "learning_rate": 7.3029674334022146e-06,
      "log_odds_chosen": 0.585436999797821,
      "log_odds_ratio": -0.55029296875,
      "logits/chosen": -1.0187499523162842,
      "logits/rejected": -0.9371093511581421,
      "logps/chosen": -1.0310547351837158,
      "logps/rejected": -1.4699218273162842,
      "loss": 1.4271,
      "nll_loss": 1.293554663658142,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.10307617485523224,
      "rewards/margins": 0.0438995361328125,
      "rewards/rejected": -0.14692382514476776,
      "step": 120
    },
    {
      "epoch": 0.009864926392472302,
      "grad_norm": 1.6794517773793187,
      "learning_rate": 7.016464154456233e-06,
      "log_odds_chosen": 0.35838621854782104,
      "log_odds_ratio": -0.6317383050918579,
      "logits/chosen": -0.9859374761581421,
      "logits/rejected": -0.9013671875,
      "logps/chosen": -1.074609398841858,
      "logps/rejected": -1.357031226158142,
      "loss": 1.4294,
      "nll_loss": 1.3332030773162842,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.10749511420726776,
      "rewards/margins": 0.02816314622759819,
      "rewards/rejected": -0.1356201171875,
      "step": 130
    },
    {
      "epoch": 0.010623766884200941,
      "grad_norm": 1.515687206113085,
      "learning_rate": 6.7612340378281325e-06,
      "log_odds_chosen": 0.49846190214157104,
      "log_odds_ratio": -0.5927734375,
      "logits/chosen": -0.960742175579071,
      "logits/rejected": -0.9111328125,
      "logps/chosen": -0.993359386920929,
      "logps/rejected": -1.381445288658142,
      "loss": 1.3889,
      "nll_loss": 1.2716796398162842,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09931640326976776,
      "rewards/margins": 0.03876953199505806,
      "rewards/rejected": -0.13808593153953552,
      "step": 140
    },
    {
      "epoch": 0.01138260737592958,
      "grad_norm": 1.5746644221408093,
      "learning_rate": 6.531972647421809e-06,
      "log_odds_chosen": 0.4237060546875,
      "log_odds_ratio": -0.609375,
      "logits/chosen": -0.996874988079071,
      "logits/rejected": -0.950976550579071,
      "logps/chosen": -0.9847656488418579,
      "logps/rejected": -1.307031273841858,
      "loss": 1.3694,
      "nll_loss": 1.302343726158142,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09843750298023224,
      "rewards/margins": 0.032186128199100494,
      "rewards/rejected": -0.13068847358226776,
      "step": 150
    },
    {
      "epoch": 0.012141447867658217,
      "grad_norm": 1.7303046396071229,
      "learning_rate": 6.3245553203367575e-06,
      "log_odds_chosen": 0.4165405333042145,
      "log_odds_ratio": -0.6290527582168579,
      "logits/chosen": -1.0310547351837158,
      "logits/rejected": -0.9527343511581421,
      "logps/chosen": -1.0068359375,
      "logps/rejected": -1.3210937976837158,
      "loss": 1.3639,
      "nll_loss": 1.32421875,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.10068359225988388,
      "rewards/margins": 0.03147735446691513,
      "rewards/rejected": -0.13212890923023224,
      "step": 160
    },
    {
      "epoch": 0.012900288359386856,
      "grad_norm": 1.5892750852136275,
      "learning_rate": 6.135719910778963e-06,
      "log_odds_chosen": 0.43085938692092896,
      "log_odds_ratio": -0.638378918170929,
      "logits/chosen": -0.9833984375,
      "logits/rejected": -0.942578136920929,
      "logps/chosen": -0.980664074420929,
      "logps/rejected": -1.329687476158142,
      "loss": 1.3067,
      "nll_loss": 1.208984375,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.09804687649011612,
      "rewards/margins": 0.03497924655675888,
      "rewards/rejected": -0.13300780951976776,
      "step": 170
    },
    {
      "epoch": 0.013659128851115495,
      "grad_norm": 1.9895227129073771,
      "learning_rate": 5.962847939999439e-06,
      "log_odds_chosen": 0.4945312440395355,
      "log_odds_ratio": -0.589648425579071,
      "logits/chosen": -0.932421863079071,
      "logits/rejected": -0.873828113079071,
      "logps/chosen": -0.9701172113418579,
      "logps/rejected": -1.3488280773162842,
      "loss": 1.3382,
      "nll_loss": 1.2804687023162842,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.09707031399011612,
      "rewards/margins": 0.03778686374425888,
      "rewards/rejected": -0.13481445610523224,
      "step": 180
    },
    {
      "epoch": 0.014417969342844134,
      "grad_norm": 1.6480729869591368,
      "learning_rate": 5.803810000880094e-06,
      "log_odds_chosen": 0.4837402403354645,
      "log_odds_ratio": -0.5904296636581421,
      "logits/chosen": -0.9173828363418579,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.9761718511581421,
      "logps/rejected": -1.346289038658142,
      "loss": 1.3017,
      "nll_loss": 1.285546898841858,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.09760741889476776,
      "rewards/margins": 0.03700103610754013,
      "rewards/rejected": -0.13461914658546448,
      "step": 190
    },
    {
      "epoch": 0.015176809834572773,
      "grad_norm": 1.503264102872236,
      "learning_rate": 5.65685424949238e-06,
      "log_odds_chosen": 0.4825683534145355,
      "log_odds_ratio": -0.582226574420929,
      "logits/chosen": -0.9466797113418579,
      "logits/rejected": -0.870898425579071,
      "logps/chosen": -0.939648449420929,
      "logps/rejected": -1.3035156726837158,
      "loss": 1.2894,
      "nll_loss": 1.217187523841858,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.09396972507238388,
      "rewards/margins": 0.036403656005859375,
      "rewards/rejected": -0.13041992485523224,
      "step": 200
    },
    {
      "epoch": 0.015935650326301412,
      "grad_norm": 1.6703995784714376,
      "learning_rate": 5.5205244747388325e-06,
      "log_odds_chosen": 0.6786133050918579,
      "log_odds_ratio": -0.579882800579071,
      "logits/chosen": -0.8832031488418579,
      "logits/rejected": -0.862109363079071,
      "logps/chosen": -0.9300781488418579,
      "logps/rejected": -1.4582030773162842,
      "loss": 1.2816,
      "nll_loss": 1.2117187976837158,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.09296874701976776,
      "rewards/margins": 0.05272827297449112,
      "rewards/rejected": -0.14572754502296448,
      "step": 210
    },
    {
      "epoch": 0.01669449081803005,
      "grad_norm": 1.5169765274707603,
      "learning_rate": 5.393598899705936e-06,
      "log_odds_chosen": 0.517822265625,
      "log_odds_ratio": -0.616503894329071,
      "logits/chosen": -0.9017578363418579,
      "logits/rejected": -0.8421875238418579,
      "logps/chosen": -0.894335925579071,
      "logps/rejected": -1.3212890625,
      "loss": 1.2526,
      "nll_loss": 1.177148461341858,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.08945312350988388,
      "rewards/margins": 0.04265594482421875,
      "rewards/rejected": -0.13203124701976776,
      "step": 220
    },
    {
      "epoch": 0.01745333130975869,
      "grad_norm": 1.4368746262060774,
      "learning_rate": 5.275043787166296e-06,
      "log_odds_chosen": 0.4863037168979645,
      "log_odds_ratio": -0.578808605670929,
      "logits/chosen": -0.8785156011581421,
      "logits/rejected": -0.7623046636581421,
      "logps/chosen": -0.9287109375,
      "logps/rejected": -1.2966797351837158,
      "loss": 1.2595,
      "nll_loss": 1.2150390148162842,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.09282226860523224,
      "rewards/margins": 0.03685913234949112,
      "rewards/rejected": -0.129638671875,
      "step": 230
    },
    {
      "epoch": 0.018212171801487327,
      "grad_norm": 1.5611472145780005,
      "learning_rate": 5.163977794943223e-06,
      "log_odds_chosen": 0.4346069395542145,
      "log_odds_ratio": -0.597363293170929,
      "logits/chosen": -0.8232421875,
      "logits/rejected": -0.8089843988418579,
      "logps/chosen": -0.871874988079071,
      "logps/rejected": -1.18359375,
      "loss": 1.2569,
      "nll_loss": 1.143164038658142,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08718261867761612,
      "rewards/margins": 0.031207656487822533,
      "rewards/rejected": -0.11833496391773224,
      "step": 240
    },
    {
      "epoch": 0.018971012293215964,
      "grad_norm": 1.460241600287341,
      "learning_rate": 5.059644256269407e-06,
      "log_odds_chosen": 0.40449219942092896,
      "log_odds_ratio": -0.616015613079071,
      "logits/chosen": -0.8539062738418579,
      "logits/rejected": -0.804492175579071,
      "logps/chosen": -0.8667968511581421,
      "logps/rejected": -1.163476586341858,
      "loss": 1.2829,
      "nll_loss": 1.1749999523162842,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08671875298023224,
      "rewards/margins": 0.02962798997759819,
      "rewards/rejected": -0.11635742336511612,
      "step": 250
    },
    {
      "epoch": 0.019729852784944605,
      "grad_norm": 1.6013195827393891,
      "learning_rate": 4.961389383568338e-06,
      "log_odds_chosen": 0.56024169921875,
      "log_odds_ratio": -0.573925793170929,
      "logits/chosen": -0.863085925579071,
      "logits/rejected": -0.7964843511581421,
      "logps/chosen": -0.834765613079071,
      "logps/rejected": -1.236328125,
      "loss": 1.2298,
      "nll_loss": 1.2373046875,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.08352050930261612,
      "rewards/margins": 0.04019775241613388,
      "rewards/rejected": -0.12368164211511612,
      "step": 260
    },
    {
      "epoch": 0.020488693276673242,
      "grad_norm": 1.5935105244091246,
      "learning_rate": 4.8686449556014755e-06,
      "log_odds_chosen": 0.693359375,
      "log_odds_ratio": -0.531054675579071,
      "logits/chosen": -0.857617199420929,
      "logits/rejected": -0.811718761920929,
      "logps/chosen": -0.777148425579071,
      "logps/rejected": -1.2861328125,
      "loss": 1.2069,
      "nll_loss": 1.126562476158142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07766113430261612,
      "rewards/margins": 0.05088958889245987,
      "rewards/rejected": -0.12858887016773224,
      "step": 270
    },
    {
      "epoch": 0.021247533768401883,
      "grad_norm": 1.3821618104135565,
      "learning_rate": 4.780914437337574e-06,
      "log_odds_chosen": 0.5213378667831421,
      "log_odds_ratio": -0.592480480670929,
      "logits/chosen": -0.8648437261581421,
      "logits/rejected": -0.8041015863418579,
      "logps/chosen": -0.811718761920929,
      "logps/rejected": -1.173437476158142,
      "loss": 1.2352,
      "nll_loss": 1.216406226158142,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.08118896186351776,
      "rewards/margins": 0.03626251220703125,
      "rewards/rejected": -0.11740722507238388,
      "step": 280
    },
    {
      "epoch": 0.02200637426013052,
      "grad_norm": 1.6247124157263477,
      "learning_rate": 4.697761756117627e-06,
      "log_odds_chosen": 0.590527355670929,
      "log_odds_ratio": -0.5450195074081421,
      "logits/chosen": -0.862109363079071,
      "logits/rejected": -0.7880859375,
      "logps/chosen": -0.7593749761581421,
      "logps/rejected": -1.153906226158142,
      "loss": 1.2422,
      "nll_loss": 1.1925780773162842,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07601318508386612,
      "rewards/margins": 0.03940429538488388,
      "rewards/rejected": -0.1153564453125,
      "step": 290
    },
    {
      "epoch": 0.02276521475185916,
      "grad_norm": 1.4425392059458433,
      "learning_rate": 4.618802153517006e-06,
      "log_odds_chosen": 0.539794921875,
      "log_odds_ratio": -0.584277331829071,
      "logits/chosen": -0.8628906011581421,
      "logits/rejected": -0.816601574420929,
      "logps/chosen": -0.7953125238418579,
      "logps/rejected": -1.162109375,
      "loss": 1.2105,
      "nll_loss": 1.14453125,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07958984375,
      "rewards/margins": 0.03663330152630806,
      "rewards/rejected": -0.11616210639476776,
      "step": 300
    },
    {
      "epoch": 0.023524055243587798,
      "grad_norm": 1.4229245984267798,
      "learning_rate": 4.543694673976518e-06,
      "log_odds_chosen": 0.553759753704071,
      "log_odds_ratio": -0.567187488079071,
      "logits/chosen": -0.891406238079071,
      "logits/rejected": -0.8333984613418579,
      "logps/chosen": -0.7958984375,
      "logps/rejected": -1.169335961341858,
      "loss": 1.2145,
      "nll_loss": 1.260351538658142,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07958984375,
      "rewards/margins": 0.03729400783777237,
      "rewards/rejected": -0.11689452826976776,
      "step": 310
    },
    {
      "epoch": 0.024282895735316435,
      "grad_norm": 1.5337475875154933,
      "learning_rate": 4.472135954999579e-06,
      "log_odds_chosen": 0.5230346918106079,
      "log_odds_ratio": -0.5665038824081421,
      "logits/chosen": -0.8939453363418579,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.8011718988418579,
      "logps/rejected": -1.1394531726837158,
      "loss": 1.2126,
      "nll_loss": 1.1082031726837158,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08017577975988388,
      "rewards/margins": 0.03389739990234375,
      "rewards/rejected": -0.11411132663488388,
      "step": 320
    },
    {
      "epoch": 0.025041736227045076,
      "grad_norm": 1.4941611010961415,
      "learning_rate": 4.403855060505443e-06,
      "log_odds_chosen": 0.5206054449081421,
      "log_odds_ratio": -0.6089843511581421,
      "logits/chosen": -0.9638671875,
      "logits/rejected": -0.9195312261581421,
      "logps/chosen": -0.776562511920929,
      "logps/rejected": -1.1339843273162842,
      "loss": 1.1647,
      "nll_loss": 1.0783202648162842,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07766113430261612,
      "rewards/margins": 0.03580169752240181,
      "rewards/rejected": -0.11337890475988388,
      "step": 330
    },
    {
      "epoch": 0.025800576718773713,
      "grad_norm": 8.234299330038246,
      "learning_rate": 4.338609156373123e-06,
      "log_odds_chosen": 0.559277355670929,
      "log_odds_ratio": -0.579296886920929,
      "logits/chosen": -0.9681640863418579,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.7923828363418579,
      "logps/rejected": -1.1804687976837158,
      "loss": 1.1985,
      "nll_loss": 1.120703101158142,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07926025241613388,
      "rewards/margins": 0.03888397291302681,
      "rewards/rejected": -0.11806640774011612,
      "step": 340
    },
    {
      "epoch": 0.026559417210502353,
      "grad_norm": 1.4986198531651644,
      "learning_rate": 4.27617987059879e-06,
      "log_odds_chosen": 0.5880126953125,
      "log_odds_ratio": -0.5694335699081421,
      "logits/chosen": -0.9203125238418579,
      "logits/rejected": -0.850390613079071,
      "logps/chosen": -0.798828125,
      "logps/rejected": -1.200781226158142,
      "loss": 1.1852,
      "nll_loss": 1.150976538658142,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07993163913488388,
      "rewards/margins": 0.04015808179974556,
      "rewards/rejected": -0.12001953274011612,
      "step": 350
    },
    {
      "epoch": 0.02731825770223099,
      "grad_norm": 1.4802049838530484,
      "learning_rate": 4.216370213557839e-06,
      "log_odds_chosen": 0.5188354253768921,
      "log_odds_ratio": -0.607617199420929,
      "logits/chosen": -0.9332031011581421,
      "logits/rejected": -0.862500011920929,
      "logps/chosen": -0.8257812261581421,
      "logps/rejected": -1.1884765625,
      "loss": 1.2022,
      "nll_loss": 1.246484398841858,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.08256836235523224,
      "rewards/margins": 0.03617553785443306,
      "rewards/rejected": -0.1187744140625,
      "step": 360
    },
    {
      "epoch": 0.02807709819395963,
      "grad_norm": 1.4330692363937183,
      "learning_rate": 4.15900195928029e-06,
      "log_odds_chosen": 0.586865246295929,
      "log_odds_ratio": -0.60498046875,
      "logits/chosen": -0.9009765386581421,
      "logits/rejected": -0.858203113079071,
      "logps/chosen": -0.797656238079071,
      "logps/rejected": -1.1896483898162842,
      "loss": 1.2002,
      "nll_loss": 1.1806640625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07974853366613388,
      "rewards/margins": 0.03930206224322319,
      "rewards/rejected": -0.11899413913488388,
      "step": 370
    },
    {
      "epoch": 0.02883593868568827,
      "grad_norm": 1.4912577480503046,
      "learning_rate": 4.103913408340617e-06,
      "log_odds_chosen": 0.540576159954071,
      "log_odds_ratio": -0.6109863519668579,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.9087890386581421,
      "logps/chosen": -0.785937488079071,
      "logps/rejected": -1.1513671875,
      "loss": 1.1835,
      "nll_loss": 1.194726586341858,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07860107719898224,
      "rewards/margins": 0.03663025051355362,
      "rewards/rejected": -0.11527099460363388,
      "step": 380
    },
    {
      "epoch": 0.029594779177416906,
      "grad_norm": 1.344518207009396,
      "learning_rate": 4.050957468334666e-06,
      "log_odds_chosen": 0.5313720703125,
      "log_odds_ratio": -0.606249988079071,
      "logits/chosen": -0.9189453125,
      "logits/rejected": -0.8687499761581421,
      "logps/chosen": -0.779101550579071,
      "logps/rejected": -1.145117163658142,
      "loss": 1.1957,
      "nll_loss": 1.102929711341858,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.07797851413488388,
      "rewards/margins": 0.03666992112994194,
      "rewards/rejected": -0.11447753757238388,
      "step": 390
    },
    {
      "epoch": 0.030353619669145546,
      "grad_norm": 4.3130074404618135,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.45671385526657104,
      "log_odds_ratio": -0.6246093511581421,
      "logits/chosen": -0.918164074420929,
      "logits/rejected": -0.857617199420929,
      "logps/chosen": -0.7632812261581421,
      "logps/rejected": -1.0556640625,
      "loss": 1.1896,
      "nll_loss": 1.123632788658142,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07639160007238388,
      "rewards/margins": 0.02919769287109375,
      "rewards/rejected": -0.10543213039636612,
      "step": 400
    },
    {
      "epoch": 0.031112460160874184,
      "grad_norm": 1.3095212395183442,
      "learning_rate": 3.950918386598359e-06,
      "log_odds_chosen": 0.7026733160018921,
      "log_odds_ratio": -0.5523437261581421,
      "logits/chosen": -0.970507800579071,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.783007800579071,
      "logps/rejected": -1.248437523841858,
      "loss": 1.1875,
      "nll_loss": 1.142187476158142,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07823486626148224,
      "rewards/margins": 0.04652862623333931,
      "rewards/rejected": -0.12485351413488388,
      "step": 410
    },
    {
      "epoch": 0.031871300652602824,
      "grad_norm": 1.538770613712715,
      "learning_rate": 3.903600291794132e-06,
      "log_odds_chosen": 0.9018310308456421,
      "log_odds_ratio": -0.4969726502895355,
      "logits/chosen": -0.8892577886581421,
      "logits/rejected": -0.81640625,
      "logps/chosen": -0.752734363079071,
      "logps/rejected": -1.372656226158142,
      "loss": 1.1881,
      "nll_loss": 1.146875023841858,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07523193210363388,
      "rewards/margins": 0.06215057522058487,
      "rewards/rejected": -0.13740234076976776,
      "step": 420
    },
    {
      "epoch": 0.032630141144331465,
      "grad_norm": 1.5135129410829358,
      "learning_rate": 3.857942577363297e-06,
      "log_odds_chosen": 0.5770508050918579,
      "log_odds_ratio": -0.5787109136581421,
      "logits/chosen": -0.9457031488418579,
      "logits/rejected": -0.872851550579071,
      "logps/chosen": -0.789257824420929,
      "logps/rejected": -1.173828125,
      "loss": 1.1877,
      "nll_loss": 1.115234375,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07891845703125,
      "rewards/margins": 0.03848876804113388,
      "rewards/rejected": -0.11740722507238388,
      "step": 430
    },
    {
      "epoch": 0.0333889816360601,
      "grad_norm": 1.5495511200199545,
      "learning_rate": 3.813850356982369e-06,
      "log_odds_chosen": 0.7400146722793579,
      "log_odds_ratio": -0.5404297113418579,
      "logits/chosen": -0.9517577886581421,
      "logits/rejected": -0.914257824420929,
      "logps/chosen": -0.779101550579071,
      "logps/rejected": -1.2970702648162842,
      "loss": 1.1755,
      "nll_loss": 1.0986328125,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07794189453125,
      "rewards/margins": 0.05182952806353569,
      "rewards/rejected": -0.12973633408546448,
      "step": 440
    },
    {
      "epoch": 0.03414782212778874,
      "grad_norm": 1.434457768263082,
      "learning_rate": 3.7712361663282537e-06,
      "log_odds_chosen": 0.7149658203125,
      "log_odds_ratio": -0.53125,
      "logits/chosen": -0.892382800579071,
      "logits/rejected": -0.848437488079071,
      "logps/chosen": -0.7767578363418579,
      "logps/rejected": -1.2646484375,
      "loss": 1.1691,
      "nll_loss": 1.088476538658142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07757568359375,
      "rewards/margins": 0.04888000339269638,
      "rewards/rejected": -0.12656250596046448,
      "step": 450
    },
    {
      "epoch": 0.03490666261951738,
      "grad_norm": 1.4647624075894172,
      "learning_rate": 3.730019232961255e-06,
      "log_odds_chosen": 0.598742663860321,
      "log_odds_ratio": -0.5794922113418579,
      "logits/chosen": -0.846875011920929,
      "logits/rejected": -0.7669922113418579,
      "logps/chosen": -0.788281261920929,
      "logps/rejected": -1.1970703601837158,
      "loss": 1.1697,
      "nll_loss": 1.101171851158142,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07880859076976776,
      "rewards/margins": 0.04080658033490181,
      "rewards/rejected": -0.11972656100988388,
      "step": 460
    },
    {
      "epoch": 0.035665503111246014,
      "grad_norm": 1.460032211323527,
      "learning_rate": 3.6901248321155403e-06,
      "log_odds_chosen": 0.535937488079071,
      "log_odds_ratio": -0.59423828125,
      "logits/chosen": -0.857226550579071,
      "logits/rejected": -0.783984363079071,
      "logps/chosen": -0.793164074420929,
      "logps/rejected": -1.162695288658142,
      "loss": 1.1848,
      "nll_loss": 1.1906249523162842,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07929687201976776,
      "rewards/margins": 0.03700103610754013,
      "rewards/rejected": -0.11623535305261612,
      "step": 470
    },
    {
      "epoch": 0.036424343602974654,
      "grad_norm": 1.3524653524950496,
      "learning_rate": 3.6514837167011073e-06,
      "log_odds_chosen": 0.646289050579071,
      "log_odds_ratio": -0.5396484136581421,
      "logits/chosen": -0.888867199420929,
      "logits/rejected": -0.8472656011581421,
      "logps/chosen": -0.755664050579071,
      "logps/rejected": -1.179296851158142,
      "loss": 1.1176,
      "nll_loss": 1.028906226158142,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.0755615234375,
      "rewards/margins": 0.04240112379193306,
      "rewards/rejected": -0.11789550632238388,
      "step": 480
    },
    {
      "epoch": 0.037183184094703295,
      "grad_norm": 1.575634651364888,
      "learning_rate": 3.6140316116210052e-06,
      "log_odds_chosen": 0.5822998285293579,
      "log_odds_ratio": -0.582324206829071,
      "logits/chosen": -0.8666015863418579,
      "logits/rejected": -0.790820300579071,
      "logps/chosen": -0.757617175579071,
      "logps/rejected": -1.1457030773162842,
      "loss": 1.1698,
      "nll_loss": 1.113671898841858,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07574462890625,
      "rewards/margins": 0.03886108472943306,
      "rewards/rejected": -0.1146240234375,
      "step": 490
    },
    {
      "epoch": 0.03794202458643193,
      "grad_norm": 1.4509304015827453,
      "learning_rate": 3.5777087639996634e-06,
      "log_odds_chosen": 0.682324230670929,
      "log_odds_ratio": -0.526074230670929,
      "logits/chosen": -0.888867199420929,
      "logits/rejected": -0.8194335699081421,
      "logps/chosen": -0.7503906488418579,
      "logps/rejected": -1.1746094226837158,
      "loss": 1.1828,
      "nll_loss": 1.1652343273162842,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.0750732421875,
      "rewards/margins": 0.0423583984375,
      "rewards/rejected": -0.117431640625,
      "step": 500
    },
    {
      "epoch": 0.03870086507816057,
      "grad_norm": 1.8399283965320115,
      "learning_rate": 3.5424595421603814e-06,
      "log_odds_chosen": 0.82373046875,
      "log_odds_ratio": -0.5107421875,
      "logits/chosen": -0.8783203363418579,
      "logits/rejected": -0.8070312738418579,
      "logps/chosen": -0.736328125,
      "logps/rejected": -1.3039062023162842,
      "loss": 1.1434,
      "nll_loss": 1.0751953125,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07369384914636612,
      "rewards/margins": 0.05676879733800888,
      "rewards/rejected": -0.13034668564796448,
      "step": 510
    },
    {
      "epoch": 0.03945970556988921,
      "grad_norm": 1.476828967448819,
      "learning_rate": 3.5082320772281165e-06,
      "log_odds_chosen": 0.8194335699081421,
      "log_odds_ratio": -0.5054687261581421,
      "logits/chosen": -0.9585937261581421,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.7845703363418579,
      "logps/rejected": -1.348046898841858,
      "loss": 1.1622,
      "nll_loss": 1.1238281726837158,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07841797173023224,
      "rewards/margins": 0.05634155124425888,
      "rewards/rejected": -0.13469238579273224,
      "step": 520
    },
    {
      "epoch": 0.04021854606161785,
      "grad_norm": 1.539593920271273,
      "learning_rate": 3.474977942104555e-06,
      "log_odds_chosen": 0.5797973871231079,
      "log_odds_ratio": -0.5848633050918579,
      "logits/chosen": -0.947460949420929,
      "logits/rejected": -0.8843749761581421,
      "logps/chosen": -0.773632824420929,
      "logps/rejected": -1.160742163658142,
      "loss": 1.1431,
      "nll_loss": 1.1005859375,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07738037407398224,
      "rewards/margins": 0.03878479078412056,
      "rewards/rejected": -0.11613769829273224,
      "step": 530
    },
    {
      "epoch": 0.040977386553346484,
      "grad_norm": 1.4842674812940488,
      "learning_rate": 3.442651863295481e-06,
      "log_odds_chosen": 0.652294933795929,
      "log_odds_ratio": -0.5572265386581421,
      "logits/chosen": -0.9189453125,
      "logits/rejected": -0.8451172113418579,
      "logps/chosen": -0.749804675579071,
      "logps/rejected": -1.199609398841858,
      "loss": 1.1613,
      "nll_loss": 1.100976586341858,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0750732421875,
      "rewards/margins": 0.04497985914349556,
      "rewards/rejected": -0.12001953274011612,
      "step": 540
    },
    {
      "epoch": 0.041736227045075125,
      "grad_norm": 1.5239315494051218,
      "learning_rate": 3.4112114616897665e-06,
      "log_odds_chosen": 0.730712890625,
      "log_odds_ratio": -0.532910168170929,
      "logits/chosen": -0.9126952886581421,
      "logits/rejected": -0.8607422113418579,
      "logps/chosen": -0.740039050579071,
      "logps/rejected": -1.240234375,
      "loss": 1.1523,
      "nll_loss": 1.071874976158142,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07404784858226776,
      "rewards/margins": 0.04995117336511612,
      "rewards/rejected": -0.12397460639476776,
      "step": 550
    },
    {
      "epoch": 0.042495067536803766,
      "grad_norm": 1.4941924034303562,
      "learning_rate": 3.3806170189140663e-06,
      "log_odds_chosen": 0.6442626714706421,
      "log_odds_ratio": -0.5541015863418579,
      "logits/chosen": -0.9439452886581421,
      "logits/rejected": -0.8671875,
      "logps/chosen": -0.7408202886581421,
      "logps/rejected": -1.167578101158142,
      "loss": 1.1318,
      "nll_loss": 1.0128905773162842,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07408447563648224,
      "rewards/margins": 0.04265136644244194,
      "rewards/rejected": -0.11665038764476776,
      "step": 560
    },
    {
      "epoch": 0.0432539080285324,
      "grad_norm": 1.5068041723722188,
      "learning_rate": 3.350831266333564e-06,
      "log_odds_chosen": 0.572314441204071,
      "log_odds_ratio": -0.5923827886581421,
      "logits/chosen": -0.942187488079071,
      "logits/rejected": -0.8658202886581421,
      "logps/chosen": -0.798632800579071,
      "logps/rejected": -1.197656273841858,
      "loss": 1.1688,
      "nll_loss": 1.111718773841858,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.079833984375,
      "rewards/margins": 0.03993072360754013,
      "rewards/rejected": -0.11982421576976776,
      "step": 570
    },
    {
      "epoch": 0.04401274852026104,
      "grad_norm": 1.4143791261814025,
      "learning_rate": 3.3218191941495984e-06,
      "log_odds_chosen": 0.768115222454071,
      "log_odds_ratio": -0.4981445372104645,
      "logits/chosen": -0.9248046875,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.746874988079071,
      "logps/rejected": -1.2615234851837158,
      "loss": 1.1714,
      "nll_loss": 1.0695312023162842,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07470703125,
      "rewards/margins": 0.05145568773150444,
      "rewards/rejected": -0.12619629502296448,
      "step": 580
    },
    {
      "epoch": 0.04477158901198968,
      "grad_norm": 1.5758760245545085,
      "learning_rate": 3.293547878370473e-06,
      "log_odds_chosen": 0.5545409917831421,
      "log_odds_ratio": -0.612597644329071,
      "logits/chosen": -0.8935546875,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.765429675579071,
      "logps/rejected": -1.1183593273162842,
      "loss": 1.1337,
      "nll_loss": 1.1160156726837158,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07655028998851776,
      "rewards/margins": 0.03520049899816513,
      "rewards/rejected": -0.11186523735523224,
      "step": 590
    },
    {
      "epoch": 0.04553042950371832,
      "grad_norm": 1.4028074979920369,
      "learning_rate": 3.2659863237109044e-06,
      "log_odds_chosen": 0.966503918170929,
      "log_odds_ratio": -0.4481445252895355,
      "logits/chosen": -0.9169921875,
      "logits/rejected": -0.8544921875,
      "logps/chosen": -0.7320312261581421,
      "logps/rejected": -1.3779296875,
      "loss": 1.1331,
      "nll_loss": 1.105859398841858,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07319335639476776,
      "rewards/margins": 0.06447754055261612,
      "rewards/rejected": -0.1376953125,
      "step": 600
    },
    {
      "epoch": 0.046289269995446955,
      "grad_norm": 1.49643970687548,
      "learning_rate": 3.239105320715664e-06,
      "log_odds_chosen": 0.6585937738418579,
      "log_odds_ratio": -0.5478515625,
      "logits/chosen": -0.9150390625,
      "logits/rejected": -0.8597656488418579,
      "logps/chosen": -0.741015613079071,
      "logps/rejected": -1.168554663658142,
      "loss": 1.1114,
      "nll_loss": 1.037109375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07413329929113388,
      "rewards/margins": 0.04278564453125,
      "rewards/rejected": -0.11684570461511612,
      "step": 610
    },
    {
      "epoch": 0.047048110487175596,
      "grad_norm": 1.4603778539467505,
      "learning_rate": 3.2128773156099956e-06,
      "log_odds_chosen": 0.7294921875,
      "log_odds_ratio": -0.5185546875,
      "logits/chosen": -0.9248046875,
      "logits/rejected": -0.872265636920929,
      "logps/chosen": -0.7431640625,
      "logps/rejected": -1.24609375,
      "loss": 1.1383,
      "nll_loss": 1.102148413658142,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07432861626148224,
      "rewards/margins": 0.05032043531537056,
      "rewards/rejected": -0.12458495795726776,
      "step": 620
    },
    {
      "epoch": 0.047806950978904236,
      "grad_norm": 1.4270703213144766,
      "learning_rate": 3.187276291558383e-06,
      "log_odds_chosen": 0.694042980670929,
      "log_odds_ratio": -0.52392578125,
      "logits/chosen": -0.9664062261581421,
      "logits/rejected": -0.9027343988418579,
      "logps/chosen": -0.7904297113418579,
      "logps/rejected": -1.267578125,
      "loss": 1.1549,
      "nll_loss": 1.1359374523162842,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07899169623851776,
      "rewards/margins": 0.047760009765625,
      "rewards/rejected": -0.126708984375,
      "step": 630
    },
    {
      "epoch": 0.04856579147063287,
      "grad_norm": 1.3640593722657062,
      "learning_rate": 3.1622776601683788e-06,
      "log_odds_chosen": 0.6767578125,
      "log_odds_ratio": -0.5350586175918579,
      "logits/chosen": -0.9214843511581421,
      "logits/rejected": -0.876171886920929,
      "logps/chosen": -0.781054675579071,
      "logps/rejected": -1.224218726158142,
      "loss": 1.1219,
      "nll_loss": 1.0398437976837158,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.078125,
      "rewards/margins": 0.04420776292681694,
      "rewards/rejected": -0.12226562201976776,
      "step": 640
    },
    {
      "epoch": 0.04932463196236151,
      "grad_norm": 1.39289555170604,
      "learning_rate": 3.1378581622109444e-06,
      "log_odds_chosen": 0.5647217035293579,
      "log_odds_ratio": -0.6126953363418579,
      "logits/chosen": -1.0076172351837158,
      "logits/rejected": -0.9320312738418579,
      "logps/chosen": -0.8109375238418579,
      "logps/rejected": -1.2166016101837158,
      "loss": 1.1219,
      "nll_loss": 1.067773461341858,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.08111572265625,
      "rewards/margins": 0.040557097643613815,
      "rewards/rejected": -0.12163086235523224,
      "step": 650
    },
    {
      "epoch": 0.05008347245409015,
      "grad_norm": 1.5575966456743369,
      "learning_rate": 3.113995776646092e-06,
      "log_odds_chosen": 0.9054199457168579,
      "log_odds_ratio": -0.4915527403354645,
      "logits/chosen": -0.9623047113418579,
      "logits/rejected": -0.898242175579071,
      "logps/chosen": -0.7110351324081421,
      "logps/rejected": -1.308203101158142,
      "loss": 1.1272,
      "nll_loss": 1.0681641101837158,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07109375298023224,
      "rewards/margins": 0.05971984937787056,
      "rewards/rejected": -0.13078613579273224,
      "step": 660
    },
    {
      "epoch": 0.05084231294581879,
      "grad_norm": 1.5820498982958486,
      "learning_rate": 3.090669637145023e-06,
      "log_odds_chosen": 0.920214831829071,
      "log_odds_ratio": -0.48076170682907104,
      "logits/chosen": -0.999218761920929,
      "logits/rejected": -0.9287109375,
      "logps/chosen": -0.7611328363418579,
      "logps/rejected": -1.382421851158142,
      "loss": 1.1265,
      "nll_loss": 1.099023461341858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07607422024011612,
      "rewards/margins": 0.0622406005859375,
      "rewards/rejected": -0.13835449516773224,
      "step": 670
    },
    {
      "epoch": 0.051601153437547426,
      "grad_norm": 1.412160791122473,
      "learning_rate": 3.0678599553894814e-06,
      "log_odds_chosen": 0.636523425579071,
      "log_odds_ratio": -0.5533202886581421,
      "logits/chosen": -0.931640625,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.754687488079071,
      "logps/rejected": -1.1794922351837158,
      "loss": 1.1396,
      "nll_loss": 1.0632812976837158,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07546386867761612,
      "rewards/margins": 0.042449951171875,
      "rewards/rejected": -0.11796875298023224,
      "step": 680
    },
    {
      "epoch": 0.052359993929276066,
      "grad_norm": 1.6469723394316966,
      "learning_rate": 3.0455479505075235e-06,
      "log_odds_chosen": 0.900097668170929,
      "log_odds_ratio": -0.47685545682907104,
      "logits/chosen": -0.951171875,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.7466796636581421,
      "logps/rejected": -1.344140648841858,
      "loss": 1.1208,
      "nll_loss": 1.0744140148162842,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07470703125,
      "rewards/margins": 0.05967559665441513,
      "rewards/rejected": -0.13437500596046448,
      "step": 690
    },
    {
      "epoch": 0.05311883442100471,
      "grad_norm": 2.299409754878046,
      "learning_rate": 3.0237157840738173e-06,
      "log_odds_chosen": 0.818359375,
      "log_odds_ratio": -0.502246081829071,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.8402343988418579,
      "logps/chosen": -0.728515625,
      "logps/rejected": -1.2646484375,
      "loss": 1.1038,
      "nll_loss": 1.045507788658142,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07291259616613388,
      "rewards/margins": 0.05359496921300888,
      "rewards/rejected": -0.12663574516773224,
      "step": 700
    },
    {
      "epoch": 0.05387767491273334,
      "grad_norm": 1.4366530156011492,
      "learning_rate": 3.002346500163206e-06,
      "log_odds_chosen": 0.676806628704071,
      "log_odds_ratio": -0.568066418170929,
      "logits/chosen": -0.892578125,
      "logits/rejected": -0.8421875238418579,
      "logps/chosen": -0.7298828363418579,
      "logps/rejected": -1.1785156726837158,
      "loss": 1.0838,
      "nll_loss": 1.046875,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.072998046875,
      "rewards/margins": 0.04483947902917862,
      "rewards/rejected": -0.1177978515625,
      "step": 710
    },
    {
      "epoch": 0.05463651540446198,
      "grad_norm": 1.4091415441420312,
      "learning_rate": 2.9814239699997195e-06,
      "log_odds_chosen": 0.730639636516571,
      "log_odds_ratio": -0.5448242425918579,
      "logits/chosen": -0.9156249761581421,
      "logits/rejected": -0.869921863079071,
      "logps/chosen": -0.7748047113418579,
      "logps/rejected": -1.267968773841858,
      "loss": 1.1281,
      "nll_loss": 1.166601538658142,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07746581733226776,
      "rewards/margins": 0.04938201978802681,
      "rewards/rejected": -0.1268310546875,
      "step": 720
    },
    {
      "epoch": 0.05539535589619062,
      "grad_norm": 1.5428528298733337,
      "learning_rate": 2.9609328407904207e-06,
      "log_odds_chosen": 0.861193835735321,
      "log_odds_ratio": -0.473388671875,
      "logits/chosen": -0.983203113079071,
      "logits/rejected": -0.8863281011581421,
      "logps/chosen": -0.708984375,
      "logps/rejected": -1.2703125476837158,
      "loss": 1.0983,
      "nll_loss": 1.019921898841858,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07081298530101776,
      "rewards/margins": 0.05611877515912056,
      "rewards/rejected": -0.12700195610523224,
      "step": 730
    },
    {
      "epoch": 0.05615419638791926,
      "grad_norm": 1.3181361407151604,
      "learning_rate": 2.940858488375231e-06,
      "log_odds_chosen": 0.657470703125,
      "log_odds_ratio": -0.5528320074081421,
      "logits/chosen": -0.9462890625,
      "logits/rejected": -0.882031261920929,
      "logps/chosen": -0.7880859375,
      "logps/rejected": -1.2468750476837158,
      "loss": 1.1081,
      "nll_loss": 1.021484375,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07878418266773224,
      "rewards/margins": 0.04585418850183487,
      "rewards/rejected": -0.12468262016773224,
      "step": 740
    },
    {
      "epoch": 0.056913036879647896,
      "grad_norm": 1.4129096893123931,
      "learning_rate": 2.9211869733608857e-06,
      "log_odds_chosen": 0.70599365234375,
      "log_odds_ratio": -0.537890613079071,
      "logits/chosen": -0.947265625,
      "logits/rejected": -0.892773449420929,
      "logps/chosen": -0.759960949420929,
      "logps/rejected": -1.2421875,
      "loss": 1.1035,
      "nll_loss": 1.037695288658142,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07593993842601776,
      "rewards/margins": 0.04822998121380806,
      "rewards/rejected": -0.124267578125,
      "step": 750
    },
    {
      "epoch": 0.05767187737137654,
      "grad_norm": 1.4910014683581294,
      "learning_rate": 2.901905000440047e-06,
      "log_odds_chosen": 0.8280273675918579,
      "log_odds_ratio": -0.504443347454071,
      "logits/chosen": -0.961132824420929,
      "logits/rejected": -0.8509765863418579,
      "logps/chosen": -0.71484375,
      "logps/rejected": -1.2763671875,
      "loss": 1.1171,
      "nll_loss": 1.068750023841858,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07152099907398224,
      "rewards/margins": 0.05616455152630806,
      "rewards/rejected": -0.12775878608226776,
      "step": 760
    },
    {
      "epoch": 0.05843071786310518,
      "grad_norm": 1.367660612325947,
      "learning_rate": 2.8829998806257885e-06,
      "log_odds_chosen": 0.779040515422821,
      "log_odds_ratio": -0.5166991949081421,
      "logits/chosen": -0.934374988079071,
      "logits/rejected": -0.845507800579071,
      "logps/chosen": -0.681347668170929,
      "logps/rejected": -1.188085913658142,
      "loss": 1.1298,
      "nll_loss": 1.0185546875,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.068115234375,
      "rewards/margins": 0.05065765231847763,
      "rewards/rejected": -0.11887206882238388,
      "step": 770
    },
    {
      "epoch": 0.05918955835483381,
      "grad_norm": 1.5187404836399663,
      "learning_rate": 2.8644594961577314e-06,
      "log_odds_chosen": 0.7642577886581421,
      "log_odds_ratio": -0.5435546636581421,
      "logits/chosen": -0.9417968988418579,
      "logits/rejected": -0.8802734613418579,
      "logps/chosen": -0.768750011920929,
      "logps/rejected": -1.3263671398162842,
      "loss": 1.1197,
      "nll_loss": 1.089453101158142,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07689209282398224,
      "rewards/margins": 0.05573882907629013,
      "rewards/rejected": -0.13266602158546448,
      "step": 780
    },
    {
      "epoch": 0.05994839884656245,
      "grad_norm": 1.3998297985203276,
      "learning_rate": 2.84627226785928e-06,
      "log_odds_chosen": 0.68170166015625,
      "log_odds_ratio": -0.550976574420929,
      "logits/chosen": -0.9701172113418579,
      "logits/rejected": -0.875,
      "logps/chosen": -0.741992175579071,
      "logps/rejected": -1.1867187023162842,
      "loss": 1.1149,
      "nll_loss": 1.048242211341858,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07418213039636612,
      "rewards/margins": 0.04451598972082138,
      "rewards/rejected": -0.11870117485523224,
      "step": 790
    },
    {
      "epoch": 0.06070723933829109,
      "grad_norm": 1.5365658645460811,
      "learning_rate": 2.82842712474619e-06,
      "log_odds_chosen": 0.8926757574081421,
      "log_odds_ratio": -0.5009765625,
      "logits/chosen": -0.938281238079071,
      "logits/rejected": -0.870312511920929,
      "logps/chosen": -0.7412109375,
      "logps/rejected": -1.378320336341858,
      "loss": 1.1089,
      "nll_loss": 1.032812476158142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07413329929113388,
      "rewards/margins": 0.06382445991039276,
      "rewards/rejected": -0.13789062201976776,
      "step": 800
    },
    {
      "epoch": 0.061466079830019726,
      "grad_norm": 1.553258112910012,
      "learning_rate": 2.810913475705226e-06,
      "log_odds_chosen": 0.827832043170929,
      "log_odds_ratio": -0.4930664002895355,
      "logits/chosen": -0.9130859375,
      "logits/rejected": -0.861523449420929,
      "logps/chosen": -0.7025390863418579,
      "logps/rejected": -1.2615234851837158,
      "loss": 1.1092,
      "nll_loss": 1.0056641101837158,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07026366889476776,
      "rewards/margins": 0.0557861328125,
      "rewards/rejected": -0.1259765625,
      "step": 810
    },
    {
      "epoch": 0.06222492032174837,
      "grad_norm": 1.6754050719010933,
      "learning_rate": 2.7937211830783128e-06,
      "log_odds_chosen": 0.825244128704071,
      "log_odds_ratio": -0.513720691204071,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.9544922113418579,
      "logps/chosen": -0.761523425579071,
      "logps/rejected": -1.33203125,
      "loss": 1.1177,
      "nll_loss": 1.129296898841858,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07611083984375,
      "rewards/margins": 0.05705719068646431,
      "rewards/rejected": -0.13322754204273224,
      "step": 820
    },
    {
      "epoch": 0.06298376081347701,
      "grad_norm": 1.4397189392753629,
      "learning_rate": 2.776840538002493e-06,
      "log_odds_chosen": 0.8416992425918579,
      "log_odds_ratio": -0.51953125,
      "logits/chosen": -0.950976550579071,
      "logits/rejected": -0.9068359136581421,
      "logps/chosen": -0.737109363079071,
      "logps/rejected": -1.309960961341858,
      "loss": 1.0814,
      "nll_loss": 0.9839843511581421,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07368163764476776,
      "rewards/margins": 0.0572509765625,
      "rewards/rejected": -0.1309814453125,
      "step": 830
    },
    {
      "epoch": 0.06374260130520565,
      "grad_norm": 1.481525111906686,
      "learning_rate": 2.7602622373694163e-06,
      "log_odds_chosen": 0.6253417730331421,
      "log_odds_ratio": -0.569628894329071,
      "logits/chosen": -0.931835949420929,
      "logits/rejected": -0.8716796636581421,
      "logps/chosen": -0.7300781011581421,
      "logps/rejected": -1.1677734851837158,
      "loss": 1.1057,
      "nll_loss": 1.015234351158142,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07297363132238388,
      "rewards/margins": 0.04375610500574112,
      "rewards/rejected": -0.1168212890625,
      "step": 840
    },
    {
      "epoch": 0.06450144179693429,
      "grad_norm": 1.4528723364813,
      "learning_rate": 2.743977362280141e-06,
      "log_odds_chosen": 0.7959960699081421,
      "log_odds_ratio": -0.5220702886581421,
      "logits/chosen": -0.975390613079071,
      "logits/rejected": -0.915234386920929,
      "logps/chosen": -0.7105468511581421,
      "logps/rejected": -1.235937476158142,
      "loss": 1.1185,
      "nll_loss": 1.0314452648162842,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07106933742761612,
      "rewards/margins": 0.0523834228515625,
      "rewards/rejected": -0.12353515625,
      "step": 850
    },
    {
      "epoch": 0.06526028228866293,
      "grad_norm": 1.4155260042724327,
      "learning_rate": 2.7279773578818937e-06,
      "log_odds_chosen": 0.774218738079071,
      "log_odds_ratio": -0.54150390625,
      "logits/chosen": -0.9326171875,
      "logits/rejected": -0.8871093988418579,
      "logps/chosen": -0.7484375238418579,
      "logps/rejected": -1.286523461341858,
      "loss": 1.1145,
      "nll_loss": 0.9873046875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07486572116613388,
      "rewards/margins": 0.05366821214556694,
      "rewards/rejected": -0.12849120795726776,
      "step": 860
    },
    {
      "epoch": 0.06601912278039156,
      "grad_norm": 1.4672576973291955,
      "learning_rate": 2.7122540144832417e-06,
      "log_odds_chosen": 0.6922851800918579,
      "log_odds_ratio": -0.5411132574081421,
      "logits/chosen": -0.9400390386581421,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.7162109613418579,
      "logps/rejected": -1.152929663658142,
      "loss": 1.1071,
      "nll_loss": 1.0554687976837158,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07163085788488388,
      "rewards/margins": 0.04363097995519638,
      "rewards/rejected": -0.11533202975988388,
      "step": 870
    },
    {
      "epoch": 0.0667779632721202,
      "grad_norm": 1.4377110735801357,
      "learning_rate": 2.696799449852968e-06,
      "log_odds_chosen": 0.7808837890625,
      "log_odds_ratio": -0.517578125,
      "logits/chosen": -0.8804687261581421,
      "logits/rejected": -0.841796875,
      "logps/chosen": -0.744335949420929,
      "logps/rejected": -1.2470703125,
      "loss": 1.1171,
      "nll_loss": 1.073828101158142,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07441405951976776,
      "rewards/margins": 0.05037841945886612,
      "rewards/rejected": -0.12473144382238388,
      "step": 880
    },
    {
      "epoch": 0.06753680376384884,
      "grad_norm": 1.5523580294418937,
      "learning_rate": 2.6816060926159636e-06,
      "log_odds_chosen": 0.8277221918106079,
      "log_odds_ratio": -0.506103515625,
      "logits/chosen": -0.948046863079071,
      "logits/rejected": -0.8501952886581421,
      "logps/chosen": -0.705078125,
      "logps/rejected": -1.2453124523162842,
      "loss": 1.0915,
      "nll_loss": 1.0099608898162842,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07052002102136612,
      "rewards/margins": 0.05402832105755806,
      "rewards/rejected": -0.12456054985523224,
      "step": 890
    },
    {
      "epoch": 0.06829564425557748,
      "grad_norm": 1.3981673710781848,
      "learning_rate": 2.6666666666666664e-06,
      "log_odds_chosen": 0.8017578125,
      "log_odds_ratio": -0.513134777545929,
      "logits/chosen": -0.940625011920929,
      "logits/rejected": -0.8638671636581421,
      "logps/chosen": -0.7030273675918579,
      "logps/rejected": -1.211328148841858,
      "loss": 1.0997,
      "nll_loss": 1.0662109851837158,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07032470405101776,
      "rewards/margins": 0.050933837890625,
      "rewards/rejected": -0.12116698920726776,
      "step": 900
    },
    {
      "epoch": 0.06905448474730612,
      "grad_norm": 1.4703260458527112,
      "learning_rate": 2.6519741765271837e-06,
      "log_odds_chosen": 0.8729492425918579,
      "log_odds_ratio": -0.4708007872104645,
      "logits/chosen": -0.9691406488418579,
      "logits/rejected": -0.8560546636581421,
      "logps/chosen": -0.6851562261581421,
      "logps/rejected": -1.2585937976837158,
      "loss": 1.1185,
      "nll_loss": 1.0173828601837158,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06849364936351776,
      "rewards/margins": 0.05747680738568306,
      "rewards/rejected": -0.12590332329273224,
      "step": 910
    },
    {
      "epoch": 0.06981332523903476,
      "grad_norm": 1.7240865708769755,
      "learning_rate": 2.637521893583148e-06,
      "log_odds_chosen": 0.833544909954071,
      "log_odds_ratio": -0.522167980670929,
      "logits/chosen": -0.9603515863418579,
      "logits/rejected": -0.895703136920929,
      "logps/chosen": -0.737500011920929,
      "logps/rejected": -1.32421875,
      "loss": 1.1254,
      "nll_loss": 1.0656249523162842,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07379150390625,
      "rewards/margins": 0.05867920070886612,
      "rewards/rejected": -0.13249512016773224,
      "step": 920
    },
    {
      "epoch": 0.0705721657307634,
      "grad_norm": 1.596536890919686,
      "learning_rate": 2.6233033431358115e-06,
      "log_odds_chosen": 0.688720703125,
      "log_odds_ratio": -0.5291992425918579,
      "logits/chosen": -0.9482421875,
      "logits/rejected": -0.8775390386581421,
      "logps/chosen": -0.698046863079071,
      "logps/rejected": -1.146093726158142,
      "loss": 1.1058,
      "nll_loss": 1.027929663658142,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06984863430261612,
      "rewards/margins": 0.04474487155675888,
      "rewards/rejected": -0.11459960788488388,
      "step": 930
    },
    {
      "epoch": 0.07133100622249203,
      "grad_norm": 1.5205195253800634,
      "learning_rate": 2.6093122922137685e-06,
      "log_odds_chosen": 0.684814453125,
      "log_odds_ratio": -0.5577148199081421,
      "logits/chosen": -0.9986327886581421,
      "logits/rejected": -0.963671863079071,
      "logps/chosen": -0.7222656011581421,
      "logps/rejected": -1.1583983898162842,
      "loss": 1.1012,
      "nll_loss": 0.95703125,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07225342094898224,
      "rewards/margins": 0.04364623874425888,
      "rewards/rejected": -0.11591796576976776,
      "step": 940
    },
    {
      "epoch": 0.07208984671422067,
      "grad_norm": 1.5984518956278635,
      "learning_rate": 2.5955427380922006e-06,
      "log_odds_chosen": 0.7061523199081421,
      "log_odds_ratio": -0.5171874761581421,
      "logits/chosen": -0.9800781011581421,
      "logits/rejected": -0.8882812261581421,
      "logps/chosen": -0.6806640625,
      "logps/rejected": -1.127539038658142,
      "loss": 1.1089,
      "nll_loss": 1.0857422351837158,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06800536811351776,
      "rewards/margins": 0.04460601881146431,
      "rewards/rejected": -0.1126708984375,
      "step": 950
    },
    {
      "epoch": 0.07284868720594931,
      "grad_norm": 1.5002616023423914,
      "learning_rate": 2.5819888974716113e-06,
      "log_odds_chosen": 0.765625,
      "log_odds_ratio": -0.555371105670929,
      "logits/chosen": -0.9507812261581421,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.7378906011581421,
      "logps/rejected": -1.265234351158142,
      "loss": 1.1068,
      "nll_loss": 1.115234375,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07376708835363388,
      "rewards/margins": 0.05278320237994194,
      "rewards/rejected": -0.12646484375,
      "step": 960
    },
    {
      "epoch": 0.07360752769767795,
      "grad_norm": 1.4651067791697183,
      "learning_rate": 2.5686451962717425e-06,
      "log_odds_chosen": 0.7742065191268921,
      "log_odds_ratio": -0.534130871295929,
      "logits/chosen": -0.9517577886581421,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.68896484375,
      "logps/rejected": -1.2111327648162842,
      "loss": 1.0792,
      "nll_loss": 1.058984398841858,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06888427585363388,
      "rewards/margins": 0.0521976463496685,
      "rewards/rejected": -0.12116698920726776,
      "step": 970
    },
    {
      "epoch": 0.07436636818940659,
      "grad_norm": 1.3951266787145413,
      "learning_rate": 2.5555062599997596e-06,
      "log_odds_chosen": 0.83544921875,
      "log_odds_ratio": -0.53076171875,
      "logits/chosen": -1.0031249523162842,
      "logits/rejected": -0.918164074420929,
      "logps/chosen": -0.7308593988418579,
      "logps/rejected": -1.3103516101837158,
      "loss": 1.0967,
      "nll_loss": 1.1240234375,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07314453274011612,
      "rewards/margins": 0.05808105319738388,
      "rewards/rejected": -0.13111571967601776,
      "step": 980
    },
    {
      "epoch": 0.07512520868113523,
      "grad_norm": 1.5995079707109392,
      "learning_rate": 2.5425669046549126e-06,
      "log_odds_chosen": 0.740466296672821,
      "log_odds_ratio": -0.553906261920929,
      "logits/chosen": -0.985156238079071,
      "logits/rejected": -0.911914050579071,
      "logps/chosen": -0.7710937261581421,
      "logps/rejected": -1.286523461341858,
      "loss": 1.1016,
      "nll_loss": 1.021875023841858,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07707519829273224,
      "rewards/margins": 0.05153656005859375,
      "rewards/rejected": -0.12871094048023224,
      "step": 990
    },
    {
      "epoch": 0.07588404917286386,
      "grad_norm": 1.6759497570386803,
      "learning_rate": 2.5298221281347034e-06,
      "log_odds_chosen": 1.0483887195587158,
      "log_odds_ratio": -0.4666992127895355,
      "logits/chosen": -1.021875023841858,
      "logits/rejected": -0.919140636920929,
      "logps/chosen": -0.7113281488418579,
      "logps/rejected": -1.450585961341858,
      "loss": 1.0925,
      "nll_loss": 1.025781273841858,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.071044921875,
      "rewards/margins": 0.07399749755859375,
      "rewards/rejected": -0.14519043266773224,
      "step": 1000
    },
    {
      "epoch": 0.0766428896645925,
      "grad_norm": 1.4286639544950612,
      "learning_rate": 2.5172671021102103e-06,
      "log_odds_chosen": 0.787768542766571,
      "log_odds_ratio": -0.5474609136581421,
      "logits/chosen": -0.984179675579071,
      "logits/rejected": -0.923632800579071,
      "logps/chosen": -0.750195324420929,
      "logps/rejected": -1.2976562976837158,
      "loss": 1.0689,
      "nll_loss": 1.0556640625,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07501220703125,
      "rewards/margins": 0.05474243313074112,
      "rewards/rejected": -0.12978515028953552,
      "step": 1010
    },
    {
      "epoch": 0.07740173015632114,
      "grad_norm": 1.6143279270234796,
      "learning_rate": 2.504897164340598e-06,
      "log_odds_chosen": 0.7210693359375,
      "log_odds_ratio": -0.561718761920929,
      "logits/chosen": -0.991015613079071,
      "logits/rejected": -0.899609386920929,
      "logps/chosen": -0.723925769329071,
      "logps/rejected": -1.206640601158142,
      "loss": 1.0968,
      "nll_loss": 1.0457031726837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07241211086511612,
      "rewards/margins": 0.04823608323931694,
      "rewards/rejected": -0.12058105319738388,
      "step": 1020
    },
    {
      "epoch": 0.07816057064804978,
      "grad_norm": 1.5764818072728237,
      "learning_rate": 2.492707811399023e-06,
      "log_odds_chosen": 1.0046875476837158,
      "log_odds_ratio": -0.4649414122104645,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.9419921636581421,
      "logps/chosen": -0.6382812261581421,
      "logps/rejected": -1.292578101158142,
      "loss": 1.0769,
      "nll_loss": 0.949999988079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06381835788488388,
      "rewards/margins": 0.06541137397289276,
      "rewards/rejected": -0.12924805283546448,
      "step": 1030
    },
    {
      "epoch": 0.07891941113977842,
      "grad_norm": 1.558748180429937,
      "learning_rate": 2.480694691784169e-06,
      "log_odds_chosen": 0.866259753704071,
      "log_odds_ratio": -0.484375,
      "logits/chosen": -0.9994140863418579,
      "logits/rejected": -0.906445324420929,
      "logps/chosen": -0.678906261920929,
      "logps/rejected": -1.224218726158142,
      "loss": 1.1036,
      "nll_loss": 1.056640625,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06789550930261612,
      "rewards/margins": 0.05458679050207138,
      "rewards/rejected": -0.12250976264476776,
      "step": 1040
    },
    {
      "epoch": 0.07967825163150706,
      "grad_norm": 1.458824951812319,
      "learning_rate": 2.4688535993934706e-06,
      "log_odds_chosen": 0.820385754108429,
      "log_odds_ratio": -0.5077148675918579,
      "logits/chosen": -1.006445288658142,
      "logits/rejected": -0.925976574420929,
      "logps/chosen": -0.7259765863418579,
      "logps/rejected": -1.2607421875,
      "loss": 1.072,
      "nll_loss": 0.988476574420929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07264403998851776,
      "rewards/margins": 0.05347289890050888,
      "rewards/rejected": -0.12607422471046448,
      "step": 1050
    },
    {
      "epoch": 0.0804370921232357,
      "grad_norm": 1.6596203290820475,
      "learning_rate": 2.457180467335805e-06,
      "log_odds_chosen": 0.642260730266571,
      "log_odds_ratio": -0.6029297113418579,
      "logits/chosen": -0.9710937738418579,
      "logits/rejected": -0.8902343511581421,
      "logps/chosen": -0.794921875,
      "logps/rejected": -1.2492187023162842,
      "loss": 1.0757,
      "nll_loss": 1.033203125,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.0794677734375,
      "rewards/margins": 0.04537353664636612,
      "rewards/rejected": -0.12482909858226776,
      "step": 1060
    },
    {
      "epoch": 0.08119593261496433,
      "grad_norm": 2.2419464467091217,
      "learning_rate": 2.4456713620629725e-06,
      "log_odds_chosen": 0.9271484613418579,
      "log_odds_ratio": -0.5121093988418579,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.878125011920929,
      "logps/chosen": -0.707812488079071,
      "logps/rejected": -1.287109375,
      "loss": 1.1071,
      "nll_loss": 1.131250023841858,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07080078125,
      "rewards/margins": 0.05796203762292862,
      "rewards/rejected": -0.128662109375,
      "step": 1070
    },
    {
      "epoch": 0.08195477310669297,
      "grad_norm": 1.5134269175876427,
      "learning_rate": 2.4343224778007378e-06,
      "log_odds_chosen": 0.7594238519668579,
      "log_odds_ratio": -0.514941394329071,
      "logits/chosen": -0.9462890625,
      "logits/rejected": -0.8998047113418579,
      "logps/chosen": -0.724609375,
      "logps/rejected": -1.2138671875,
      "loss": 1.0804,
      "nll_loss": 1.0427734851837158,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07246093451976776,
      "rewards/margins": 0.04888305813074112,
      "rewards/rejected": -0.12131347507238388,
      "step": 1080
    },
    {
      "epoch": 0.08271361359842161,
      "grad_norm": 1.525216640704541,
      "learning_rate": 2.4231301312615306e-06,
      "log_odds_chosen": 0.975512683391571,
      "log_odds_ratio": -0.5089355707168579,
      "logits/chosen": -0.9750000238418579,
      "logits/rejected": -0.8974609375,
      "logps/chosen": -0.7203124761581421,
      "logps/rejected": -1.395898461341858,
      "loss": 1.0719,
      "nll_loss": 0.993359386920929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07199706882238388,
      "rewards/margins": 0.067596435546875,
      "rewards/rejected": -0.13955077528953552,
      "step": 1090
    },
    {
      "epoch": 0.08347245409015025,
      "grad_norm": 1.4027404178012934,
      "learning_rate": 2.412090756622109e-06,
      "log_odds_chosen": 0.7673705816268921,
      "log_odds_ratio": -0.5249999761581421,
      "logits/chosen": -0.951367199420929,
      "logits/rejected": -0.884960949420929,
      "logps/chosen": -0.738574206829071,
      "logps/rejected": -1.2646484375,
      "loss": 1.0735,
      "nll_loss": 0.994921863079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07377929985523224,
      "rewards/margins": 0.05257873609662056,
      "rewards/rejected": -0.12646484375,
      "step": 1100
    },
    {
      "epoch": 0.08423129458187889,
      "grad_norm": 1.6015960818651562,
      "learning_rate": 2.401200900750657e-06,
      "log_odds_chosen": 0.680188000202179,
      "log_odds_ratio": -0.5425781011581421,
      "logits/chosen": -0.969531238079071,
      "logits/rejected": -0.9166015386581421,
      "logps/chosen": -0.766406238079071,
      "logps/rejected": -1.244531273841858,
      "loss": 1.0568,
      "nll_loss": 0.9955078363418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07664795219898224,
      "rewards/margins": 0.04779357835650444,
      "rewards/rejected": -0.12448730319738388,
      "step": 1110
    },
    {
      "epoch": 0.08499013507360753,
      "grad_norm": 1.4462554991694485,
      "learning_rate": 2.390457218668787e-06,
      "log_odds_chosen": 0.910449206829071,
      "log_odds_ratio": -0.4771484434604645,
      "logits/chosen": -0.9351562261581421,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.7552734613418579,
      "logps/rejected": -1.367773413658142,
      "loss": 1.0932,
      "nll_loss": 1.1320312023162842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07550048828125,
      "rewards/margins": 0.06117553636431694,
      "rewards/rejected": -0.1365966796875,
      "step": 1120
    },
    {
      "epoch": 0.08574897556533617,
      "grad_norm": 1.3771017614094005,
      "learning_rate": 2.379856469234918e-06,
      "log_odds_chosen": 0.89666748046875,
      "log_odds_ratio": -0.4830078184604645,
      "logits/chosen": -0.9310547113418579,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.725781261920929,
      "logps/rejected": -1.322656273841858,
      "loss": 1.0739,
      "nll_loss": 1.034765601158142,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07258300483226776,
      "rewards/margins": 0.05972747877240181,
      "rewards/rejected": -0.13229981064796448,
      "step": 1130
    },
    {
      "epoch": 0.0865078160570648,
      "grad_norm": 1.5724289030509337,
      "learning_rate": 2.369395511036369e-06,
      "log_odds_chosen": 0.9185791015625,
      "log_odds_ratio": -0.48344725370407104,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.904101550579071,
      "logps/chosen": -0.7250000238418579,
      "logps/rejected": -1.345703125,
      "loss": 1.0835,
      "nll_loss": 1.1042969226837158,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07246093451976776,
      "rewards/margins": 0.0621337890625,
      "rewards/rejected": -0.13469238579273224,
      "step": 1140
    },
    {
      "epoch": 0.08726665654879344,
      "grad_norm": 1.5105580372447085,
      "learning_rate": 2.359071298478354e-06,
      "log_odds_chosen": 0.7783447504043579,
      "log_odds_ratio": -0.5380859375,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.8990234136581421,
      "logps/chosen": -0.73828125,
      "logps/rejected": -1.2673828601837158,
      "loss": 1.0531,
      "nll_loss": 1.0275390148162842,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07381591945886612,
      "rewards/margins": 0.05291900783777237,
      "rewards/rejected": -0.12678222358226776,
      "step": 1150
    },
    {
      "epoch": 0.08802549704052208,
      "grad_norm": 1.5063726205865329,
      "learning_rate": 2.3488808780588137e-06,
      "log_odds_chosen": 0.800000011920929,
      "log_odds_ratio": -0.5308593511581421,
      "logits/chosen": -0.9722656011581421,
      "logits/rejected": -0.8876953125,
      "logps/chosen": -0.73486328125,
      "logps/rejected": -1.263671875,
      "loss": 1.0574,
      "nll_loss": 1.08984375,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07351074367761612,
      "rewards/margins": 0.05285186693072319,
      "rewards/rejected": -0.12639160454273224,
      "step": 1160
    },
    {
      "epoch": 0.08878433753225072,
      "grad_norm": 1.640218129700523,
      "learning_rate": 2.3388213848187446e-06,
      "log_odds_chosen": 0.7334228754043579,
      "log_odds_ratio": -0.54052734375,
      "logits/chosen": -0.974414050579071,
      "logits/rejected": -0.880078136920929,
      "logps/chosen": -0.7416015863418579,
      "logps/rejected": -1.2433593273162842,
      "loss": 1.0795,
      "nll_loss": 0.9974609613418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07415771484375,
      "rewards/margins": 0.05020294338464737,
      "rewards/rejected": -0.12434081733226776,
      "step": 1170
    },
    {
      "epoch": 0.08954317802397936,
      "grad_norm": 1.4607546113706387,
      "learning_rate": 2.328890038958328e-06,
      "log_odds_chosen": 0.8072265386581421,
      "log_odds_ratio": -0.529101550579071,
      "logits/chosen": -0.927929699420929,
      "logits/rejected": -0.8916015625,
      "logps/chosen": -0.747265636920929,
      "logps/rejected": -1.2888672351837158,
      "loss": 1.084,
      "nll_loss": 1.023828148841858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07471923530101776,
      "rewards/margins": 0.05414123460650444,
      "rewards/rejected": -0.12890625,
      "step": 1180
    },
    {
      "epoch": 0.090302018515708,
      "grad_norm": 1.590480942561905,
      "learning_rate": 2.3190841426097937e-06,
      "log_odds_chosen": 0.7764648199081421,
      "log_odds_ratio": -0.4970703125,
      "logits/chosen": -0.935546875,
      "logits/rejected": -0.858593761920929,
      "logps/chosen": -0.7076171636581421,
      "logps/rejected": -1.2365233898162842,
      "loss": 1.0711,
      "nll_loss": 1.057031273841858,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07077636569738388,
      "rewards/margins": 0.05285339429974556,
      "rewards/rejected": -0.12360839545726776,
      "step": 1190
    },
    {
      "epoch": 0.09106085900743664,
      "grad_norm": 1.5911920566602926,
      "learning_rate": 2.309401076758503e-06,
      "log_odds_chosen": 0.7508484125137329,
      "log_odds_ratio": -0.55712890625,
      "logits/chosen": -1.025781273841858,
      "logits/rejected": -0.9849609136581421,
      "logps/chosen": -0.7730468511581421,
      "logps/rejected": -1.286718726158142,
      "loss": 1.0467,
      "nll_loss": 1.0251953601837158,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07729492336511612,
      "rewards/margins": 0.05129394680261612,
      "rewards/rejected": -0.12856444716453552,
      "step": 1200
    },
    {
      "epoch": 0.09181969949916527,
      "grad_norm": 1.6321906040324805,
      "learning_rate": 2.299838298304276e-06,
      "log_odds_chosen": 0.943359375,
      "log_odds_ratio": -0.474853515625,
      "logits/chosen": -0.9476562738418579,
      "logits/rejected": -0.896679699420929,
      "logps/chosen": -0.692578136920929,
      "logps/rejected": -1.282812476158142,
      "loss": 1.0844,
      "nll_loss": 1.0271484851837158,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06927490234375,
      "rewards/margins": 0.05902709811925888,
      "rewards/rejected": -0.12834472954273224,
      "step": 1210
    },
    {
      "epoch": 0.09257853999089391,
      "grad_norm": 1.7462695798120413,
      "learning_rate": 2.2903933372554728e-06,
      "log_odds_chosen": 0.7892090082168579,
      "log_odds_ratio": -0.5526367425918579,
      "logits/chosen": -1.009765625,
      "logits/rejected": -0.9203125238418579,
      "logps/chosen": -0.761425793170929,
      "logps/rejected": -1.3087890148162842,
      "loss": 1.0657,
      "nll_loss": 1.0515625476837158,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07618407905101776,
      "rewards/margins": 0.05482025071978569,
      "rewards/rejected": -0.1309814453125,
      "step": 1220
    },
    {
      "epoch": 0.09333738048262255,
      "grad_norm": 1.5708827607202138,
      "learning_rate": 2.281063794048804e-06,
      "log_odds_chosen": 0.9153808355331421,
      "log_odds_ratio": -0.5028320550918579,
      "logits/chosen": -0.988476574420929,
      "logits/rejected": -0.9078124761581421,
      "logps/chosen": -0.7201172113418579,
      "logps/rejected": -1.3582031726837158,
      "loss": 1.0552,
      "nll_loss": 0.9388672113418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07205810397863388,
      "rewards/margins": 0.06379318237304688,
      "rewards/rejected": -0.13588866591453552,
      "step": 1230
    },
    {
      "epoch": 0.09409622097435119,
      "grad_norm": 1.4432294029947919,
      "learning_rate": 2.271847336988259e-06,
      "log_odds_chosen": 0.806347668170929,
      "log_odds_ratio": -0.517871081829071,
      "logits/chosen": -1.0128905773162842,
      "logits/rejected": -0.901562511920929,
      "logps/chosen": -0.7451171875,
      "logps/rejected": -1.31640625,
      "loss": 1.0815,
      "nll_loss": 0.957226574420929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07449951022863388,
      "rewards/margins": 0.05713500827550888,
      "rewards/rejected": -0.13168945908546448,
      "step": 1240
    },
    {
      "epoch": 0.09485506146607983,
      "grad_norm": 1.5543647286170723,
      "learning_rate": 2.262741699796952e-06,
      "log_odds_chosen": 0.958984375,
      "log_odds_ratio": -0.501269519329071,
      "logits/chosen": -0.9544922113418579,
      "logits/rejected": -0.9039062261581421,
      "logps/chosen": -0.7734375,
      "logps/rejected": -1.426171898841858,
      "loss": 1.0468,
      "nll_loss": 1.048437476158142,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07730712741613388,
      "rewards/margins": 0.06527099758386612,
      "rewards/rejected": -0.14252929389476776,
      "step": 1250
    },
    {
      "epoch": 0.09561390195780847,
      "grad_norm": 1.674123203497309,
      "learning_rate": 2.253744679276044e-06,
      "log_odds_chosen": 0.7455810308456421,
      "log_odds_ratio": -0.524609386920929,
      "logits/chosen": -0.959179699420929,
      "logits/rejected": -0.8783203363418579,
      "logps/chosen": -0.7276366949081421,
      "logps/rejected": -1.2091796398162842,
      "loss": 1.0556,
      "nll_loss": 1.010156273841858,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07276611030101776,
      "rewards/margins": 0.04802246019244194,
      "rewards/rejected": -0.12080077826976776,
      "step": 1260
    },
    {
      "epoch": 0.09637274244953711,
      "grad_norm": 1.5730945328477453,
      "learning_rate": 2.244854133065255e-06,
      "log_odds_chosen": 0.91387939453125,
      "log_odds_ratio": -0.529248058795929,
      "logits/chosen": -0.990234375,
      "logits/rejected": -0.9085937738418579,
      "logps/chosen": -0.8076171875,
      "logps/rejected": -1.453515648841858,
      "loss": 1.0977,
      "nll_loss": 1.0802733898162842,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.08072509616613388,
      "rewards/margins": 0.06455688178539276,
      "rewards/rejected": -0.14531250298023224,
      "step": 1270
    },
    {
      "epoch": 0.09713158294126574,
      "grad_norm": 1.564022477389795,
      "learning_rate": 2.2360679774997895e-06,
      "log_odds_chosen": 0.7984374761581421,
      "log_odds_ratio": -0.5282226800918579,
      "logits/chosen": -0.9302734136581421,
      "logits/rejected": -0.863476574420929,
      "logps/chosen": -0.705859363079071,
      "logps/rejected": -1.2312500476837158,
      "loss": 1.0847,
      "nll_loss": 1.0154297351837158,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07062987983226776,
      "rewards/margins": 0.05250854417681694,
      "rewards/rejected": -0.12302245944738388,
      "step": 1280
    },
    {
      "epoch": 0.09789042343299438,
      "grad_norm": 1.498086269280391,
      "learning_rate": 2.2273841855588183e-06,
      "log_odds_chosen": 0.8680175542831421,
      "log_odds_ratio": -0.501660168170929,
      "logits/chosen": -0.9849609136581421,
      "logits/rejected": -0.880078136920929,
      "logps/chosen": -0.716601550579071,
      "logps/rejected": -1.306640625,
      "loss": 1.0609,
      "nll_loss": 1.064453125,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07170410454273224,
      "rewards/margins": 0.05905456468462944,
      "rewards/rejected": -0.1307373046875,
      "step": 1290
    },
    {
      "epoch": 0.09864926392472302,
      "grad_norm": 1.7728116233156204,
      "learning_rate": 2.2188007849009167e-06,
      "log_odds_chosen": 0.8995116949081421,
      "log_odds_ratio": -0.5074707269668579,
      "logits/chosen": -0.966992199420929,
      "logits/rejected": -0.923046886920929,
      "logps/chosen": -0.740039050579071,
      "logps/rejected": -1.3748047351837158,
      "loss": 1.0752,
      "nll_loss": 1.068750023841858,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07395019382238388,
      "rewards/margins": 0.06353454291820526,
      "rewards/rejected": -0.13752441108226776,
      "step": 1300
    },
    {
      "epoch": 0.09940810441645166,
      "grad_norm": 1.7158654014174184,
      "learning_rate": 2.2103158559821502e-06,
      "log_odds_chosen": 1.112890601158142,
      "log_odds_ratio": -0.43603515625,
      "logits/chosen": -0.960156261920929,
      "logits/rejected": -0.853320300579071,
      "logps/chosen": -0.6685546636581421,
      "logps/rejected": -1.397070288658142,
      "loss": 1.03,
      "nll_loss": 0.949999988079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06690673530101776,
      "rewards/margins": 0.072845458984375,
      "rewards/rejected": -0.13969726860523224,
      "step": 1310
    },
    {
      "epoch": 0.1001669449081803,
      "grad_norm": 1.7372509904613194,
      "learning_rate": 2.2019275302527213e-06,
      "log_odds_chosen": 1.05224609375,
      "log_odds_ratio": -0.4795898497104645,
      "logits/chosen": -0.982421875,
      "logits/rejected": -0.8744140863418579,
      "logps/chosen": -0.6839843988418579,
      "logps/rejected": -1.416406273841858,
      "loss": 1.0835,
      "nll_loss": 0.9994140863418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06842041015625,
      "rewards/margins": 0.07326050102710724,
      "rewards/rejected": -0.1417236328125,
      "step": 1320
    },
    {
      "epoch": 0.10092578539990894,
      "grad_norm": 1.5946406192046245,
      "learning_rate": 2.193633988428327e-06,
      "log_odds_chosen": 0.7843261957168579,
      "log_odds_ratio": -0.521679699420929,
      "logits/chosen": -0.9453125,
      "logits/rejected": -0.89453125,
      "logps/chosen": -0.7162109613418579,
      "logps/rejected": -1.2314453125,
      "loss": 1.0827,
      "nll_loss": 1.0789062976837158,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07159423828125,
      "rewards/margins": 0.05157775804400444,
      "rewards/rejected": -0.12312011420726776,
      "step": 1330
    },
    {
      "epoch": 0.10168462589163758,
      "grad_norm": 1.4243961328613546,
      "learning_rate": 2.185433458832612e-06,
      "log_odds_chosen": 0.6023925542831421,
      "log_odds_ratio": -0.574999988079071,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.9302734136581421,
      "logps/chosen": -0.713085949420929,
      "logps/rejected": -1.095703125,
      "loss": 1.0959,
      "nll_loss": 1.042382836341858,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.0712890625,
      "rewards/margins": 0.03832244873046875,
      "rewards/rejected": -0.10966797173023224,
      "step": 1340
    },
    {
      "epoch": 0.10244346638336621,
      "grad_norm": 1.495406382498458,
      "learning_rate": 2.177324215807269e-06,
      "log_odds_chosen": 1.001367211341858,
      "log_odds_ratio": -0.441650390625,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.881640613079071,
      "logps/chosen": -0.700488269329071,
      "logps/rejected": -1.345312476158142,
      "loss": 1.0643,
      "nll_loss": 1.0226562023162842,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07004394382238388,
      "rewards/margins": 0.064483642578125,
      "rewards/rejected": -0.13447265326976776,
      "step": 1350
    },
    {
      "epoch": 0.10320230687509485,
      "grad_norm": 1.4865252925204064,
      "learning_rate": 2.1693045781865616e-06,
      "log_odds_chosen": 1.048974633216858,
      "log_odds_ratio": -0.4736328125,
      "logits/chosen": -0.910937488079071,
      "logits/rejected": -0.861328125,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -1.437109351158142,
      "loss": 1.0463,
      "nll_loss": 1.000390648841858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07167968899011612,
      "rewards/margins": 0.07203521579504013,
      "rewards/rejected": -0.14365234971046448,
      "step": 1360
    },
    {
      "epoch": 0.10396114736682349,
      "grad_norm": 2.887947200974786,
      "learning_rate": 2.1613729078331965e-06,
      "log_odds_chosen": 0.974902331829071,
      "log_odds_ratio": -0.46269530057907104,
      "logits/chosen": -0.9169921875,
      "logits/rejected": -0.8324218988418579,
      "logps/chosen": -0.7001953125,
      "logps/rejected": -1.377343773841858,
      "loss": 1.0644,
      "nll_loss": 0.994140625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07004394382238388,
      "rewards/margins": 0.06776733696460724,
      "rewards/rejected": -0.13779297471046448,
      "step": 1370
    },
    {
      "epoch": 0.10471998785855213,
      "grad_norm": 1.5824770881296906,
      "learning_rate": 2.1535276082326617e-06,
      "log_odds_chosen": 0.932446300983429,
      "log_odds_ratio": -0.5478515625,
      "logits/chosen": -0.9814453125,
      "logits/rejected": -0.9380859136581421,
      "logps/chosen": -0.7708984613418579,
      "logps/rejected": -1.4343750476837158,
      "loss": 1.0571,
      "nll_loss": 1.02734375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07706298679113388,
      "rewards/margins": 0.06638336181640625,
      "rewards/rejected": -0.14340820908546448,
      "step": 1380
    },
    {
      "epoch": 0.10547882835028077,
      "grad_norm": 2.06581191948831,
      "learning_rate": 2.14576712314328e-06,
      "log_odds_chosen": 0.946362316608429,
      "log_odds_ratio": -0.5375000238418579,
      "logits/chosen": -0.9765625,
      "logits/rejected": -0.904101550579071,
      "logps/chosen": -0.7408202886581421,
      "logps/rejected": -1.4210937023162842,
      "loss": 1.0305,
      "nll_loss": 0.9906250238418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.0740966796875,
      "rewards/margins": 0.06792907416820526,
      "rewards/rejected": -0.14211425185203552,
      "step": 1390
    },
    {
      "epoch": 0.10623766884200941,
      "grad_norm": 1.503598713934619,
      "learning_rate": 2.138089935299395e-06,
      "log_odds_chosen": 1.009765625,
      "log_odds_ratio": -0.4737792909145355,
      "logits/chosen": -0.95703125,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.7177734375,
      "logps/rejected": -1.396875023841858,
      "loss": 1.0736,
      "nll_loss": 1.0187499523162842,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07183837890625,
      "rewards/margins": 0.06788787990808487,
      "rewards/rejected": -0.13967284560203552,
      "step": 1400
    },
    {
      "epoch": 0.10699650933373805,
      "grad_norm": 1.492803973103788,
      "learning_rate": 2.1304945651652297e-06,
      "log_odds_chosen": 0.8885742425918579,
      "log_odds_ratio": -0.538867175579071,
      "logits/chosen": -0.9580078125,
      "logits/rejected": -0.864062488079071,
      "logps/chosen": -0.7298828363418579,
      "logps/rejected": -1.323632836341858,
      "loss": 1.0644,
      "nll_loss": 1.1648437976837158,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07294921576976776,
      "rewards/margins": 0.05940551683306694,
      "rewards/rejected": -0.13237304985523224,
      "step": 1410
    },
    {
      "epoch": 0.10775534982546668,
      "grad_norm": 1.665221406365988,
      "learning_rate": 2.122979569737101e-06,
      "log_odds_chosen": 0.9991210699081421,
      "log_odds_ratio": -0.47651368379592896,
      "logits/chosen": -1.0068359375,
      "logits/rejected": -0.9341796636581421,
      "logps/chosen": -0.6558593511581421,
      "logps/rejected": -1.295507788658142,
      "loss": 1.0489,
      "nll_loss": 0.9662109613418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0655517578125,
      "rewards/margins": 0.06397704780101776,
      "rewards/rejected": -0.12954100966453552,
      "step": 1420
    },
    {
      "epoch": 0.10851419031719532,
      "grad_norm": 2.0746542535178603,
      "learning_rate": 2.11554354139178e-06,
      "log_odds_chosen": 0.9710448980331421,
      "log_odds_ratio": -0.5282226800918579,
      "logits/chosen": -0.9888671636581421,
      "logits/rejected": -0.9351562261581421,
      "logps/chosen": -0.8203125,
      "logps/rejected": -1.509765625,
      "loss": 1.0624,
      "nll_loss": 1.026953101158142,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.08211670070886612,
      "rewards/margins": 0.06894836574792862,
      "rewards/rejected": -0.15102538466453552,
      "step": 1430
    },
    {
      "epoch": 0.10927303080892396,
      "grad_norm": 1.592392453133004,
      "learning_rate": 2.1081851067789196e-06,
      "log_odds_chosen": 0.970703125,
      "log_odds_ratio": -0.47978514432907104,
      "logits/chosen": -0.912109375,
      "logits/rejected": -0.851367175579071,
      "logps/chosen": -0.677734375,
      "logps/rejected": -1.3093750476837158,
      "loss": 1.0409,
      "nll_loss": 0.966796875,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0677490234375,
      "rewards/margins": 0.06310882419347763,
      "rewards/rejected": -0.13095703721046448,
      "step": 1440
    },
    {
      "epoch": 0.1100318713006526,
      "grad_norm": 1.622693092846365,
      "learning_rate": 2.1009029257555606e-06,
      "log_odds_chosen": 0.923657238483429,
      "log_odds_ratio": -0.487548828125,
      "logits/chosen": -0.943164050579071,
      "logits/rejected": -0.868359386920929,
      "logps/chosen": -0.6958984136581421,
      "logps/rejected": -1.3054687976837158,
      "loss": 1.0425,
      "nll_loss": 1.00390625,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06960449367761612,
      "rewards/margins": 0.06094970554113388,
      "rewards/rejected": -0.13059082627296448,
      "step": 1450
    },
    {
      "epoch": 0.11079071179238124,
      "grad_norm": 5.009691330274977,
      "learning_rate": 2.0936956903608545e-06,
      "log_odds_chosen": 0.996289074420929,
      "log_odds_ratio": -0.47612303495407104,
      "logits/chosen": -0.931835949420929,
      "logits/rejected": -0.8363281488418579,
      "logps/chosen": -0.7001953125,
      "logps/rejected": -1.349218726158142,
      "loss": 1.0934,
      "nll_loss": 1.058007836341858,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07009277492761612,
      "rewards/margins": 0.06481323391199112,
      "rewards/rejected": -0.13486328721046448,
      "step": 1460
    },
    {
      "epoch": 0.11154955228410988,
      "grad_norm": 1.5133282112647446,
      "learning_rate": 2.0865621238292046e-06,
      "log_odds_chosen": 0.929492175579071,
      "log_odds_ratio": -0.4931640625,
      "logits/chosen": -0.900390625,
      "logits/rejected": -0.8248046636581421,
      "logps/chosen": -0.71484375,
      "logps/rejected": -1.3298828601837158,
      "loss": 1.0446,
      "nll_loss": 1.064062476158142,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.071533203125,
      "rewards/margins": 0.06144409254193306,
      "rewards/rejected": -0.13300780951976776,
      "step": 1470
    },
    {
      "epoch": 0.11230839277583853,
      "grad_norm": 1.6448887742190683,
      "learning_rate": 2.079500979640145e-06,
      "log_odds_chosen": 0.8451293706893921,
      "log_odds_ratio": -0.503125011920929,
      "logits/chosen": -0.9248046875,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.701855480670929,
      "logps/rejected": -1.2658202648162842,
      "loss": 1.0617,
      "nll_loss": 0.9662109613418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0701904296875,
      "rewards/margins": 0.05644073337316513,
      "rewards/rejected": -0.12651367485523224,
      "step": 1480
    },
    {
      "epoch": 0.11306723326756715,
      "grad_norm": 1.6079772952545273,
      "learning_rate": 2.072511040603359e-06,
      "log_odds_chosen": 0.8741699457168579,
      "log_odds_ratio": -0.5518554449081421,
      "logits/chosen": -0.915820300579071,
      "logits/rejected": -0.830859363079071,
      "logps/chosen": -0.749804675579071,
      "logps/rejected": -1.3640625476837158,
      "loss": 1.0472,
      "nll_loss": 0.9970703125,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07498779147863388,
      "rewards/margins": 0.061373136937618256,
      "rewards/rejected": -0.13645020127296448,
      "step": 1490
    },
    {
      "epoch": 0.11382607375929579,
      "grad_norm": 1.5317978336165587,
      "learning_rate": 2.065591117977289e-06,
      "log_odds_chosen": 0.8371948003768921,
      "log_odds_ratio": -0.5206054449081421,
      "logits/chosen": -0.9642578363418579,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.7547851800918579,
      "logps/rejected": -1.3203125,
      "loss": 1.0712,
      "nll_loss": 1.0656249523162842,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07547607272863388,
      "rewards/margins": 0.05657196044921875,
      "rewards/rejected": -0.13210448622703552,
      "step": 1500
    },
    {
      "epoch": 0.11458491425102443,
      "grad_norm": 2.3627141683478374,
      "learning_rate": 2.058740050619915e-06,
      "log_odds_chosen": 1.017968773841858,
      "log_odds_ratio": -0.46826171875,
      "logits/chosen": -1.0009765625,
      "logits/rejected": -0.8857421875,
      "logps/chosen": -0.7376953363418579,
      "logps/rejected": -1.4552733898162842,
      "loss": 1.0386,
      "nll_loss": 0.9716796875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07375488430261612,
      "rewards/margins": 0.07177734375,
      "rewards/rejected": -0.1455078125,
      "step": 1510
    },
    {
      "epoch": 0.11534375474275307,
      "grad_norm": 1.8229312547459466,
      "learning_rate": 2.0519567041703083e-06,
      "log_odds_chosen": 0.9739745855331421,
      "log_odds_ratio": -0.4912109375,
      "logits/chosen": -0.9917968511581421,
      "logits/rejected": -0.9136718511581421,
      "logps/chosen": -0.746874988079071,
      "logps/rejected": -1.441796898841858,
      "loss": 1.0527,
      "nll_loss": 0.9945312738418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07463379204273224,
      "rewards/margins": 0.069488525390625,
      "rewards/rejected": -0.14421387016773224,
      "step": 1520
    },
    {
      "epoch": 0.11610259523448171,
      "grad_norm": 1.781377185920502,
      "learning_rate": 2.0452399702596544e-06,
      "log_odds_chosen": 0.925097644329071,
      "log_odds_ratio": -0.48115235567092896,
      "logits/chosen": -0.938281238079071,
      "logits/rejected": -0.8638671636581421,
      "logps/chosen": -0.7284179925918579,
      "logps/rejected": -1.318750023841858,
      "loss": 1.033,
      "nll_loss": 0.90234375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07282714545726776,
      "rewards/margins": 0.05898742750287056,
      "rewards/rejected": -0.1317138671875,
      "step": 1530
    },
    {
      "epoch": 0.11686143572621036,
      "grad_norm": 1.7487600964578844,
      "learning_rate": 2.0385887657505017e-06,
      "log_odds_chosen": 1.035742163658142,
      "log_odds_ratio": -0.4712890684604645,
      "logits/chosen": -0.9691406488418579,
      "logits/rejected": -0.9072265625,
      "logps/chosen": -0.739453136920929,
      "logps/rejected": -1.449804663658142,
      "loss": 1.0245,
      "nll_loss": 0.939453125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07387695461511612,
      "rewards/margins": 0.071044921875,
      "rewards/rejected": -0.14497070014476776,
      "step": 1540
    },
    {
      "epoch": 0.117620276217939,
      "grad_norm": 1.454469728477322,
      "learning_rate": 2.032002032003048e-06,
      "log_odds_chosen": 0.838940441608429,
      "log_odds_ratio": -0.51806640625,
      "logits/chosen": -0.9722656011581421,
      "logits/rejected": -0.8675781488418579,
      "logps/chosen": -0.70751953125,
      "logps/rejected": -1.2646484375,
      "loss": 1.0354,
      "nll_loss": 1.0078125,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07076416164636612,
      "rewards/margins": 0.05572357028722763,
      "rewards/rejected": -0.12663574516773224,
      "step": 1550
    },
    {
      "epoch": 0.11837911670966762,
      "grad_norm": 1.6474342799970592,
      "learning_rate": 2.025478734167333e-06,
      "log_odds_chosen": 0.848681628704071,
      "log_odds_ratio": -0.5018554925918579,
      "logits/chosen": -0.921875,
      "logits/rejected": -0.810351550579071,
      "logps/chosen": -0.7069336175918579,
      "logps/rejected": -1.231054663658142,
      "loss": 1.0514,
      "nll_loss": 0.983203113079071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07071533054113388,
      "rewards/margins": 0.05249633640050888,
      "rewards/rejected": -0.12314452975988388,
      "step": 1560
    },
    {
      "epoch": 0.11913795720139626,
      "grad_norm": 1.6449397249587745,
      "learning_rate": 2.0190178605002747e-06,
      "log_odds_chosen": 0.708850085735321,
      "log_odds_ratio": -0.52001953125,
      "logits/chosen": -0.9468749761581421,
      "logits/rejected": -0.879101574420929,
      "logps/chosen": -0.728320300579071,
      "logps/rejected": -1.185156226158142,
      "loss": 1.0435,
      "nll_loss": 0.990429699420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07283935695886612,
      "rewards/margins": 0.04575958102941513,
      "rewards/rejected": -0.11857910454273224,
      "step": 1570
    },
    {
      "epoch": 0.1198967976931249,
      "grad_norm": 1.5827948770408402,
      "learning_rate": 2.0126184217065104e-06,
      "log_odds_chosen": 1.0593750476837158,
      "log_odds_ratio": -0.46733397245407104,
      "logits/chosen": -1.004296898841858,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.705078125,
      "logps/rejected": -1.451562523841858,
      "loss": 1.0502,
      "nll_loss": 0.982226550579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07044677436351776,
      "rewards/margins": 0.07470397651195526,
      "rewards/rejected": -0.1451416015625,
      "step": 1580
    },
    {
      "epoch": 0.12065563818485354,
      "grad_norm": 1.6904017282178356,
      "learning_rate": 2.0062794503020765e-06,
      "log_odds_chosen": 1.045166015625,
      "log_odds_ratio": -0.44599610567092896,
      "logits/chosen": -0.9761718511581421,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.6561523675918579,
      "logps/rejected": -1.330078125,
      "loss": 1.0226,
      "nll_loss": 0.9820312261581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06564941257238388,
      "rewards/margins": 0.06737060844898224,
      "rewards/rejected": -0.13310547173023224,
      "step": 1590
    },
    {
      "epoch": 0.12141447867658219,
      "grad_norm": 1.668297709829456,
      "learning_rate": 2e-06,
      "log_odds_chosen": 1.0166747570037842,
      "log_odds_ratio": -0.4783691465854645,
      "logits/chosen": -0.9869140386581421,
      "logits/rejected": -0.8951171636581421,
      "logps/chosen": -0.7251952886581421,
      "logps/rejected": -1.4386718273162842,
      "loss": 1.0305,
      "nll_loss": 1.0388672351837158,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07248535007238388,
      "rewards/margins": 0.07139892876148224,
      "rewards/rejected": -0.14394530653953552,
      "step": 1600
    },
    {
      "epoch": 0.12217331916831083,
      "grad_norm": 1.6682675310046382,
      "learning_rate": 1.993779145116907e-06,
      "log_odds_chosen": 1.00390625,
      "log_odds_ratio": -0.4740234315395355,
      "logits/chosen": -0.9330078363418579,
      "logits/rejected": -0.853710949420929,
      "logps/chosen": -0.6610351800918579,
      "logps/rejected": -1.3292968273162842,
      "loss": 1.0441,
      "nll_loss": 1.013281226158142,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06612548977136612,
      "rewards/margins": 0.0667724609375,
      "rewards/rejected": -0.13295897841453552,
      "step": 1610
    },
    {
      "epoch": 0.12293215966003945,
      "grad_norm": 1.7474719643333534,
      "learning_rate": 1.987615979999813e-06,
      "log_odds_chosen": 0.926440417766571,
      "log_odds_ratio": -0.503369152545929,
      "logits/chosen": -0.912890613079071,
      "logits/rejected": -0.834179699420929,
      "logps/chosen": -0.751953125,
      "logps/rejected": -1.3935546875,
      "loss": 1.0461,
      "nll_loss": 1.0597655773162842,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07512207329273224,
      "rewards/margins": 0.06424407660961151,
      "rewards/rejected": -0.13930663466453552,
      "step": 1620
    },
    {
      "epoch": 0.1236910001517681,
      "grad_norm": 1.6993682853829732,
      "learning_rate": 1.9815096184722797e-06,
      "log_odds_chosen": 0.7530761957168579,
      "log_odds_ratio": -0.5435546636581421,
      "logits/chosen": -0.955273449420929,
      "logits/rejected": -0.853320300579071,
      "logps/chosen": -0.714062511920929,
      "logps/rejected": -1.2175781726837158,
      "loss": 1.0468,
      "nll_loss": 1.0089843273162842,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07138671725988388,
      "rewards/margins": 0.05036468431353569,
      "rewards/rejected": -0.12172851711511612,
      "step": 1630
    },
    {
      "epoch": 0.12444984064349673,
      "grad_norm": 1.6683755597518508,
      "learning_rate": 1.9754591932991793e-06,
      "log_odds_chosen": 1.045678734779358,
      "log_odds_ratio": -0.462646484375,
      "logits/chosen": -1.0167968273162842,
      "logits/rejected": -0.935546875,
      "logps/chosen": -0.717480480670929,
      "logps/rejected": -1.446874976158142,
      "loss": 1.0296,
      "nll_loss": 1.0029296875,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07177734375,
      "rewards/margins": 0.0728912353515625,
      "rewards/rejected": -0.1446533203125,
      "step": 1640
    },
    {
      "epoch": 0.12520868113522537,
      "grad_norm": 1.8852024815139483,
      "learning_rate": 1.9694638556693235e-06,
      "log_odds_chosen": 0.8907715082168579,
      "log_odds_ratio": -0.5005859136581421,
      "logits/chosen": -0.937304675579071,
      "logits/rejected": -0.86328125,
      "logps/chosen": -0.686328113079071,
      "logps/rejected": -1.248437523841858,
      "loss": 1.0571,
      "nll_loss": 0.938671886920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06870117038488388,
      "rewards/margins": 0.056203462183475494,
      "rewards/rejected": -0.12495116889476776,
      "step": 1650
    },
    {
      "epoch": 0.12596752162695402,
      "grad_norm": 1.6508004109020493,
      "learning_rate": 1.963522774695264e-06,
      "log_odds_chosen": 1.0453612804412842,
      "log_odds_ratio": -0.4659667909145355,
      "logits/chosen": -0.986132800579071,
      "logits/rejected": -0.87109375,
      "logps/chosen": -0.696972668170929,
      "logps/rejected": -1.4103515148162842,
      "loss": 1.0341,
      "nll_loss": 0.9345703125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06975097954273224,
      "rewards/margins": 0.0713348388671875,
      "rewards/rejected": -0.14116211235523224,
      "step": 1660
    },
    {
      "epoch": 0.12672636211868266,
      "grad_norm": 1.521239804311641,
      "learning_rate": 1.9576351369295853e-06,
      "log_odds_chosen": 1.26806640625,
      "log_odds_ratio": -0.4283203184604645,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.876757800579071,
      "logps/chosen": -0.6666015386581421,
      "logps/rejected": -1.5470702648162842,
      "loss": 1.0098,
      "nll_loss": 0.956835925579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06671142578125,
      "rewards/margins": 0.08796997368335724,
      "rewards/rejected": -0.15471191704273224,
      "step": 1670
    },
    {
      "epoch": 0.1274852026104113,
      "grad_norm": 1.5577811905592742,
      "learning_rate": 1.951800145897066e-06,
      "log_odds_chosen": 0.88671875,
      "log_odds_ratio": -0.5093749761581421,
      "logits/chosen": -0.9527343511581421,
      "logits/rejected": -0.8958984613418579,
      "logps/chosen": -0.727734386920929,
      "logps/rejected": -1.3302733898162842,
      "loss": 1.0335,
      "nll_loss": 1.004296898841858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07281494140625,
      "rewards/margins": 0.06018371507525444,
      "rewards/rejected": -0.13300780951976776,
      "step": 1680
    },
    {
      "epoch": 0.12824404310213994,
      "grad_norm": 1.7603246411817988,
      "learning_rate": 1.9460170216420797e-06,
      "log_odds_chosen": 0.936328113079071,
      "log_odds_ratio": -0.500732421875,
      "logits/chosen": -0.977343738079071,
      "logits/rejected": -0.8462890386581421,
      "logps/chosen": -0.71875,
      "logps/rejected": -1.3875000476837158,
      "loss": 1.0302,
      "nll_loss": 0.973828136920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07187499850988388,
      "rewards/margins": 0.06686095893383026,
      "rewards/rejected": -0.13859863579273224,
      "step": 1690
    },
    {
      "epoch": 0.12900288359386858,
      "grad_norm": 1.8509322922788338,
      "learning_rate": 1.9402850002906637e-06,
      "log_odds_chosen": 0.94775390625,
      "log_odds_ratio": -0.5142577886581421,
      "logits/chosen": -0.9898437261581421,
      "logits/rejected": -0.9117187261581421,
      "logps/chosen": -0.7020508050918579,
      "logps/rejected": -1.3449218273162842,
      "loss": 1.0089,
      "nll_loss": 0.9169921875,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.0701904296875,
      "rewards/margins": 0.06424560397863388,
      "rewards/rejected": -0.13444824516773224,
      "step": 1700
    },
    {
      "epoch": 0.12976172408559722,
      "grad_norm": 1.6167006582441545,
      "learning_rate": 1.9346033336266974e-06,
      "log_odds_chosen": 0.915332019329071,
      "log_odds_ratio": -0.5011230707168579,
      "logits/chosen": -0.997265636920929,
      "logits/rejected": -0.8988281488418579,
      "logps/chosen": -0.695117175579071,
      "logps/rejected": -1.2619140148162842,
      "loss": 1.0415,
      "nll_loss": 0.981640636920929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06951904296875,
      "rewards/margins": 0.05672607570886612,
      "rewards/rejected": -0.12624511122703552,
      "step": 1710
    },
    {
      "epoch": 0.13052056457732586,
      "grad_norm": 1.6591686774932126,
      "learning_rate": 1.9289712886816486e-06,
      "log_odds_chosen": 0.6905517578125,
      "log_odds_ratio": -0.588671863079071,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.923632800579071,
      "logps/chosen": -0.75,
      "logps/rejected": -1.218359351158142,
      "loss": 1.0406,
      "nll_loss": 0.9466797113418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07501220703125,
      "rewards/margins": 0.04682159423828125,
      "rewards/rejected": -0.12175293266773224,
      "step": 1720
    },
    {
      "epoch": 0.13127940506905447,
      "grad_norm": 1.585308028498714,
      "learning_rate": 1.92338814733738e-06,
      "log_odds_chosen": 1.0014159679412842,
      "log_odds_ratio": -0.4735351502895355,
      "logits/chosen": -0.9554687738418579,
      "logits/rejected": -0.865039050579071,
      "logps/chosen": -0.671679675579071,
      "logps/rejected": -1.2966797351837158,
      "loss": 1.0322,
      "nll_loss": 0.9839843511581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06717529147863388,
      "rewards/margins": 0.06257476657629013,
      "rewards/rejected": -0.12978515028953552,
      "step": 1730
    },
    {
      "epoch": 0.1320382455607831,
      "grad_norm": 1.7984044160498958,
      "learning_rate": 1.9178532059415367e-06,
      "log_odds_chosen": 0.91259765625,
      "log_odds_ratio": -0.5243164300918579,
      "logits/chosen": -0.9722656011581421,
      "logits/rejected": -0.883593738079071,
      "logps/chosen": -0.701953113079071,
      "logps/rejected": -1.354882836341858,
      "loss": 1.0332,
      "nll_loss": 0.9810546636581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0701904296875,
      "rewards/margins": 0.0653533935546875,
      "rewards/rejected": -0.13559570908546448,
      "step": 1740
    },
    {
      "epoch": 0.13279708605251175,
      "grad_norm": 1.6269570980620953,
      "learning_rate": 1.9123657749350298e-06,
      "log_odds_chosen": 0.9396728277206421,
      "log_odds_ratio": -0.529589831829071,
      "logits/chosen": -0.9664062261581421,
      "logits/rejected": -0.855273425579071,
      "logps/chosen": -0.6953125,
      "logps/rejected": -1.3576171398162842,
      "loss": 1.041,
      "nll_loss": 1.004296898841858,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06959228217601776,
      "rewards/margins": 0.0662078857421875,
      "rewards/rejected": -0.13571777939796448,
      "step": 1750
    },
    {
      "epoch": 0.1335559265442404,
      "grad_norm": 1.6259186946124007,
      "learning_rate": 1.9069251784911844e-06,
      "log_odds_chosen": 0.8407348394393921,
      "log_odds_ratio": -0.5379883050918579,
      "logits/chosen": -0.993945300579071,
      "logits/rejected": -0.9056640863418579,
      "logps/chosen": -0.70849609375,
      "logps/rejected": -1.2921874523162842,
      "loss": 1.0583,
      "nll_loss": 0.951953113079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07082519680261612,
      "rewards/margins": 0.05832824856042862,
      "rewards/rejected": -0.12924805283546448,
      "step": 1760
    },
    {
      "epoch": 0.13431476703596903,
      "grad_norm": 1.7328381362199747,
      "learning_rate": 1.9015307541661132e-06,
      "log_odds_chosen": 0.80621337890625,
      "log_odds_ratio": -0.551562488079071,
      "logits/chosen": -0.9144531488418579,
      "logits/rejected": -0.888867199420929,
      "logps/chosen": -0.720703125,
      "logps/rejected": -1.255273461341858,
      "loss": 1.0261,
      "nll_loss": 0.962109386920929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07207031548023224,
      "rewards/margins": 0.053497314453125,
      "rewards/rejected": -0.1256103515625,
      "step": 1770
    },
    {
      "epoch": 0.13507360752769768,
      "grad_norm": 1.6704809725658407,
      "learning_rate": 1.8961818525599089e-06,
      "log_odds_chosen": 0.929638683795929,
      "log_odds_ratio": -0.5340820550918579,
      "logits/chosen": -1.002343773841858,
      "logits/rejected": -0.9302734136581421,
      "logps/chosen": -0.705859363079071,
      "logps/rejected": -1.334375023841858,
      "loss": 1.0152,
      "nll_loss": 0.9947265386581421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07058105617761612,
      "rewards/margins": 0.06288299709558487,
      "rewards/rejected": -0.13349609076976776,
      "step": 1780
    },
    {
      "epoch": 0.13583244801942632,
      "grad_norm": 1.5345342962965025,
      "learning_rate": 1.890877836988262e-06,
      "log_odds_chosen": 0.905834972858429,
      "log_odds_ratio": -0.513378918170929,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.876171886920929,
      "logps/chosen": -0.7416015863418579,
      "logps/rejected": -1.358984351158142,
      "loss": 1.019,
      "nll_loss": 0.966015636920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07421875,
      "rewards/margins": 0.06169281154870987,
      "rewards/rejected": -0.13581542670726776,
      "step": 1790
    },
    {
      "epoch": 0.13659128851115496,
      "grad_norm": 1.6843206698869806,
      "learning_rate": 1.8856180831641269e-06,
      "log_odds_chosen": 1.1140625476837158,
      "log_odds_ratio": -0.4765625,
      "logits/chosen": -0.9654296636581421,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.7650390863418579,
      "logps/rejected": -1.559960961341858,
      "loss": 1.0252,
      "nll_loss": 1.0236327648162842,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07652588188648224,
      "rewards/margins": 0.07954101264476776,
      "rewards/rejected": -0.1561279296875,
      "step": 1800
    },
    {
      "epoch": 0.1373501290028836,
      "grad_norm": 1.5515481390991603,
      "learning_rate": 1.8804019788890737e-06,
      "log_odds_chosen": 1.1181640625,
      "log_odds_ratio": -0.4608398377895355,
      "logits/chosen": -0.9326171875,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.7177734375,
      "logps/rejected": -1.4988281726837158,
      "loss": 1.022,
      "nll_loss": 1.052734375,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07178954780101776,
      "rewards/margins": 0.07809142768383026,
      "rewards/rejected": -0.14985351264476776,
      "step": 1810
    },
    {
      "epoch": 0.13810896949461224,
      "grad_norm": 1.6376595591782022,
      "learning_rate": 1.8752289237539816e-06,
      "log_odds_chosen": 1.2386963367462158,
      "log_odds_ratio": -0.43408203125,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.8837890625,
      "logps/chosen": -0.698925793170929,
      "logps/rejected": -1.574609398841858,
      "loss": 1.0241,
      "nll_loss": 0.9544922113418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06990966945886612,
      "rewards/margins": 0.08756866306066513,
      "rewards/rejected": -0.15737304091453552,
      "step": 1820
    },
    {
      "epoch": 0.13886780998634088,
      "grad_norm": 1.7008481895449235,
      "learning_rate": 1.8700983288487376e-06,
      "log_odds_chosen": 1.112207055091858,
      "log_odds_ratio": -0.4563964903354645,
      "logits/chosen": -1.0037109851837158,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.701953113079071,
      "logps/rejected": -1.4744141101837158,
      "loss": 1.0151,
      "nll_loss": 0.9326171875,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07020263373851776,
      "rewards/margins": 0.0773773193359375,
      "rewards/rejected": -0.14755859971046448,
      "step": 1830
    },
    {
      "epoch": 0.13962665047806952,
      "grad_norm": 1.9175312329784642,
      "learning_rate": 1.8650096164806275e-06,
      "log_odds_chosen": 1.190039038658142,
      "log_odds_ratio": -0.4319824278354645,
      "logits/chosen": -0.9283202886581421,
      "logits/rejected": -0.8541015386581421,
      "logps/chosen": -0.649707019329071,
      "logps/rejected": -1.4619140625,
      "loss": 1.0337,
      "nll_loss": 0.9732421636581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06492920219898224,
      "rewards/margins": 0.08115844428539276,
      "rewards/rejected": -0.14606933295726776,
      "step": 1840
    },
    {
      "epoch": 0.14038549096979816,
      "grad_norm": 2.443144967546165,
      "learning_rate": 1.8599622199011084e-06,
      "log_odds_chosen": 1.020605444908142,
      "log_odds_ratio": -0.4903808534145355,
      "logits/chosen": -0.9642578363418579,
      "logits/rejected": -0.889843761920929,
      "logps/chosen": -0.6878906488418579,
      "logps/rejected": -1.416406273841858,
      "loss": 1.0489,
      "nll_loss": 0.9966796636581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06878662109375,
      "rewards/margins": 0.07291259616613388,
      "rewards/rejected": -0.1417236328125,
      "step": 1850
    },
    {
      "epoch": 0.1411443314615268,
      "grad_norm": 1.663086728989741,
      "learning_rate": 1.854955583040673e-06,
      "log_odds_chosen": 1.0613281726837158,
      "log_odds_ratio": -0.4869140684604645,
      "logits/chosen": -0.9642578363418579,
      "logits/rejected": -0.901171863079071,
      "logps/chosen": -0.7315429449081421,
      "logps/rejected": -1.4523437023162842,
      "loss": 0.9994,
      "nll_loss": 0.9876953363418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07315673679113388,
      "rewards/margins": 0.07206115871667862,
      "rewards/rejected": -0.14519043266773224,
      "step": 1860
    },
    {
      "epoch": 0.1419031719532554,
      "grad_norm": 1.9117980136195871,
      "learning_rate": 1.849989160251521e-06,
      "log_odds_chosen": 0.9891723394393921,
      "log_odds_ratio": -0.5174804925918579,
      "logits/chosen": -0.968554675579071,
      "logits/rejected": -0.898242175579071,
      "logps/chosen": -0.693554699420929,
      "logps/rejected": -1.387304663658142,
      "loss": 1.0329,
      "nll_loss": 0.9984375238418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06934814155101776,
      "rewards/margins": 0.06931991875171661,
      "rewards/rejected": -0.13864746689796448,
      "step": 1870
    },
    {
      "epoch": 0.14266201244498405,
      "grad_norm": 2.0856627792753577,
      "learning_rate": 1.8450624160577701e-06,
      "log_odds_chosen": 1.063867211341858,
      "log_odds_ratio": -0.49394530057907104,
      "logits/chosen": -1.012304663658142,
      "logits/rejected": -0.9169921875,
      "logps/chosen": -0.740234375,
      "logps/rejected": -1.5066406726837158,
      "loss": 1.007,
      "nll_loss": 1.001367211341858,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07404784858226776,
      "rewards/margins": 0.07670287787914276,
      "rewards/rejected": -0.15078124403953552,
      "step": 1880
    },
    {
      "epoch": 0.1434208529367127,
      "grad_norm": 1.576304279008401,
      "learning_rate": 1.8401748249129445e-06,
      "log_odds_chosen": 1.0471680164337158,
      "log_odds_ratio": -0.4671386778354645,
      "logits/chosen": -1.000390648841858,
      "logits/rejected": -0.910351574420929,
      "logps/chosen": -0.6888672113418579,
      "logps/rejected": -1.420507788658142,
      "loss": 1.015,
      "nll_loss": 0.985546886920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06890869140625,
      "rewards/margins": 0.0731201171875,
      "rewards/rejected": -0.14199218153953552,
      "step": 1890
    },
    {
      "epoch": 0.14417969342844134,
      "grad_norm": 2.2943205801406075,
      "learning_rate": 1.8353258709644938e-06,
      "log_odds_chosen": 1.0422852039337158,
      "log_odds_ratio": -0.4927734434604645,
      "logits/chosen": -0.955859363079071,
      "logits/rejected": -0.901562511920929,
      "logps/chosen": -0.7401367425918579,
      "logps/rejected": -1.451562523841858,
      "loss": 1.0349,
      "nll_loss": 1.0314452648162842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07406006008386612,
      "rewards/margins": 0.07120971381664276,
      "rewards/rejected": -0.14523926377296448,
      "step": 1900
    },
    {
      "epoch": 0.14493853392016998,
      "grad_norm": 2.0490705861246323,
      "learning_rate": 1.830515047825102e-06,
      "log_odds_chosen": 0.9310058355331421,
      "log_odds_ratio": -0.512890636920929,
      "logits/chosen": -0.9560546875,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.7147461175918579,
      "logps/rejected": -1.365625023841858,
      "loss": 1.0137,
      "nll_loss": 0.9736328125,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07149658352136612,
      "rewards/margins": 0.06497955322265625,
      "rewards/rejected": -0.13642577826976776,
      "step": 1910
    },
    {
      "epoch": 0.14569737441189862,
      "grad_norm": 1.7749586371627204,
      "learning_rate": 1.8257418583505536e-06,
      "log_odds_chosen": 1.212499976158142,
      "log_odds_ratio": -0.42607420682907104,
      "logits/chosen": -0.954882800579071,
      "logits/rejected": -0.861132800579071,
      "logps/chosen": -0.6689453125,
      "logps/rejected": -1.5017578601837158,
      "loss": 1.0019,
      "nll_loss": 0.922656238079071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06691894680261612,
      "rewards/margins": 0.08331298828125,
      "rewards/rejected": -0.15012207627296448,
      "step": 1920
    },
    {
      "epoch": 0.14645621490362726,
      "grad_norm": 1.6902217952090752,
      "learning_rate": 1.8210058144239416e-06,
      "log_odds_chosen": 0.8382812738418579,
      "log_odds_ratio": -0.5379883050918579,
      "logits/chosen": -0.9527343511581421,
      "logits/rejected": -0.8731445074081421,
      "logps/chosen": -0.73974609375,
      "logps/rejected": -1.356054663658142,
      "loss": 1.0088,
      "nll_loss": 1.009179711341858,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07390137016773224,
      "rewards/margins": 0.06163330003619194,
      "rewards/rejected": -0.13544921576976776,
      "step": 1930
    },
    {
      "epoch": 0.1472150553953559,
      "grad_norm": 1.8073168305465488,
      "learning_rate": 1.816306436745999e-06,
      "log_odds_chosen": 0.97314453125,
      "log_odds_ratio": -0.520800769329071,
      "logits/chosen": -0.963085949420929,
      "logits/rejected": -0.865429699420929,
      "logps/chosen": -0.755078136920929,
      "logps/rejected": -1.433984398841858,
      "loss": 1.0157,
      "nll_loss": 0.952929675579071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07548828423023224,
      "rewards/margins": 0.06783447414636612,
      "rewards/rejected": -0.143310546875,
      "step": 1940
    },
    {
      "epoch": 0.14797389588708454,
      "grad_norm": 2.1030165974817656,
      "learning_rate": 1.8116432546313529e-06,
      "log_odds_chosen": 0.807910144329071,
      "log_odds_ratio": -0.547119140625,
      "logits/chosen": -0.967968761920929,
      "logits/rejected": -0.854296863079071,
      "logps/chosen": -0.7210937738418579,
      "logps/rejected": -1.2625000476837158,
      "loss": 1.0078,
      "nll_loss": 0.974804699420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07205810397863388,
      "rewards/margins": 0.054046630859375,
      "rewards/rejected": -0.12607422471046448,
      "step": 1950
    },
    {
      "epoch": 0.14873273637881318,
      "grad_norm": 1.7734373496612974,
      "learning_rate": 1.8070158058105026e-06,
      "log_odds_chosen": 1.068139672279358,
      "log_odds_ratio": -0.47832030057907104,
      "logits/chosen": -0.9847656488418579,
      "logits/rejected": -0.9007812738418579,
      "logps/chosen": -0.6900390386581421,
      "logps/rejected": -1.4294922351837158,
      "loss": 1.0195,
      "nll_loss": 0.934374988079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06900634616613388,
      "rewards/margins": 0.0738983154296875,
      "rewards/rejected": -0.14304199814796448,
      "step": 1960
    },
    {
      "epoch": 0.14949157687054182,
      "grad_norm": 1.5424244301559844,
      "learning_rate": 1.8024236362373315e-06,
      "log_odds_chosen": 0.9842773675918579,
      "log_odds_ratio": -0.46113282442092896,
      "logits/chosen": -0.9634765386581421,
      "logits/rejected": -0.8519531488418579,
      "logps/chosen": -0.681835949420929,
      "logps/rejected": -1.3474609851837158,
      "loss": 1.0132,
      "nll_loss": 0.969921886920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06818847358226776,
      "rewards/margins": 0.0665283203125,
      "rewards/rejected": -0.13474121689796448,
      "step": 1970
    },
    {
      "epoch": 0.15025041736227046,
      "grad_norm": 4.294685050248751,
      "learning_rate": 1.7978662999019787e-06,
      "log_odds_chosen": 1.1591796875,
      "log_odds_ratio": -0.4561523497104645,
      "logits/chosen": -0.9873046875,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.675000011920929,
      "logps/rejected": -1.481835961341858,
      "loss": 1.018,
      "nll_loss": 0.981640636920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06749267876148224,
      "rewards/margins": 0.080718994140625,
      "rewards/rejected": -0.14836426079273224,
      "step": 1980
    },
    {
      "epoch": 0.1510092578539991,
      "grad_norm": 1.8285917174807669,
      "learning_rate": 1.793343358648881e-06,
      "log_odds_chosen": 1.26025390625,
      "log_odds_ratio": -0.43413084745407104,
      "logits/chosen": -0.942187488079071,
      "logits/rejected": -0.839062511920929,
      "logps/chosen": -0.688281238079071,
      "logps/rejected": -1.578710913658142,
      "loss": 0.9978,
      "nll_loss": 0.955273449420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06879882514476776,
      "rewards/margins": 0.0889892578125,
      "rewards/rejected": -0.15781250596046448,
      "step": 1990
    },
    {
      "epoch": 0.15176809834572771,
      "grad_norm": 1.7246984869527726,
      "learning_rate": 1.7888543819998317e-06,
      "log_odds_chosen": 1.23291015625,
      "log_odds_ratio": -0.46337890625,
      "logits/chosen": -0.979296863079071,
      "logits/rejected": -0.8853515386581421,
      "logps/chosen": -0.7085937261581421,
      "logps/rejected": -1.5851562023162842,
      "loss": 1.0057,
      "nll_loss": 0.959765613079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07087402045726776,
      "rewards/margins": 0.08778686821460724,
      "rewards/rejected": -0.1585693359375,
      "step": 2000
    },
    {
      "epoch": 0.15252693883745636,
      "grad_norm": 1.8078525016188747,
      "learning_rate": 1.7843989469818819e-06,
      "log_odds_chosen": 0.66796875,
      "log_odds_ratio": -0.581250011920929,
      "logits/chosen": -0.9097656011581421,
      "logits/rejected": -0.8248046636581421,
      "logps/chosen": -0.721484363079071,
      "logps/rejected": -1.169531226158142,
      "loss": 1.0303,
      "nll_loss": 0.930859386920929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07215575873851776,
      "rewards/margins": 0.044759370386600494,
      "rewards/rejected": -0.11696777492761612,
      "step": 2010
    },
    {
      "epoch": 0.153285779329185,
      "grad_norm": 1.750955590287212,
      "learning_rate": 1.779976637959939e-06,
      "log_odds_chosen": 1.244970679283142,
      "log_odds_ratio": -0.44819337129592896,
      "logits/chosen": -0.9283202886581421,
      "logits/rejected": -0.814648449420929,
      "logps/chosen": -0.7040039300918579,
      "logps/rejected": -1.57421875,
      "loss": 1.0132,
      "nll_loss": 0.975781261920929,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07039795070886612,
      "rewards/margins": 0.0870361328125,
      "rewards/rejected": -0.15751953423023224,
      "step": 2020
    },
    {
      "epoch": 0.15404461982091364,
      "grad_norm": 1.8459833504202043,
      "learning_rate": 1.7755870464739012e-06,
      "log_odds_chosen": 0.91015625,
      "log_odds_ratio": -0.5287109613418579,
      "logits/chosen": -0.950976550579071,
      "logits/rejected": -0.8560546636581421,
      "logps/chosen": -0.6919921636581421,
      "logps/rejected": -1.326562523841858,
      "loss": 1.0335,
      "nll_loss": 0.981249988079071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06920166313648224,
      "rewards/margins": 0.06353607028722763,
      "rewards/rejected": -0.13271483778953552,
      "step": 2030
    },
    {
      "epoch": 0.15480346031264228,
      "grad_norm": 1.7056677979498798,
      "learning_rate": 1.7712297710801907e-06,
      "log_odds_chosen": 1.076074242591858,
      "log_odds_ratio": -0.47509765625,
      "logits/chosen": -0.9117187261581421,
      "logits/rejected": -0.833789050579071,
      "logps/chosen": -0.6922851800918579,
      "logps/rejected": -1.421484351158142,
      "loss": 1.0171,
      "nll_loss": 0.967968761920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0692138671875,
      "rewards/margins": 0.07306518405675888,
      "rewards/rejected": -0.14218750596046448,
      "step": 2040
    },
    {
      "epoch": 0.15556230080437092,
      "grad_norm": 1.605351728289974,
      "learning_rate": 1.7669044171975444e-06,
      "log_odds_chosen": 1.0661499500274658,
      "log_odds_ratio": -0.4999023377895355,
      "logits/chosen": -0.9091796875,
      "logits/rejected": -0.8255859613418579,
      "logps/chosen": -0.7171875238418579,
      "logps/rejected": -1.4519531726837158,
      "loss": 1.0172,
      "nll_loss": 0.9818359613418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07167968899011612,
      "rewards/margins": 0.07357482612133026,
      "rewards/rejected": -0.14523926377296448,
      "step": 2050
    },
    {
      "epoch": 0.15632114129609956,
      "grad_norm": 1.6960342772381833,
      "learning_rate": 1.7626105969569268e-06,
      "log_odds_chosen": 1.058984398841858,
      "log_odds_ratio": -0.460693359375,
      "logits/chosen": -0.9281250238418579,
      "logits/rejected": -0.8433593511581421,
      "logps/chosen": -0.7040039300918579,
      "logps/rejected": -1.4226562976837158,
      "loss": 1.0291,
      "nll_loss": 1.0041015148162842,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07033691555261612,
      "rewards/margins": 0.07182617485523224,
      "rewards/rejected": -0.14223632216453552,
      "step": 2060
    },
    {
      "epoch": 0.1570799817878282,
      "grad_norm": 2.0842118121138205,
      "learning_rate": 1.758347929055432e-06,
      "log_odds_chosen": 1.00048828125,
      "log_odds_ratio": -0.455810546875,
      "logits/chosen": -0.9208984375,
      "logits/rejected": -0.8349609375,
      "logps/chosen": -0.6654297113418579,
      "logps/rejected": -1.3132812976837158,
      "loss": 0.9886,
      "nll_loss": 0.919140636920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06655273586511612,
      "rewards/margins": 0.06477661430835724,
      "rewards/rejected": -0.13137206435203552,
      "step": 2070
    },
    {
      "epoch": 0.15783882227955684,
      "grad_norm": 2.120569477558702,
      "learning_rate": 1.7541160386140582e-06,
      "log_odds_chosen": 1.3015625476837158,
      "log_odds_ratio": -0.4002929627895355,
      "logits/chosen": -0.9339843988418579,
      "logits/rejected": -0.81640625,
      "logps/chosen": -0.648242175579071,
      "logps/rejected": -1.5167968273162842,
      "loss": 0.9989,
      "nll_loss": 0.984375,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06483153998851776,
      "rewards/margins": 0.08698730170726776,
      "rewards/rejected": -0.15175780653953552,
      "step": 2080
    },
    {
      "epoch": 0.15859766277128548,
      "grad_norm": 1.6539249012178663,
      "learning_rate": 1.7499145570392284e-06,
      "log_odds_chosen": 1.0309326648712158,
      "log_odds_ratio": -0.49125975370407104,
      "logits/chosen": -1.0031249523162842,
      "logits/rejected": -0.8828125,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -1.4093749523162842,
      "loss": 1.002,
      "nll_loss": 0.9527343511581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07062987983226776,
      "rewards/margins": 0.07036437839269638,
      "rewards/rejected": -0.140869140625,
      "step": 2090
    },
    {
      "epoch": 0.15935650326301412,
      "grad_norm": 1.6299919394171718,
      "learning_rate": 1.745743121887939e-06,
      "log_odds_chosen": 0.9908691644668579,
      "log_odds_ratio": -0.49169921875,
      "logits/chosen": -0.9585937261581421,
      "logits/rejected": -0.833984375,
      "logps/chosen": -0.6826171875,
      "logps/rejected": -1.361914038658142,
      "loss": 1.0307,
      "nll_loss": 0.9857422113418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06827392429113388,
      "rewards/margins": 0.06794128566980362,
      "rewards/rejected": -0.13613280653953552,
      "step": 2100
    },
    {
      "epoch": 0.16011534375474276,
      "grad_norm": 1.5610258992867714,
      "learning_rate": 1.7416013767364324e-06,
      "log_odds_chosen": 0.9908202886581421,
      "log_odds_ratio": -0.4573730528354645,
      "logits/chosen": -0.960156261920929,
      "logits/rejected": -0.8716796636581421,
      "logps/chosen": -0.6415039300918579,
      "logps/rejected": -1.2478516101837158,
      "loss": 0.9743,
      "nll_loss": 0.8705078363418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06418456882238388,
      "rewards/margins": 0.06059875339269638,
      "rewards/rejected": -0.12473144382238388,
      "step": 2110
    },
    {
      "epoch": 0.1608741842464714,
      "grad_norm": 1.7782745410513578,
      "learning_rate": 1.7374889710522776e-06,
      "log_odds_chosen": 1.322363257408142,
      "log_odds_ratio": -0.4240478575229645,
      "logits/chosen": -0.943554699420929,
      "logits/rejected": -0.85546875,
      "logps/chosen": -0.656542956829071,
      "logps/rejected": -1.5681641101837158,
      "loss": 0.9845,
      "nll_loss": 0.860156238079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06564941257238388,
      "rewards/margins": 0.09116820991039276,
      "rewards/rejected": -0.15683594346046448,
      "step": 2120
    },
    {
      "epoch": 0.16163302473820004,
      "grad_norm": 1.6843050386022687,
      "learning_rate": 1.7334055600697579e-06,
      "log_odds_chosen": 1.218359351158142,
      "log_odds_ratio": -0.44462889432907104,
      "logits/chosen": -0.924023449420929,
      "logits/rejected": -0.867382824420929,
      "logps/chosen": -0.6788085699081421,
      "logps/rejected": -1.494726538658142,
      "loss": 1.0162,
      "nll_loss": 0.98046875,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06787109375,
      "rewards/margins": 0.08159790188074112,
      "rewards/rejected": -0.14948730170726776,
      "step": 2130
    },
    {
      "epoch": 0.16239186522992866,
      "grad_norm": 1.8290613554640052,
      "learning_rate": 1.7293508046684678e-06,
      "log_odds_chosen": 0.8995605707168579,
      "log_odds_ratio": -0.5249999761581421,
      "logits/chosen": -0.9195312261581421,
      "logits/rejected": -0.853320300579071,
      "logps/chosen": -0.704296886920929,
      "logps/rejected": -1.318359375,
      "loss": 0.9991,
      "nll_loss": 0.9419921636581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07045898586511612,
      "rewards/margins": 0.06148681789636612,
      "rewards/rejected": -0.13188476860523224,
      "step": 2140
    },
    {
      "epoch": 0.1631507057216573,
      "grad_norm": 1.7340261251194247,
      "learning_rate": 1.7253243712550145e-06,
      "log_odds_chosen": 0.962109386920929,
      "log_odds_ratio": -0.5,
      "logits/chosen": -0.900390625,
      "logits/rejected": -0.802441418170929,
      "logps/chosen": -0.6602538824081421,
      "logps/rejected": -1.2755858898162842,
      "loss": 1.0306,
      "nll_loss": 1.052148461341858,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06602783501148224,
      "rewards/margins": 0.06157531589269638,
      "rewards/rejected": -0.12758788466453552,
      "step": 2150
    },
    {
      "epoch": 0.16390954621338594,
      "grad_norm": 1.6987763744425957,
      "learning_rate": 1.7213259316477406e-06,
      "log_odds_chosen": 1.0940430164337158,
      "log_odds_ratio": -0.5050293207168579,
      "logits/chosen": -0.9619140625,
      "logits/rejected": -0.8824218511581421,
      "logps/chosen": -0.757031261920929,
      "logps/rejected": -1.5478515625,
      "loss": 1.0029,
      "nll_loss": 0.9525390863418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07576904445886612,
      "rewards/margins": 0.07902832329273224,
      "rewards/rejected": -0.15480956435203552,
      "step": 2160
    },
    {
      "epoch": 0.16466838670511458,
      "grad_norm": 1.9183213512351749,
      "learning_rate": 1.7173551629643674e-06,
      "log_odds_chosen": 1.0694091320037842,
      "log_odds_ratio": -0.48066407442092896,
      "logits/chosen": -0.904101550579071,
      "logits/rejected": -0.8335937261581421,
      "logps/chosen": -0.738085925579071,
      "logps/rejected": -1.4951171875,
      "loss": 1.0171,
      "nll_loss": 1.003515601158142,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07376708835363388,
      "rewards/margins": 0.07562103122472763,
      "rewards/rejected": -0.14951172471046448,
      "step": 2170
    },
    {
      "epoch": 0.16542722719684322,
      "grad_norm": 1.7591486737480413,
      "learning_rate": 1.713411747512477e-06,
      "log_odds_chosen": 1.2626953125,
      "log_odds_ratio": -0.4422851502895355,
      "logits/chosen": -0.9052734375,
      "logits/rejected": -0.803906261920929,
      "logps/chosen": -0.6700195074081421,
      "logps/rejected": -1.5408203601837158,
      "loss": 1.0147,
      "nll_loss": 0.922656238079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06697998195886612,
      "rewards/margins": 0.08709716796875,
      "rewards/rejected": -0.15410156548023224,
      "step": 2180
    },
    {
      "epoch": 0.16618606768857186,
      "grad_norm": 2.2730392023754424,
      "learning_rate": 1.709495372682753e-06,
      "log_odds_chosen": 1.2048828601837158,
      "log_odds_ratio": -0.47001951932907104,
      "logits/chosen": -0.890429675579071,
      "logits/rejected": -0.804492175579071,
      "logps/chosen": -0.738085925579071,
      "logps/rejected": -1.5867187976837158,
      "loss": 0.9889,
      "nll_loss": 0.924023449420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07381591945886612,
      "rewards/margins": 0.08494262397289276,
      "rewards/rejected": -0.15876464545726776,
      "step": 2190
    },
    {
      "epoch": 0.1669449081803005,
      "grad_norm": 2.037458706640104,
      "learning_rate": 1.7056057308448832e-06,
      "log_odds_chosen": 1.121679663658142,
      "log_odds_ratio": -0.43730467557907104,
      "logits/chosen": -0.916210949420929,
      "logits/rejected": -0.8021484613418579,
      "logps/chosen": -0.645312488079071,
      "logps/rejected": -1.4080078601837158,
      "loss": 0.9718,
      "nll_loss": 0.9302734136581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06459961086511612,
      "rewards/margins": 0.07626952975988388,
      "rewards/rejected": -0.14077147841453552,
      "step": 2200
    },
    {
      "epoch": 0.16770374867202914,
      "grad_norm": 2.046099172625031,
      "learning_rate": 1.701742519246068e-06,
      "log_odds_chosen": 0.951708972454071,
      "log_odds_ratio": -0.4844726622104645,
      "logits/chosen": -0.9341796636581421,
      "logits/rejected": -0.8324218988418579,
      "logps/chosen": -0.7157226800918579,
      "logps/rejected": -1.3728516101837158,
      "loss": 1.0105,
      "nll_loss": 0.9869140386581421,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07159423828125,
      "rewards/margins": 0.065643310546875,
      "rewards/rejected": -0.13715820014476776,
      "step": 2210
    },
    {
      "epoch": 0.16846258916375778,
      "grad_norm": 1.6540454740677957,
      "learning_rate": 1.6979054399120355e-06,
      "log_odds_chosen": 1.15185546875,
      "log_odds_ratio": -0.47382813692092896,
      "logits/chosen": -0.906054675579071,
      "logits/rejected": -0.828320324420929,
      "logps/chosen": -0.669628918170929,
      "logps/rejected": -1.450781226158142,
      "loss": 0.9966,
      "nll_loss": 0.908984363079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06693115085363388,
      "rewards/margins": 0.07802734524011612,
      "rewards/rejected": -0.14506836235523224,
      "step": 2220
    },
    {
      "epoch": 0.16922142965548642,
      "grad_norm": 1.6756053571033922,
      "learning_rate": 1.6940941995505069e-06,
      "log_odds_chosen": 0.881176769733429,
      "log_odds_ratio": -0.536328136920929,
      "logits/chosen": -0.912890613079071,
      "logits/rejected": -0.8296874761581421,
      "logps/chosen": -0.773242175579071,
      "logps/rejected": -1.3869140148162842,
      "loss": 1.0266,
      "nll_loss": 1.0226562023162842,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07731933891773224,
      "rewards/margins": 0.06138915941119194,
      "rewards/rejected": -0.13857421278953552,
      "step": 2230
    },
    {
      "epoch": 0.16998027014721506,
      "grad_norm": 1.555734920343979,
      "learning_rate": 1.6903085094570331e-06,
      "log_odds_chosen": 0.941113293170929,
      "log_odds_ratio": -0.49560546875,
      "logits/chosen": -0.943359375,
      "logits/rejected": -0.8056640625,
      "logps/chosen": -0.69580078125,
      "logps/rejected": -1.3240234851837158,
      "loss": 0.9995,
      "nll_loss": 0.9052734375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06951904296875,
      "rewards/margins": 0.06293945014476776,
      "rewards/rejected": -0.13237304985523224,
      "step": 2240
    },
    {
      "epoch": 0.1707391106389437,
      "grad_norm": 1.7021231359836484,
      "learning_rate": 1.6865480854231356e-06,
      "log_odds_chosen": 0.9696289300918579,
      "log_odds_ratio": -0.51904296875,
      "logits/chosen": -0.891796886920929,
      "logits/rejected": -0.807421863079071,
      "logps/chosen": -0.720898449420929,
      "logps/rejected": -1.3908202648162842,
      "loss": 1.0088,
      "nll_loss": 0.9750000238418579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07203368842601776,
      "rewards/margins": 0.06697998195886612,
      "rewards/rejected": -0.13901367783546448,
      "step": 2250
    },
    {
      "epoch": 0.17149795113067234,
      "grad_norm": 1.657905024680801,
      "learning_rate": 1.682812647646685e-06,
      "log_odds_chosen": 1.2488281726837158,
      "log_odds_ratio": -0.39140623807907104,
      "logits/chosen": -0.9117187261581421,
      "logits/rejected": -0.821484386920929,
      "logps/chosen": -0.67041015625,
      "logps/rejected": -1.491796851158142,
      "loss": 0.9972,
      "nll_loss": 0.909375011920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06705322116613388,
      "rewards/margins": 0.08208008110523224,
      "rewards/rejected": -0.14914551377296448,
      "step": 2260
    },
    {
      "epoch": 0.17225679162240098,
      "grad_norm": 1.6187004410213444,
      "learning_rate": 1.6791019206444541e-06,
      "log_odds_chosen": 0.9906250238418579,
      "log_odds_ratio": -0.4784179627895355,
      "logits/chosen": -0.935351550579071,
      "logits/rejected": -0.831250011920929,
      "logps/chosen": -0.707324206829071,
      "logps/rejected": -1.4052734375,
      "loss": 1.0238,
      "nll_loss": 0.9925781488418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07073974609375,
      "rewards/margins": 0.06994018703699112,
      "rewards/rejected": -0.140625,
      "step": 2270
    },
    {
      "epoch": 0.1730156321141296,
      "grad_norm": 2.3583378920089197,
      "learning_rate": 1.675415633166782e-06,
      "log_odds_chosen": 1.3103516101837158,
      "log_odds_ratio": -0.42724609375,
      "logits/chosen": -0.949023425579071,
      "logits/rejected": -0.8480468988418579,
      "logps/chosen": -0.6993163824081421,
      "logps/rejected": -1.608984351158142,
      "loss": 0.9873,
      "nll_loss": 0.891406238079071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06988525390625,
      "rewards/margins": 0.09085693210363388,
      "rewards/rejected": -0.160888671875,
      "step": 2280
    },
    {
      "epoch": 0.17377447260585824,
      "grad_norm": 1.8684296943890162,
      "learning_rate": 1.6717535181142914e-06,
      "log_odds_chosen": 0.8514159917831421,
      "log_odds_ratio": -0.5181640386581421,
      "logits/chosen": -0.957812488079071,
      "logits/rejected": -0.8353515863418579,
      "logps/chosen": -0.724609375,
      "logps/rejected": -1.302734375,
      "loss": 1.0093,
      "nll_loss": 0.948437511920929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07241211086511612,
      "rewards/margins": 0.05777130275964737,
      "rewards/rejected": -0.13020019233226776,
      "step": 2290
    },
    {
      "epoch": 0.17453331309758688,
      "grad_norm": 1.7443057982305137,
      "learning_rate": 1.668115312456598e-06,
      "log_odds_chosen": 0.9159179925918579,
      "log_odds_ratio": -0.482666015625,
      "logits/chosen": -0.918749988079071,
      "logits/rejected": -0.8291015625,
      "logps/chosen": -0.676953136920929,
      "logps/rejected": -1.273828148841858,
      "loss": 0.9937,
      "nll_loss": 0.9429687261581421,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06765136867761612,
      "rewards/margins": 0.05970611423254013,
      "rewards/rejected": -0.12736816704273224,
      "step": 2300
    },
    {
      "epoch": 0.17529215358931552,
      "grad_norm": 1.7495476536042223,
      "learning_rate": 1.6645007571529578e-06,
      "log_odds_chosen": 1.193359375,
      "log_odds_ratio": -0.4368896484375,
      "logits/chosen": -0.9228515625,
      "logits/rejected": -0.783984363079071,
      "logps/chosen": -0.710156261920929,
      "logps/rejected": -1.5369141101837158,
      "loss": 0.9818,
      "nll_loss": 0.991406261920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07098388671875,
      "rewards/margins": 0.082611083984375,
      "rewards/rejected": -0.15371093153953552,
      "step": 2310
    },
    {
      "epoch": 0.17605099408104416,
      "grad_norm": 2.061978268838713,
      "learning_rate": 1.6609095970747992e-06,
      "log_odds_chosen": 0.933422863483429,
      "log_odds_ratio": -0.5176757574081421,
      "logits/chosen": -0.9085937738418579,
      "logits/rejected": -0.8267577886581421,
      "logps/chosen": -0.770312488079071,
      "logps/rejected": -1.447265625,
      "loss": 1.0063,
      "nll_loss": 1.0353515148162842,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07706298679113388,
      "rewards/margins": 0.06778564304113388,
      "rewards/rejected": -0.14484862983226776,
      "step": 2320
    },
    {
      "epoch": 0.1768098345727728,
      "grad_norm": 1.9864693037076206,
      "learning_rate": 1.6573415809300833e-06,
      "log_odds_chosen": 0.85400390625,
      "log_odds_ratio": -0.526562511920929,
      "logits/chosen": -0.8955078125,
      "logits/rejected": -0.846484363079071,
      "logps/chosen": -0.7152343988418579,
      "logps/rejected": -1.2712891101837158,
      "loss": 0.999,
      "nll_loss": 1.0095703601837158,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07152099907398224,
      "rewards/margins": 0.05556793138384819,
      "rewards/rejected": -0.12712402641773224,
      "step": 2330
    },
    {
      "epoch": 0.17756867506450144,
      "grad_norm": 1.5250036352910814,
      "learning_rate": 1.6537964611894462e-06,
      "log_odds_chosen": 0.8726806640625,
      "log_odds_ratio": -0.4984374940395355,
      "logits/chosen": -0.9457031488418579,
      "logits/rejected": -0.881054699420929,
      "logps/chosen": -0.6864258050918579,
      "logps/rejected": -1.264257788658142,
      "loss": 0.9864,
      "nll_loss": 0.995898425579071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06871338188648224,
      "rewards/margins": 0.05774535983800888,
      "rewards/rejected": -0.12641601264476776,
      "step": 2340
    },
    {
      "epoch": 0.17832751555623008,
      "grad_norm": 1.7843935011591716,
      "learning_rate": 1.6502739940140692e-06,
      "log_odds_chosen": 0.805798351764679,
      "log_odds_ratio": -0.510449230670929,
      "logits/chosen": -0.935546875,
      "logits/rejected": -0.809765636920929,
      "logps/chosen": -0.7056640386581421,
      "logps/rejected": -1.2316405773162842,
      "loss": 1.0055,
      "nll_loss": 1.05859375,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07052002102136612,
      "rewards/margins": 0.05268402025103569,
      "rewards/rejected": -0.12321777641773224,
      "step": 2350
    },
    {
      "epoch": 0.17908635604795872,
      "grad_norm": 1.7329387674163914,
      "learning_rate": 1.6467739391852364e-06,
      "log_odds_chosen": 0.932861328125,
      "log_odds_ratio": -0.492919921875,
      "logits/chosen": -0.903124988079071,
      "logits/rejected": -0.8291015625,
      "logps/chosen": -0.696582019329071,
      "logps/rejected": -1.316015601158142,
      "loss": 0.972,
      "nll_loss": 0.9710937738418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06962890923023224,
      "rewards/margins": 0.06194610521197319,
      "rewards/rejected": -0.13166503608226776,
      "step": 2360
    },
    {
      "epoch": 0.17984519653968736,
      "grad_norm": 1.6461497139628067,
      "learning_rate": 1.6432960600355221e-06,
      "log_odds_chosen": 1.0046875476837158,
      "log_odds_ratio": -0.46044921875,
      "logits/chosen": -0.933398425579071,
      "logits/rejected": -0.8314453363418579,
      "logps/chosen": -0.6783202886581421,
      "logps/rejected": -1.338281273841858,
      "loss": 0.9891,
      "nll_loss": 0.9117187261581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06779785454273224,
      "rewards/margins": 0.0659637451171875,
      "rewards/rejected": -0.13381347060203552,
      "step": 2370
    },
    {
      "epoch": 0.180604037031416,
      "grad_norm": 1.925964568167815,
      "learning_rate": 1.6398401233815756e-06,
      "log_odds_chosen": 1.0750000476837158,
      "log_odds_ratio": -0.47919923067092896,
      "logits/chosen": -0.955078125,
      "logits/rejected": -0.859375,
      "logps/chosen": -0.7408202886581421,
      "logps/rejected": -1.4873046875,
      "loss": 0.9939,
      "nll_loss": 0.900585949420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07413329929113388,
      "rewards/margins": 0.07451782375574112,
      "rewards/rejected": -0.14860840141773224,
      "step": 2380
    },
    {
      "epoch": 0.18136287752314464,
      "grad_norm": 1.8137519236514643,
      "learning_rate": 1.6364058994584524e-06,
      "log_odds_chosen": 0.9222656488418579,
      "log_odds_ratio": -0.5028320550918579,
      "logits/chosen": -0.8755859136581421,
      "logits/rejected": -0.799023449420929,
      "logps/chosen": -0.7115234136581421,
      "logps/rejected": -1.3359375,
      "loss": 0.9888,
      "nll_loss": 0.9359375238418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07113037258386612,
      "rewards/margins": 0.06235351413488388,
      "rewards/rejected": -0.13352051377296448,
      "step": 2390
    },
    {
      "epoch": 0.18212171801487329,
      "grad_norm": 1.688614794458292,
      "learning_rate": 1.6329931618554522e-06,
      "log_odds_chosen": 1.150390625,
      "log_odds_ratio": -0.453125,
      "logits/chosen": -0.930468738079071,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.686230480670929,
      "logps/rejected": -1.476171851158142,
      "loss": 0.9823,
      "nll_loss": 0.946484386920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06862793117761612,
      "rewards/margins": 0.079010009765625,
      "rewards/rejected": -0.14760741591453552,
      "step": 2400
    },
    {
      "epoch": 0.18288055850660193,
      "grad_norm": 1.8014616595320827,
      "learning_rate": 1.6296016874534209e-06,
      "log_odds_chosen": 1.108154296875,
      "log_odds_ratio": -0.458984375,
      "logits/chosen": -0.911328136920929,
      "logits/rejected": -0.850390613079071,
      "logps/chosen": -0.693554699420929,
      "logps/rejected": -1.4207031726837158,
      "loss": 0.9831,
      "nll_loss": 0.981640636920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06934814155101776,
      "rewards/margins": 0.07284469902515411,
      "rewards/rejected": -0.14213867485523224,
      "step": 2410
    },
    {
      "epoch": 0.18363939899833054,
      "grad_norm": 1.7304693115361636,
      "learning_rate": 1.6262312563634835e-06,
      "log_odds_chosen": 1.104882836341858,
      "log_odds_ratio": -0.4971679747104645,
      "logits/chosen": -0.998828113079071,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.7134765386581421,
      "logps/rejected": -1.490820288658142,
      "loss": 0.9875,
      "nll_loss": 0.934765636920929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07139892876148224,
      "rewards/margins": 0.07766113430261612,
      "rewards/rejected": -0.14907225966453552,
      "step": 2420
    },
    {
      "epoch": 0.18439823949005918,
      "grad_norm": 2.032034928793834,
      "learning_rate": 1.6228816518671587e-06,
      "log_odds_chosen": 1.02294921875,
      "log_odds_ratio": -0.4745117127895355,
      "logits/chosen": -0.957812488079071,
      "logits/rejected": -0.842968761920929,
      "logps/chosen": -0.710156261920929,
      "logps/rejected": -1.451171875,
      "loss": 0.9879,
      "nll_loss": 0.9244140386581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07102050632238388,
      "rewards/margins": 0.07415161281824112,
      "rewards/rejected": -0.14511719346046448,
      "step": 2430
    },
    {
      "epoch": 0.18515707998178782,
      "grad_norm": 2.0892961715952696,
      "learning_rate": 1.619552660357832e-06,
      "log_odds_chosen": 1.128198266029358,
      "log_odds_ratio": -0.49091798067092896,
      "logits/chosen": -0.934765636920929,
      "logits/rejected": -0.8541015386581421,
      "logps/chosen": -0.7372070550918579,
      "logps/rejected": -1.5187499523162842,
      "loss": 0.99,
      "nll_loss": 0.9781249761581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07374267280101776,
      "rewards/margins": 0.07811889797449112,
      "rewards/rejected": -0.15183106064796448,
      "step": 2440
    },
    {
      "epoch": 0.18591592047351646,
      "grad_norm": 1.66781243666752,
      "learning_rate": 1.616244071283537e-06,
      "log_odds_chosen": 1.009863257408142,
      "log_odds_ratio": -0.4766601622104645,
      "logits/chosen": -0.9652343988418579,
      "logits/rejected": -0.8421875238418579,
      "logps/chosen": -0.7496093511581421,
      "logps/rejected": -1.4240233898162842,
      "loss": 0.9889,
      "nll_loss": 0.971484363079071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07501220703125,
      "rewards/margins": 0.06739501655101776,
      "rewards/rejected": -0.14240722358226776,
      "step": 2450
    },
    {
      "epoch": 0.1866747609652451,
      "grad_norm": 1.8462614488666282,
      "learning_rate": 1.6129556770910235e-06,
      "log_odds_chosen": 1.1233398914337158,
      "log_odds_ratio": -0.4259277284145355,
      "logits/chosen": -0.985546886920929,
      "logits/rejected": -0.8519531488418579,
      "logps/chosen": -0.646679699420929,
      "logps/rejected": -1.3787109851837158,
      "loss": 0.9742,
      "nll_loss": 0.9052734375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06461181491613388,
      "rewards/margins": 0.07322387397289276,
      "rewards/rejected": -0.13774414360523224,
      "step": 2460
    },
    {
      "epoch": 0.18743360145697374,
      "grad_norm": 2.077714916187083,
      "learning_rate": 1.6096872731710673e-06,
      "log_odds_chosen": 1.1902344226837158,
      "log_odds_ratio": -0.42304688692092896,
      "logits/chosen": -0.9332031011581421,
      "logits/rejected": -0.84375,
      "logps/chosen": -0.743457019329071,
      "logps/rejected": -1.571679711341858,
      "loss": 0.9835,
      "nll_loss": 0.9898437261581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07436523586511612,
      "rewards/margins": 0.0826416015625,
      "rewards/rejected": -0.15708008408546448,
      "step": 2470
    },
    {
      "epoch": 0.18819244194870238,
      "grad_norm": 1.8616129591738162,
      "learning_rate": 1.6064386578049978e-06,
      "log_odds_chosen": 1.16064453125,
      "log_odds_ratio": -0.47441405057907104,
      "logits/chosen": -0.904296875,
      "logits/rejected": -0.820507824420929,
      "logps/chosen": -0.693164050579071,
      "logps/rejected": -1.5470702648162842,
      "loss": 0.9871,
      "nll_loss": 0.9693359136581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06928710639476776,
      "rewards/margins": 0.08541259914636612,
      "rewards/rejected": -0.1546630859375,
      "step": 2480
    },
    {
      "epoch": 0.18895128244043102,
      "grad_norm": 2.1166882079269342,
      "learning_rate": 1.6032096321124046e-06,
      "log_odds_chosen": 0.8385375738143921,
      "log_odds_ratio": -0.5474609136581421,
      "logits/chosen": -0.9267578125,
      "logits/rejected": -0.8587890863418579,
      "logps/chosen": -0.755175769329071,
      "logps/rejected": -1.3542969226837158,
      "loss": 0.9952,
      "nll_loss": 0.971484363079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07552490383386612,
      "rewards/margins": 0.05993194505572319,
      "rewards/rejected": -0.13544921576976776,
      "step": 2490
    },
    {
      "epoch": 0.18971012293215966,
      "grad_norm": 2.110336575040382,
      "learning_rate": 1.6e-06,
      "log_odds_chosen": 1.1823241710662842,
      "log_odds_ratio": -0.46098631620407104,
      "logits/chosen": -0.9212890863418579,
      "logits/rejected": -0.8544921875,
      "logps/chosen": -0.652636706829071,
      "logps/rejected": -1.503320336341858,
      "loss": 0.9806,
      "nll_loss": 0.9033203125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06525878608226776,
      "rewards/margins": 0.0849609375,
      "rewards/rejected": -0.15021972358226776,
      "step": 2500
    },
    {
      "epoch": 0.1904689634238883,
      "grad_norm": 1.6348076610742528,
      "learning_rate": 1.5968095681115984e-06,
      "log_odds_chosen": 1.142578125,
      "log_odds_ratio": -0.486328125,
      "logits/chosen": -0.9554687738418579,
      "logits/rejected": -0.8511718511581421,
      "logps/chosen": -0.7681640386581421,
      "logps/rejected": -1.591796875,
      "loss": 0.9933,
      "nll_loss": 0.9615234136581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07675781100988388,
      "rewards/margins": 0.08242187649011612,
      "rewards/rejected": -0.15913085639476776,
      "step": 2510
    },
    {
      "epoch": 0.19122780391561695,
      "grad_norm": 2.136414388223389,
      "learning_rate": 1.5936381457791914e-06,
      "log_odds_chosen": 1.12158203125,
      "log_odds_ratio": -0.4488281309604645,
      "logits/chosen": -1.003515601158142,
      "logits/rejected": -0.9029296636581421,
      "logps/chosen": -0.730761706829071,
      "logps/rejected": -1.494726538658142,
      "loss": 0.9646,
      "nll_loss": 0.960156261920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07314453274011612,
      "rewards/margins": 0.076385498046875,
      "rewards/rejected": -0.14958496391773224,
      "step": 2520
    },
    {
      "epoch": 0.19198664440734559,
      "grad_norm": 1.709686144800437,
      "learning_rate": 1.590485544975088e-06,
      "log_odds_chosen": 1.1706054210662842,
      "log_odds_ratio": -0.43974608182907104,
      "logits/chosen": -0.9906250238418579,
      "logits/rejected": -0.8759765625,
      "logps/chosen": -0.724804699420929,
      "logps/rejected": -1.545507788658142,
      "loss": 0.9767,
      "nll_loss": 1.0037109851837158,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07244873046875,
      "rewards/margins": 0.08211059868335724,
      "rewards/rejected": -0.15458984673023224,
      "step": 2530
    },
    {
      "epoch": 0.19274548489907423,
      "grad_norm": 1.8255597130329309,
      "learning_rate": 1.5873515802650901e-06,
      "log_odds_chosen": 0.9172118902206421,
      "log_odds_ratio": -0.507275402545929,
      "logits/chosen": -0.990429699420929,
      "logits/rejected": -0.904101550579071,
      "logps/chosen": -0.6786133050918579,
      "logps/rejected": -1.294531226158142,
      "loss": 0.9685,
      "nll_loss": 0.9468749761581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06785888969898224,
      "rewards/margins": 0.06157226487994194,
      "rewards/rejected": -0.12946777045726776,
      "step": 2540
    },
    {
      "epoch": 0.19350432539080287,
      "grad_norm": 1.789326725731532,
      "learning_rate": 1.584236068762679e-06,
      "log_odds_chosen": 1.466210961341858,
      "log_odds_ratio": -0.3741455078125,
      "logits/chosen": -0.9769531488418579,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.6670898199081421,
      "logps/rejected": -1.676367163658142,
      "loss": 1.0203,
      "nll_loss": 0.9556640386581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06667480617761612,
      "rewards/margins": 0.10093994438648224,
      "rewards/rejected": -0.16752929985523224,
      "step": 2550
    },
    {
      "epoch": 0.19426316588253148,
      "grad_norm": 1.8900962347223054,
      "learning_rate": 1.5811388300841894e-06,
      "log_odds_chosen": 1.261132836341858,
      "log_odds_ratio": -0.4341796934604645,
      "logits/chosen": -0.9541015625,
      "logits/rejected": -0.8617187738418579,
      "logps/chosen": -0.699023425579071,
      "logps/rejected": -1.5593750476837158,
      "loss": 0.9719,
      "nll_loss": 1.002539038658142,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06990966945886612,
      "rewards/margins": 0.08603973686695099,
      "rewards/rejected": -0.15598145127296448,
      "step": 2560
    },
    {
      "epoch": 0.19502200637426012,
      "grad_norm": 1.6750647053286092,
      "learning_rate": 1.5780596863049431e-06,
      "log_odds_chosen": 1.4158203601837158,
      "log_odds_ratio": -0.38447266817092896,
      "logits/chosen": -0.9541015625,
      "logits/rejected": -0.8203125,
      "logps/chosen": -0.653613269329071,
      "logps/rejected": -1.607031226158142,
      "loss": 0.9651,
      "nll_loss": 0.8833984136581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06544189155101776,
      "rewards/margins": 0.09538574516773224,
      "rewards/rejected": -0.16069336235523224,
      "step": 2570
    },
    {
      "epoch": 0.19578084686598876,
      "grad_norm": 1.9326493464721346,
      "learning_rate": 1.5749984619163156e-06,
      "log_odds_chosen": 1.1027343273162842,
      "log_odds_ratio": -0.4658203125,
      "logits/chosen": -0.9818359613418579,
      "logits/rejected": -0.8500000238418579,
      "logps/chosen": -0.693066418170929,
      "logps/rejected": -1.4611327648162842,
      "loss": 0.9866,
      "nll_loss": 0.9554687738418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06929931789636612,
      "rewards/margins": 0.07676391303539276,
      "rewards/rejected": -0.14606933295726776,
      "step": 2580
    },
    {
      "epoch": 0.1965396873577174,
      "grad_norm": 1.67777232703704,
      "learning_rate": 1.5719549837837187e-06,
      "log_odds_chosen": 0.979296863079071,
      "log_odds_ratio": -0.48750001192092896,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.916210949420929,
      "logps/chosen": -0.693652331829071,
      "logps/rejected": -1.3572266101837158,
      "loss": 0.9592,
      "nll_loss": 0.8955078125,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06942138820886612,
      "rewards/margins": 0.06621094048023224,
      "rewards/rejected": -0.13564452528953552,
      "step": 2590
    },
    {
      "epoch": 0.19729852784944604,
      "grad_norm": 1.847138545524677,
      "learning_rate": 1.5689290811054722e-06,
      "log_odds_chosen": 1.246191382408142,
      "log_odds_ratio": -0.446044921875,
      "logits/chosen": -0.973437488079071,
      "logits/rejected": -0.8705078363418579,
      "logps/chosen": -0.675000011920929,
      "logps/rejected": -1.5144531726837158,
      "loss": 0.9621,
      "nll_loss": 0.9115234613418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06757812201976776,
      "rewards/margins": 0.08388672024011612,
      "rewards/rejected": -0.1513671875,
      "step": 2600
    },
    {
      "epoch": 0.19805736834117468,
      "grad_norm": 1.8218694874948793,
      "learning_rate": 1.5659205853725426e-06,
      "log_odds_chosen": 1.093359351158142,
      "log_odds_ratio": -0.47246092557907104,
      "logits/chosen": -0.990234375,
      "logits/rejected": -0.8970702886581421,
      "logps/chosen": -0.67236328125,
      "logps/rejected": -1.4619140625,
      "loss": 0.9777,
      "nll_loss": 0.9501953125,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06721191108226776,
      "rewards/margins": 0.07899780571460724,
      "rewards/rejected": -0.14621582627296448,
      "step": 2610
    },
    {
      "epoch": 0.19881620883290332,
      "grad_norm": 1.837412074872585,
      "learning_rate": 1.562929330329127e-06,
      "log_odds_chosen": 0.895214855670929,
      "log_odds_ratio": -0.4990234375,
      "logits/chosen": -0.9296875,
      "logits/rejected": -0.865234375,
      "logps/chosen": -0.706835925579071,
      "logps/rejected": -1.3162109851837158,
      "loss": 0.9985,
      "nll_loss": 1.008398413658142,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07066650688648224,
      "rewards/margins": 0.06090698391199112,
      "rewards/rejected": -0.13166503608226776,
      "step": 2620
    },
    {
      "epoch": 0.19957504932463196,
      "grad_norm": 1.7043281951830767,
      "learning_rate": 1.5599551519340636e-06,
      "log_odds_chosen": 1.339453101158142,
      "log_odds_ratio": -0.38862305879592896,
      "logits/chosen": -1.014257788658142,
      "logits/rejected": -0.9058593511581421,
      "logps/chosen": -0.6318359375,
      "logps/rejected": -1.546289086341858,
      "loss": 0.9407,
      "nll_loss": 0.896679699420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06322021782398224,
      "rewards/margins": 0.091400146484375,
      "rewards/rejected": -0.15456542372703552,
      "step": 2630
    },
    {
      "epoch": 0.2003338898163606,
      "grad_norm": 1.7931003101605318,
      "learning_rate": 1.556997888323046e-06,
      "log_odds_chosen": 1.1256835460662842,
      "log_odds_ratio": -0.44111329317092896,
      "logits/chosen": -0.958984375,
      "logits/rejected": -0.8802734613418579,
      "logps/chosen": -0.689257800579071,
      "logps/rejected": -1.437890648841858,
      "loss": 0.9724,
      "nll_loss": 0.929492175579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06889648735523224,
      "rewards/margins": 0.0749664306640625,
      "rewards/rejected": -0.14387206733226776,
      "step": 2640
    },
    {
      "epoch": 0.20109273030808925,
      "grad_norm": 1.571974698546549,
      "learning_rate": 1.5540573797716226e-06,
      "log_odds_chosen": 1.1706054210662842,
      "log_odds_ratio": -0.45585936307907104,
      "logits/chosen": -0.959765613079071,
      "logits/rejected": -0.8968750238418579,
      "logps/chosen": -0.676953136920929,
      "logps/rejected": -1.4599609375,
      "loss": 0.9565,
      "nll_loss": 0.900585949420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06772460788488388,
      "rewards/margins": 0.078369140625,
      "rewards/rejected": -0.14604492485523224,
      "step": 2650
    },
    {
      "epoch": 0.2018515707998179,
      "grad_norm": 1.8202591383722864,
      "learning_rate": 1.5511334686589623e-06,
      "log_odds_chosen": 1.417089819908142,
      "log_odds_ratio": -0.3907714784145355,
      "logits/chosen": -0.9935547113418579,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -0.65087890625,
      "logps/rejected": -1.615625023841858,
      "loss": 0.9577,
      "nll_loss": 0.876171886920929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06508789211511612,
      "rewards/margins": 0.09650878608226776,
      "rewards/rejected": -0.16157226264476776,
      "step": 2660
    },
    {
      "epoch": 0.20261041129154653,
      "grad_norm": 1.8834458505225085,
      "learning_rate": 1.548225999432367e-06,
      "log_odds_chosen": 1.3351562023162842,
      "log_odds_ratio": -0.41259765625,
      "logits/chosen": -0.982617199420929,
      "logits/rejected": -0.847851574420929,
      "logps/chosen": -0.672167956829071,
      "logps/rejected": -1.5802733898162842,
      "loss": 0.9765,
      "nll_loss": 0.915820300579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06721191108226776,
      "rewards/margins": 0.09076537936925888,
      "rewards/rejected": -0.15788574516773224,
      "step": 2670
    },
    {
      "epoch": 0.20336925178327517,
      "grad_norm": 2.0250133124028173,
      "learning_rate": 1.5453348185725114e-06,
      "log_odds_chosen": 0.9356933832168579,
      "log_odds_ratio": -0.510546863079071,
      "logits/chosen": -0.9546874761581421,
      "logits/rejected": -0.8519531488418579,
      "logps/chosen": -0.7032226324081421,
      "logps/rejected": -1.329687476158142,
      "loss": 1.0076,
      "nll_loss": 0.931445300579071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07032470405101776,
      "rewards/margins": 0.06263580173254013,
      "rewards/rejected": -0.13298340141773224,
      "step": 2680
    },
    {
      "epoch": 0.20412809227500378,
      "grad_norm": 2.386913601696167,
      "learning_rate": 1.542459774559398e-06,
      "log_odds_chosen": 1.2929198741912842,
      "log_odds_ratio": -0.4048828184604645,
      "logits/chosen": -0.96484375,
      "logits/rejected": -0.892578125,
      "logps/chosen": -0.708789050579071,
      "logps/rejected": -1.5808594226837158,
      "loss": 0.9777,
      "nll_loss": 0.9580078125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07087402045726776,
      "rewards/margins": 0.08717040717601776,
      "rewards/rejected": -0.15803222358226776,
      "step": 2690
    },
    {
      "epoch": 0.20488693276673242,
      "grad_norm": 1.8473571912546065,
      "learning_rate": 1.539600717839002e-06,
      "log_odds_chosen": 1.0650513172149658,
      "log_odds_ratio": -0.501953125,
      "logits/chosen": -0.962695300579071,
      "logits/rejected": -0.890429675579071,
      "logps/chosen": -0.742968738079071,
      "logps/rejected": -1.5226562023162842,
      "loss": 0.9632,
      "nll_loss": 0.9105468988418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07431640475988388,
      "rewards/margins": 0.07787322998046875,
      "rewards/rejected": -0.15224608778953552,
      "step": 2700
    },
    {
      "epoch": 0.20564577325846106,
      "grad_norm": 1.872727304303063,
      "learning_rate": 1.536757500790597e-06,
      "log_odds_chosen": 1.247460961341858,
      "log_odds_ratio": -0.4300781190395355,
      "logits/chosen": -0.9798828363418579,
      "logits/rejected": -0.8876953125,
      "logps/chosen": -0.7367187738418579,
      "logps/rejected": -1.5886719226837158,
      "loss": 0.9677,
      "nll_loss": 0.928906261920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07359619438648224,
      "rewards/margins": 0.08521118015050888,
      "rewards/rejected": -0.15895995497703552,
      "step": 2710
    },
    {
      "epoch": 0.2064046137501897,
      "grad_norm": 1.7627286806989997,
      "learning_rate": 1.5339299776947407e-06,
      "log_odds_chosen": 1.3813965320587158,
      "log_odds_ratio": -0.44189453125,
      "logits/chosen": -0.9408203363418579,
      "logits/rejected": -0.8451172113418579,
      "logps/chosen": -0.6982421875,
      "logps/rejected": -1.683203101158142,
      "loss": 0.967,
      "nll_loss": 0.920703113079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06981201469898224,
      "rewards/margins": 0.09839019924402237,
      "rewards/rejected": -0.16828612983226776,
      "step": 2720
    },
    {
      "epoch": 0.20716345424191834,
      "grad_norm": 2.225315838235998,
      "learning_rate": 1.5311180047019054e-06,
      "log_odds_chosen": 0.9217284917831421,
      "log_odds_ratio": -0.5398925542831421,
      "logits/chosen": -0.8974609375,
      "logits/rejected": -0.8558593988418579,
      "logps/chosen": -0.7349609136581421,
      "logps/rejected": -1.3703124523162842,
      "loss": 0.9745,
      "nll_loss": 0.9449218511581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07352294772863388,
      "rewards/margins": 0.06348266452550888,
      "rewards/rejected": -0.136962890625,
      "step": 2730
    },
    {
      "epoch": 0.20792229473364698,
      "grad_norm": 1.7123621724820337,
      "learning_rate": 1.5283214398017402e-06,
      "log_odds_chosen": 1.113378882408142,
      "log_odds_ratio": -0.4542480409145355,
      "logits/chosen": -1.009765625,
      "logits/rejected": -0.9136718511581421,
      "logps/chosen": -0.660937488079071,
      "logps/rejected": -1.4210937023162842,
      "loss": 0.9636,
      "nll_loss": 0.865429699420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0660400390625,
      "rewards/margins": 0.07600097358226776,
      "rewards/rejected": -0.14206543564796448,
      "step": 2740
    },
    {
      "epoch": 0.20868113522537562,
      "grad_norm": 1.8912075942460569,
      "learning_rate": 1.5255401427929477e-06,
      "log_odds_chosen": 1.3098633289337158,
      "log_odds_ratio": -0.4039062559604645,
      "logits/chosen": -0.9701172113418579,
      "logits/rejected": -0.884570300579071,
      "logps/chosen": -0.669628918170929,
      "logps/rejected": -1.5916016101837158,
      "loss": 0.9523,
      "nll_loss": 0.807910144329071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06696777045726776,
      "rewards/margins": 0.09216918796300888,
      "rewards/rejected": -0.15913085639476776,
      "step": 2750
    },
    {
      "epoch": 0.20943997571710427,
      "grad_norm": 1.9111616410727177,
      "learning_rate": 1.5227739752537617e-06,
      "log_odds_chosen": 1.12451171875,
      "log_odds_ratio": -0.4161621034145355,
      "logits/chosen": -0.992382824420929,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.6781250238418579,
      "logps/rejected": -1.424414038658142,
      "loss": 0.9826,
      "nll_loss": 0.9384765625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06783447414636612,
      "rewards/margins": 0.07457275688648224,
      "rewards/rejected": -0.1424560546875,
      "step": 2760
    },
    {
      "epoch": 0.2101988162088329,
      "grad_norm": 2.054694787863413,
      "learning_rate": 1.5200228005130127e-06,
      "log_odds_chosen": 1.0719726085662842,
      "log_odds_ratio": -0.48759764432907104,
      "logits/chosen": -0.9869140386581421,
      "logits/rejected": -0.9039062261581421,
      "logps/chosen": -0.7115234136581421,
      "logps/rejected": -1.458398461341858,
      "loss": 0.9641,
      "nll_loss": 0.932421863079071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07111816108226776,
      "rewards/margins": 0.07462158054113388,
      "rewards/rejected": -0.145751953125,
      "step": 2770
    },
    {
      "epoch": 0.21095765670056155,
      "grad_norm": 1.9682083058062396,
      "learning_rate": 1.5172864836217631e-06,
      "log_odds_chosen": 1.0216796398162842,
      "log_odds_ratio": -0.513427734375,
      "logits/chosen": -0.9623047113418579,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.7049804925918579,
      "logps/rejected": -1.436914086341858,
      "loss": 0.9801,
      "nll_loss": 0.90283203125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07052002102136612,
      "rewards/margins": 0.07314453274011612,
      "rewards/rejected": -0.14362792670726776,
      "step": 2780
    },
    {
      "epoch": 0.2117164971922902,
      "grad_norm": 1.7076823298104469,
      "learning_rate": 1.514564891325506e-06,
      "log_odds_chosen": 1.11669921875,
      "log_odds_ratio": -0.45429688692092896,
      "logits/chosen": -0.9652343988418579,
      "logits/rejected": -0.8560546636581421,
      "logps/chosen": -0.624707043170929,
      "logps/rejected": -1.333984375,
      "loss": 0.9698,
      "nll_loss": 0.918749988079071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06241454929113388,
      "rewards/margins": 0.07102050632238388,
      "rewards/rejected": -0.13339844346046448,
      "step": 2790
    },
    {
      "epoch": 0.21247533768401883,
      "grad_norm": 1.8454933653782706,
      "learning_rate": 1.5118578920369086e-06,
      "log_odds_chosen": 1.041650414466858,
      "log_odds_ratio": -0.47929686307907104,
      "logits/chosen": -0.971484363079071,
      "logits/rejected": -0.8687499761581421,
      "logps/chosen": -0.73779296875,
      "logps/rejected": -1.4578125476837158,
      "loss": 0.9761,
      "nll_loss": 0.9339843988418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07375488430261612,
      "rewards/margins": 0.07195434719324112,
      "rewards/rejected": -0.14582519233226776,
      "step": 2800
    },
    {
      "epoch": 0.21323417817574747,
      "grad_norm": 2.3111648063323242,
      "learning_rate": 1.5091653558090898e-06,
      "log_odds_chosen": 1.108007788658142,
      "log_odds_ratio": -0.448486328125,
      "logits/chosen": -0.935351550579071,
      "logits/rejected": -0.8687499761581421,
      "logps/chosen": -0.7041015625,
      "logps/rejected": -1.473046898841858,
      "loss": 0.9538,
      "nll_loss": 0.876757800579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07039795070886612,
      "rewards/margins": 0.076904296875,
      "rewards/rejected": -0.14731445908546448,
      "step": 2810
    },
    {
      "epoch": 0.2139930186674761,
      "grad_norm": 4.148395754218525,
      "learning_rate": 1.506487154309419e-06,
      "log_odds_chosen": 1.150976538658142,
      "log_odds_ratio": -0.45585936307907104,
      "logits/chosen": -0.928515613079071,
      "logits/rejected": -0.889453113079071,
      "logps/chosen": -0.7118164300918579,
      "logps/rejected": -1.466406226158142,
      "loss": 0.9455,
      "nll_loss": 0.878710925579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07124023139476776,
      "rewards/margins": 0.07537536323070526,
      "rewards/rejected": -0.14663085341453552,
      "step": 2820
    },
    {
      "epoch": 0.21475185915920472,
      "grad_norm": 1.9726707897843883,
      "learning_rate": 1.5038231607938247e-06,
      "log_odds_chosen": 1.0893065929412842,
      "log_odds_ratio": -0.48725587129592896,
      "logits/chosen": -0.9554687738418579,
      "logits/rejected": -0.862109363079071,
      "logps/chosen": -0.7093750238418579,
      "logps/rejected": -1.4777343273162842,
      "loss": 0.9787,
      "nll_loss": 0.935742199420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07087402045726776,
      "rewards/margins": 0.07680968940258026,
      "rewards/rejected": -0.1475830078125,
      "step": 2830
    },
    {
      "epoch": 0.21551069965093336,
      "grad_norm": 1.8387982254575808,
      "learning_rate": 1.501173250081603e-06,
      "log_odds_chosen": 1.326562523841858,
      "log_odds_ratio": -0.43168944120407104,
      "logits/chosen": -0.973828136920929,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.6865234375,
      "logps/rejected": -1.63671875,
      "loss": 0.9605,
      "nll_loss": 0.9320312738418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.068603515625,
      "rewards/margins": 0.09507445991039276,
      "rewards/rejected": -0.16372069716453552,
      "step": 2840
    },
    {
      "epoch": 0.216269540142662,
      "grad_norm": 2.2021903588634215,
      "learning_rate": 1.4985372985307103e-06,
      "log_odds_chosen": 1.35150146484375,
      "log_odds_ratio": -0.40996092557907104,
      "logits/chosen": -0.947265625,
      "logits/rejected": -0.8636718988418579,
      "logps/chosen": -0.686328113079071,
      "logps/rejected": -1.6365234851837158,
      "loss": 0.9485,
      "nll_loss": 0.9105468988418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06868896633386612,
      "rewards/margins": 0.09504852443933487,
      "rewards/rejected": -0.16367188096046448,
      "step": 2850
    },
    {
      "epoch": 0.21702838063439064,
      "grad_norm": 1.7415261017566501,
      "learning_rate": 1.4959151840135313e-06,
      "log_odds_chosen": 1.178320288658142,
      "log_odds_ratio": -0.4439453184604645,
      "logits/chosen": -0.990429699420929,
      "logits/rejected": -0.90234375,
      "logps/chosen": -0.7320312261581421,
      "logps/rejected": -1.5597655773162842,
      "loss": 0.9675,
      "nll_loss": 0.963085949420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07321777194738388,
      "rewards/margins": 0.08275756984949112,
      "rewards/rejected": -0.15598145127296448,
      "step": 2860
    },
    {
      "epoch": 0.21778722112611928,
      "grad_norm": 1.7962173674694326,
      "learning_rate": 1.4933067858931148e-06,
      "log_odds_chosen": 1.2593505382537842,
      "log_odds_ratio": -0.41767579317092896,
      "logits/chosen": -0.9828125238418579,
      "logits/rejected": -0.885546863079071,
      "logps/chosen": -0.66357421875,
      "logps/rejected": -1.4919922351837158,
      "loss": 0.9735,
      "nll_loss": 0.9029296636581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06641845405101776,
      "rewards/margins": 0.08284759521484375,
      "rewards/rejected": -0.14924316108226776,
      "step": 2870
    },
    {
      "epoch": 0.21854606161784793,
      "grad_norm": 1.8444965000372955,
      "learning_rate": 1.4907119849998597e-06,
      "log_odds_chosen": 1.254296898841858,
      "log_odds_ratio": -0.45166015625,
      "logits/chosen": -0.9599609375,
      "logits/rejected": -0.887890636920929,
      "logps/chosen": -0.6917968988418579,
      "logps/rejected": -1.537109375,
      "loss": 0.9678,
      "nll_loss": 0.9164062738418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.069091796875,
      "rewards/margins": 0.08457031100988388,
      "rewards/rejected": -0.15373535454273224,
      "step": 2880
    },
    {
      "epoch": 0.21930490210957657,
      "grad_norm": 1.7432037993290854,
      "learning_rate": 1.488130663608649e-06,
      "log_odds_chosen": 1.1090819835662842,
      "log_odds_ratio": -0.43535155057907104,
      "logits/chosen": -0.935546875,
      "logits/rejected": -0.8447265625,
      "logps/chosen": -0.64599609375,
      "logps/rejected": -1.378515601158142,
      "loss": 0.9707,
      "nll_loss": 0.8880859613418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0645751953125,
      "rewards/margins": 0.07330322265625,
      "rewards/rejected": -0.13786621391773224,
      "step": 2890
    },
    {
      "epoch": 0.2200637426013052,
      "grad_norm": 1.8270299370282952,
      "learning_rate": 1.4855627054164149e-06,
      "log_odds_chosen": 1.2333984375,
      "log_odds_ratio": -0.42988282442092896,
      "logits/chosen": -0.9619140625,
      "logits/rejected": -0.8291015625,
      "logps/chosen": -0.6923828125,
      "logps/rejected": -1.574609398841858,
      "loss": 0.9413,
      "nll_loss": 0.9408203363418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06927490234375,
      "rewards/margins": 0.08817748725414276,
      "rewards/rejected": -0.157470703125,
      "step": 2900
    },
    {
      "epoch": 0.22082258309303385,
      "grad_norm": 1.9292240039959558,
      "learning_rate": 1.4830079955201294e-06,
      "log_odds_chosen": 1.2151367664337158,
      "log_odds_ratio": -0.44414061307907104,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.8570312261581421,
      "logps/chosen": -0.715039074420929,
      "logps/rejected": -1.593164086341858,
      "loss": 0.9494,
      "nll_loss": 0.9208984375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07150878757238388,
      "rewards/margins": 0.08788452297449112,
      "rewards/rejected": -0.15939942002296448,
      "step": 2910
    },
    {
      "epoch": 0.2215814235847625,
      "grad_norm": 2.581943848053353,
      "learning_rate": 1.4804664203952103e-06,
      "log_odds_chosen": 1.3484375476837158,
      "log_odds_ratio": -0.41411131620407104,
      "logits/chosen": -0.9193359613418579,
      "logits/rejected": -0.8314453363418579,
      "logps/chosen": -0.697460949420929,
      "logps/rejected": -1.636328101158142,
      "loss": 0.9562,
      "nll_loss": 0.967968761920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06971435248851776,
      "rewards/margins": 0.09394530951976776,
      "rewards/rejected": -0.16376952826976776,
      "step": 2920
    },
    {
      "epoch": 0.22234026407649113,
      "grad_norm": 1.7988904180171552,
      "learning_rate": 1.4779378678743327e-06,
      "log_odds_chosen": 1.0883057117462158,
      "log_odds_ratio": -0.501660168170929,
      "logits/chosen": -0.9693359136581421,
      "logits/rejected": -0.90234375,
      "logps/chosen": -0.715136706829071,
      "logps/rejected": -1.501953125,
      "loss": 0.9604,
      "nll_loss": 0.933789074420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07152099907398224,
      "rewards/margins": 0.07857055962085724,
      "rewards/rejected": -0.15004882216453552,
      "step": 2930
    },
    {
      "epoch": 0.22309910456821977,
      "grad_norm": 1.9160010373129888,
      "learning_rate": 1.4754222271266348e-06,
      "log_odds_chosen": 1.13330078125,
      "log_odds_ratio": -0.47773438692092896,
      "logits/chosen": -0.9613281488418579,
      "logits/rejected": -0.8382812738418579,
      "logps/chosen": -0.697070300579071,
      "logps/rejected": -1.5056641101837158,
      "loss": 0.9634,
      "nll_loss": 0.8658202886581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06971435248851776,
      "rewards/margins": 0.08093567192554474,
      "rewards/rejected": -0.15053710341453552,
      "step": 2940
    },
    {
      "epoch": 0.2238579450599484,
      "grad_norm": 1.861090283579612,
      "learning_rate": 1.4729193886373175e-06,
      "log_odds_chosen": 1.147375464439392,
      "log_odds_ratio": -0.4544921815395355,
      "logits/chosen": -0.9873046875,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.658203125,
      "logps/rejected": -1.4572265148162842,
      "loss": 0.9554,
      "nll_loss": 0.822460949420929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06580810248851776,
      "rewards/margins": 0.07994689792394638,
      "rewards/rejected": -0.14572754502296448,
      "step": 2950
    },
    {
      "epoch": 0.22461678555167705,
      "grad_norm": 2.8696331367440693,
      "learning_rate": 1.4704292441876156e-06,
      "log_odds_chosen": 1.3240234851837158,
      "log_odds_ratio": -0.398193359375,
      "logits/chosen": -0.9740234613418579,
      "logits/rejected": -0.857226550579071,
      "logps/chosen": -0.65380859375,
      "logps/rejected": -1.5812499523162842,
      "loss": 0.949,
      "nll_loss": 0.836718738079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0654296875,
      "rewards/margins": 0.09274597465991974,
      "rewards/rejected": -0.15810546278953552,
      "step": 2960
    },
    {
      "epoch": 0.22537562604340566,
      "grad_norm": 1.6682423889188396,
      "learning_rate": 1.4679516868351474e-06,
      "log_odds_chosen": 1.2998046875,
      "log_odds_ratio": -0.4522460997104645,
      "logits/chosen": -0.9380859136581421,
      "logits/rejected": -0.828320324420929,
      "logps/chosen": -0.75,
      "logps/rejected": -1.726953148841858,
      "loss": 0.9733,
      "nll_loss": 0.953320324420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07496337592601776,
      "rewards/margins": 0.09773864597082138,
      "rewards/rejected": -0.1727294921875,
      "step": 2970
    },
    {
      "epoch": 0.2261344665351343,
      "grad_norm": 1.8546074916340258,
      "learning_rate": 1.4654866108946234e-06,
      "log_odds_chosen": 0.9668945074081421,
      "log_odds_ratio": -0.508056640625,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.862109363079071,
      "logps/chosen": -0.647753894329071,
      "logps/rejected": -1.292382836341858,
      "loss": 0.9674,
      "nll_loss": 0.953320324420929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06480713188648224,
      "rewards/margins": 0.06438598781824112,
      "rewards/rejected": -0.12919922173023224,
      "step": 2980
    },
    {
      "epoch": 0.22689330702686294,
      "grad_norm": 1.7596453202428828,
      "learning_rate": 1.4630339119189101e-06,
      "log_odds_chosen": 0.931640625,
      "log_odds_ratio": -0.5057617425918579,
      "logits/chosen": -0.935546875,
      "logits/rejected": -0.845898449420929,
      "logps/chosen": -0.7242187261581421,
      "logps/rejected": -1.37890625,
      "loss": 0.9486,
      "nll_loss": 0.899609386920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07239989936351776,
      "rewards/margins": 0.06554565578699112,
      "rewards/rejected": -0.13789062201976776,
      "step": 2990
    },
    {
      "epoch": 0.22765214751859159,
      "grad_norm": 1.847450689305971,
      "learning_rate": 1.4605934866804429e-06,
      "log_odds_chosen": 1.257592797279358,
      "log_odds_ratio": -0.400146484375,
      "logits/chosen": -0.987500011920929,
      "logits/rejected": -0.8763672113418579,
      "logps/chosen": -0.6636718511581421,
      "logps/rejected": -1.506250023841858,
      "loss": 0.9456,
      "nll_loss": 0.8394531011581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06638183444738388,
      "rewards/margins": 0.08422088623046875,
      "rewards/rejected": -0.15065917372703552,
      "step": 3000
    },
    {
      "epoch": 0.22841098801032023,
      "grad_norm": 1.7257030329384566,
      "learning_rate": 1.4581652331529784e-06,
      "log_odds_chosen": 1.3411133289337158,
      "log_odds_ratio": -0.4209960997104645,
      "logits/chosen": -0.9423828125,
      "logits/rejected": -0.8199218511581421,
      "logps/chosen": -0.6571289300918579,
      "logps/rejected": -1.5955078601837158,
      "loss": 0.9403,
      "nll_loss": 0.8746093511581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06572265923023224,
      "rewards/margins": 0.09383697807788849,
      "rewards/rejected": -0.15966796875,
      "step": 3010
    },
    {
      "epoch": 0.22916982850204887,
      "grad_norm": 1.6543337971685594,
      "learning_rate": 1.4557490504936778e-06,
      "log_odds_chosen": 0.958984375,
      "log_odds_ratio": -0.4776367247104645,
      "logits/chosen": -0.913867175579071,
      "logits/rejected": -0.82421875,
      "logps/chosen": -0.692187488079071,
      "logps/rejected": -1.3244140148162842,
      "loss": 0.9727,
      "nll_loss": 0.958203136920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06922607123851776,
      "rewards/margins": 0.06323852390050888,
      "rewards/rejected": -0.132568359375,
      "step": 3020
    },
    {
      "epoch": 0.2299286689937775,
      "grad_norm": 1.9978653383725091,
      "learning_rate": 1.453344839025519e-06,
      "log_odds_chosen": 1.125830054283142,
      "log_odds_ratio": -0.46538084745407104,
      "logits/chosen": -0.9478515386581421,
      "logits/rejected": -0.8636718988418579,
      "logps/chosen": -0.657910168170929,
      "logps/rejected": -1.415624976158142,
      "loss": 0.943,
      "nll_loss": 0.846484363079071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06577148288488388,
      "rewards/margins": 0.07586517184972763,
      "rewards/rejected": -0.1416015625,
      "step": 3030
    },
    {
      "epoch": 0.23068750948550615,
      "grad_norm": 1.9001759526352406,
      "learning_rate": 1.4509525002200234e-06,
      "log_odds_chosen": 1.4861938953399658,
      "log_odds_ratio": -0.40898436307907104,
      "logits/chosen": -0.9076172113418579,
      "logits/rejected": -0.833789050579071,
      "logps/chosen": -0.7176758050918579,
      "logps/rejected": -1.7810547351837158,
      "loss": 0.9498,
      "nll_loss": 0.9462890625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07175292819738388,
      "rewards/margins": 0.10638885200023651,
      "rewards/rejected": -0.17822265625,
      "step": 3040
    },
    {
      "epoch": 0.2314463499772348,
      "grad_norm": 1.812933320589776,
      "learning_rate": 1.4485719366802965e-06,
      "log_odds_chosen": 1.245507836341858,
      "log_odds_ratio": -0.4149414002895355,
      "logits/chosen": -0.9673827886581421,
      "logits/rejected": -0.8486328125,
      "logps/chosen": -0.669140636920929,
      "logps/rejected": -1.4578125476837158,
      "loss": 0.9417,
      "nll_loss": 0.8720703125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06687011569738388,
      "rewards/margins": 0.07879638671875,
      "rewards/rejected": -0.14580078423023224,
      "step": 3050
    },
    {
      "epoch": 0.23220519046896343,
      "grad_norm": 1.787462020163273,
      "learning_rate": 1.4462030521243742e-06,
      "log_odds_chosen": 1.0899536609649658,
      "log_odds_ratio": -0.4447265565395355,
      "logits/chosen": -0.9195312261581421,
      "logits/rejected": -0.7939453125,
      "logps/chosen": -0.6776367425918579,
      "logps/rejected": -1.380859375,
      "loss": 0.9589,
      "nll_loss": 0.9677734375,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06778564304113388,
      "rewards/margins": 0.07031555473804474,
      "rewards/rejected": -0.13803711533546448,
      "step": 3060
    },
    {
      "epoch": 0.23296403096069207,
      "grad_norm": 1.6858354000752622,
      "learning_rate": 1.443845751368867e-06,
      "log_odds_chosen": 1.0482909679412842,
      "log_odds_ratio": -0.48193359375,
      "logits/chosen": -0.9501953125,
      "logits/rejected": -0.8919922113418579,
      "logps/chosen": -0.6900390386581421,
      "logps/rejected": -1.3957030773162842,
      "loss": 0.9766,
      "nll_loss": 0.935546875,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06901855766773224,
      "rewards/margins": 0.07039795070886612,
      "rewards/rejected": -0.13947753608226776,
      "step": 3070
    },
    {
      "epoch": 0.2337228714524207,
      "grad_norm": 1.8339758263485157,
      "learning_rate": 1.4414999403128943e-06,
      "log_odds_chosen": 1.2374999523162842,
      "log_odds_ratio": -0.44316405057907104,
      "logits/chosen": -0.887890636920929,
      "logits/rejected": -0.8041015863418579,
      "logps/chosen": -0.6859375238418579,
      "logps/rejected": -1.5460937023162842,
      "loss": 0.9417,
      "nll_loss": 0.9515625238418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06850586086511612,
      "rewards/margins": 0.08607177436351776,
      "rewards/rejected": -0.15461425483226776,
      "step": 3080
    },
    {
      "epoch": 0.23448171194414935,
      "grad_norm": 2.068493502798679,
      "learning_rate": 1.439165525922309e-06,
      "log_odds_chosen": 1.157812476158142,
      "log_odds_ratio": -0.4925781190395355,
      "logits/chosen": -0.904492199420929,
      "logits/rejected": -0.8500000238418579,
      "logps/chosen": -0.6498047113418579,
      "logps/rejected": -1.4328124523162842,
      "loss": 0.9524,
      "nll_loss": 0.868359386920929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06501464545726776,
      "rewards/margins": 0.07831726223230362,
      "rewards/rejected": -0.1434326171875,
      "step": 3090
    },
    {
      "epoch": 0.235240552435878,
      "grad_norm": 1.8901274723008876,
      "learning_rate": 1.4368424162141992e-06,
      "log_odds_chosen": 1.166015625,
      "log_odds_ratio": -0.482177734375,
      "logits/chosen": -0.915820300579071,
      "logits/rejected": -0.81640625,
      "logps/chosen": -0.7708984613418579,
      "logps/rejected": -1.6150391101837158,
      "loss": 0.948,
      "nll_loss": 0.9623047113418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07711181789636612,
      "rewards/margins": 0.08429565280675888,
      "rewards/rejected": -0.1614990234375,
      "step": 3100
    },
    {
      "epoch": 0.2359993929276066,
      "grad_norm": 1.9627846293262432,
      "learning_rate": 1.434530520241665e-06,
      "log_odds_chosen": 1.162011742591858,
      "log_odds_ratio": -0.47773438692092896,
      "logits/chosen": -0.9302734136581421,
      "logits/rejected": -0.835742175579071,
      "logps/chosen": -0.658496081829071,
      "logps/rejected": -1.464453101158142,
      "loss": 0.9522,
      "nll_loss": 0.9443359375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06583251804113388,
      "rewards/margins": 0.08067016303539276,
      "rewards/rejected": -0.14645996689796448,
      "step": 3110
    },
    {
      "epoch": 0.23675823341933525,
      "grad_norm": 1.8470239258734957,
      "learning_rate": 1.4322297480788657e-06,
      "log_odds_chosen": 1.0634765625,
      "log_odds_ratio": -0.49458009004592896,
      "logits/chosen": -0.911914050579071,
      "logits/rejected": -0.8257812261581421,
      "logps/chosen": -0.753125011920929,
      "logps/rejected": -1.4982421398162842,
      "loss": 0.9471,
      "nll_loss": 0.931835949420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07525634765625,
      "rewards/margins": 0.074554443359375,
      "rewards/rejected": -0.14982910454273224,
      "step": 3120
    },
    {
      "epoch": 0.2375170739110639,
      "grad_norm": 1.9227094808395646,
      "learning_rate": 1.4299400108063247e-06,
      "log_odds_chosen": 1.065527319908142,
      "log_odds_ratio": -0.4964843690395355,
      "logits/chosen": -0.9326171875,
      "logits/rejected": -0.841796875,
      "logps/chosen": -0.6968749761581421,
      "logps/rejected": -1.4296875,
      "loss": 0.92,
      "nll_loss": 0.9214843511581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0697021484375,
      "rewards/margins": 0.07349243015050888,
      "rewards/rejected": -0.14306640625,
      "step": 3130
    },
    {
      "epoch": 0.23827591440279253,
      "grad_norm": 1.8725285434199292,
      "learning_rate": 1.4276612204964992e-06,
      "log_odds_chosen": 1.26513671875,
      "log_odds_ratio": -0.45307618379592896,
      "logits/chosen": -0.918164074420929,
      "logits/rejected": -0.8291015625,
      "logps/chosen": -0.675585925579071,
      "logps/rejected": -1.580078125,
      "loss": 0.9348,
      "nll_loss": 0.960742175579071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06754150241613388,
      "rewards/margins": 0.09055785834789276,
      "rewards/rejected": -0.15803222358226776,
      "step": 3140
    },
    {
      "epoch": 0.23903475489452117,
      "grad_norm": 1.7486311411327649,
      "learning_rate": 1.4253932901995967e-06,
      "log_odds_chosen": 1.062890648841858,
      "log_odds_ratio": -0.4705566465854645,
      "logits/chosen": -0.893359363079071,
      "logits/rejected": -0.814257800579071,
      "logps/chosen": -0.67822265625,
      "logps/rejected": -1.414648413658142,
      "loss": 0.9437,
      "nll_loss": 0.8882812261581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06781005859375,
      "rewards/margins": 0.07365112006664276,
      "rewards/rejected": -0.14145508408546448,
      "step": 3150
    },
    {
      "epoch": 0.2397935953862498,
      "grad_norm": 1.871562780745004,
      "learning_rate": 1.42313613392964e-06,
      "log_odds_chosen": 1.2360718250274658,
      "log_odds_ratio": -0.41899412870407104,
      "logits/chosen": -0.942187488079071,
      "logits/rejected": -0.836718738079071,
      "logps/chosen": -0.6473633050918579,
      "logps/rejected": -1.494140625,
      "loss": 0.927,
      "nll_loss": 0.8384765386581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06468506157398224,
      "rewards/margins": 0.08463592827320099,
      "rewards/rejected": -0.14943847060203552,
      "step": 3160
    },
    {
      "epoch": 0.24055243587797845,
      "grad_norm": 1.9319998881984413,
      "learning_rate": 1.4208896666507756e-06,
      "log_odds_chosen": 1.1755859851837158,
      "log_odds_ratio": -0.47407227754592896,
      "logits/chosen": -0.883984386920929,
      "logits/rejected": -0.807421863079071,
      "logps/chosen": -0.6717773675918579,
      "logps/rejected": -1.48046875,
      "loss": 0.9449,
      "nll_loss": 0.9410156011581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06711425632238388,
      "rewards/margins": 0.08091125637292862,
      "rewards/rejected": -0.1480712890625,
      "step": 3170
    },
    {
      "epoch": 0.2413112763697071,
      "grad_norm": 2.812202784971474,
      "learning_rate": 1.4186538042638173e-06,
      "log_odds_chosen": 1.2048828601837158,
      "log_odds_ratio": -0.439453125,
      "logits/chosen": -0.9654296636581421,
      "logits/rejected": -0.8451172113418579,
      "logps/chosen": -0.736328125,
      "logps/rejected": -1.6017577648162842,
      "loss": 0.9318,
      "nll_loss": 0.9462890625,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07366943359375,
      "rewards/margins": 0.08660583198070526,
      "rewards/rejected": -0.16021728515625,
      "step": 3180
    },
    {
      "epoch": 0.24207011686143573,
      "grad_norm": 1.8365782277502412,
      "learning_rate": 1.416428463593022e-06,
      "log_odds_chosen": 1.343164086341858,
      "log_odds_ratio": -0.4187988340854645,
      "logits/chosen": -0.939648449420929,
      "logits/rejected": -0.803906261920929,
      "logps/chosen": -0.6578124761581421,
      "logps/rejected": -1.591796875,
      "loss": 0.9503,
      "nll_loss": 0.8871093988418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06582031399011612,
      "rewards/margins": 0.09349365532398224,
      "rewards/rejected": -0.15922851860523224,
      "step": 3190
    },
    {
      "epoch": 0.24282895735316437,
      "grad_norm": 1.8323563524467135,
      "learning_rate": 1.414213562373095e-06,
      "log_odds_chosen": 1.247460961341858,
      "log_odds_ratio": -0.44145506620407104,
      "logits/chosen": -0.9507812261581421,
      "logits/rejected": -0.8531249761581421,
      "logps/chosen": -0.6361328363418579,
      "logps/rejected": -1.495507836341858,
      "loss": 0.9085,
      "nll_loss": 0.9072265625,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.063720703125,
      "rewards/margins": 0.0858154296875,
      "rewards/rejected": -0.14948730170726776,
      "step": 3200
    },
    {
      "epoch": 0.243587797844893,
      "grad_norm": 1.9099153823357262,
      "learning_rate": 1.4120090192364154e-06,
      "log_odds_chosen": 1.0369141101837158,
      "log_odds_ratio": -0.4970703125,
      "logits/chosen": -0.969531238079071,
      "logits/rejected": -0.8443359136581421,
      "logps/chosen": -0.691601574420929,
      "logps/rejected": -1.372656226158142,
      "loss": 0.9456,
      "nll_loss": 0.9193359613418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06915283203125,
      "rewards/margins": 0.06818237155675888,
      "rewards/rejected": -0.1373291015625,
      "step": 3210
    },
    {
      "epoch": 0.24434663833662165,
      "grad_norm": 2.0375256450793087,
      "learning_rate": 1.4098147537004828e-06,
      "log_odds_chosen": 1.133935570716858,
      "log_odds_ratio": -0.44086915254592896,
      "logits/chosen": -0.9365234375,
      "logits/rejected": -0.8529297113418579,
      "logps/chosen": -0.646679699420929,
      "logps/rejected": -1.3826172351837158,
      "loss": 0.9461,
      "nll_loss": 0.9072265625,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06463623046875,
      "rewards/margins": 0.07357482612133026,
      "rewards/rejected": -0.13820800185203552,
      "step": 3220
    },
    {
      "epoch": 0.2451054788283503,
      "grad_norm": 1.9912908730465613,
      "learning_rate": 1.4076306861555735e-06,
      "log_odds_chosen": 1.2411620616912842,
      "log_odds_ratio": -0.41997069120407104,
      "logits/chosen": -0.9458984136581421,
      "logits/rejected": -0.8486328125,
      "logps/chosen": -0.67578125,
      "logps/rejected": -1.5251953601837158,
      "loss": 0.9424,
      "nll_loss": 0.9007812738418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06752929836511612,
      "rewards/margins": 0.08501434326171875,
      "rewards/rejected": -0.15249022841453552,
      "step": 3230
    },
    {
      "epoch": 0.2458643193200789,
      "grad_norm": 2.274586805113359,
      "learning_rate": 1.405456737852613e-06,
      "log_odds_chosen": 1.1833984851837158,
      "log_odds_ratio": -0.4684081971645355,
      "logits/chosen": -0.9150390625,
      "logits/rejected": -0.835156261920929,
      "logps/chosen": -0.70703125,
      "logps/rejected": -1.558007836341858,
      "loss": 0.9503,
      "nll_loss": 0.878125011920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0706787109375,
      "rewards/margins": 0.08520660549402237,
      "rewards/rejected": -0.15581054985523224,
      "step": 3240
    },
    {
      "epoch": 0.24662315981180755,
      "grad_norm": 1.931992941128289,
      "learning_rate": 1.4032928308912468e-06,
      "log_odds_chosen": 1.2144043445587158,
      "log_odds_ratio": -0.4669433534145355,
      "logits/chosen": -0.9580078125,
      "logits/rejected": -0.8726562261581421,
      "logps/chosen": -0.6917968988418579,
      "logps/rejected": -1.5554687976837158,
      "loss": 0.9286,
      "nll_loss": 0.8759765625,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06925048679113388,
      "rewards/margins": 0.08652496337890625,
      "rewards/rejected": -0.15571288764476776,
      "step": 3250
    },
    {
      "epoch": 0.2473820003035362,
      "grad_norm": 1.889574173524338,
      "learning_rate": 1.4011388882081175e-06,
      "log_odds_chosen": 1.2414062023162842,
      "log_odds_ratio": -0.44091796875,
      "logits/chosen": -0.9189453125,
      "logits/rejected": -0.818164050579071,
      "logps/chosen": -0.604687511920929,
      "logps/rejected": -1.4462890625,
      "loss": 0.9378,
      "nll_loss": 0.8216797113418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06051025539636612,
      "rewards/margins": 0.08405761420726776,
      "rewards/rejected": -0.14462891221046448,
      "step": 3260
    },
    {
      "epoch": 0.24814084079526483,
      "grad_norm": 1.9404559201119644,
      "learning_rate": 1.3989948335653378e-06,
      "log_odds_chosen": 1.404687523841858,
      "log_odds_ratio": -0.427978515625,
      "logits/chosen": -1.0126953125,
      "logits/rejected": -0.861523449420929,
      "logps/chosen": -0.62841796875,
      "logps/rejected": -1.622460961341858,
      "loss": 0.9268,
      "nll_loss": 0.807910144329071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06280517578125,
      "rewards/margins": 0.09949950873851776,
      "rewards/rejected": -0.1622314453125,
      "step": 3270
    },
    {
      "epoch": 0.24889968128699347,
      "grad_norm": 1.7452378668385211,
      "learning_rate": 1.3968605915391564e-06,
      "log_odds_chosen": 1.380468726158142,
      "log_odds_ratio": -0.4466796815395355,
      "logits/chosen": -0.944531261920929,
      "logits/rejected": -0.848437488079071,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -1.7205078601837158,
      "loss": 0.9524,
      "nll_loss": 0.9287109375,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07064209133386612,
      "rewards/margins": 0.10134277492761612,
      "rewards/rejected": -0.17197266221046448,
      "step": 3280
    },
    {
      "epoch": 0.2496585217787221,
      "grad_norm": 1.9903896388016717,
      "learning_rate": 1.3947360875088132e-06,
      "log_odds_chosen": 1.1735351085662842,
      "log_odds_ratio": -0.46064454317092896,
      "logits/chosen": -0.9371093511581421,
      "logits/rejected": -0.8333984613418579,
      "logps/chosen": -0.70166015625,
      "logps/rejected": -1.510156273841858,
      "loss": 0.9344,
      "nll_loss": 0.8818359375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07014159858226776,
      "rewards/margins": 0.08088989555835724,
      "rewards/rejected": -0.15109863877296448,
      "step": 3290
    },
    {
      "epoch": 0.25041736227045075,
      "grad_norm": 1.825929681479405,
      "learning_rate": 1.3926212476455828e-06,
      "log_odds_chosen": 1.066686987876892,
      "log_odds_ratio": -0.496337890625,
      "logits/chosen": -0.9189453125,
      "logits/rejected": -0.8451172113418579,
      "logps/chosen": -0.681640625,
      "logps/rejected": -1.4054687023162842,
      "loss": 0.9379,
      "nll_loss": 0.9091796875,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06812743842601776,
      "rewards/margins": 0.07235412299633026,
      "rewards/rejected": -0.140625,
      "step": 3300
    },
    {
      "epoch": 0.2511762027621794,
      "grad_norm": 2.1511451858131787,
      "learning_rate": 1.3905159989019964e-06,
      "log_odds_chosen": 1.366601586341858,
      "log_odds_ratio": -0.42021483182907104,
      "logits/chosen": -0.9537109136581421,
      "logits/rejected": -0.8388671875,
      "logps/chosen": -0.6988281011581421,
      "logps/rejected": -1.6652343273162842,
      "loss": 0.9458,
      "nll_loss": 0.897265613079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06990966945886612,
      "rewards/margins": 0.09661865234375,
      "rewards/rejected": -0.16650390625,
      "step": 3310
    },
    {
      "epoch": 0.25193504325390803,
      "grad_norm": 2.4080145026266675,
      "learning_rate": 1.3884202690012465e-06,
      "log_odds_chosen": 1.415136694908142,
      "log_odds_ratio": -0.40556639432907104,
      "logits/chosen": -0.946484386920929,
      "logits/rejected": -0.8427734375,
      "logps/chosen": -0.737011730670929,
      "logps/rejected": -1.7296874523162842,
      "loss": 0.9546,
      "nll_loss": 0.9208984375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07371826469898224,
      "rewards/margins": 0.099334716796875,
      "rewards/rejected": -0.17304687201976776,
      "step": 3320
    },
    {
      "epoch": 0.25269388374563667,
      "grad_norm": 2.136961368070725,
      "learning_rate": 1.3863339864267636e-06,
      "log_odds_chosen": 0.906054675579071,
      "log_odds_ratio": -0.49775391817092896,
      "logits/chosen": -0.9507812261581421,
      "logits/rejected": -0.841601550579071,
      "logps/chosen": -0.6709960699081421,
      "logps/rejected": -1.2195312976837158,
      "loss": 0.9513,
      "nll_loss": 0.88671875,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06708984076976776,
      "rewards/margins": 0.05487060546875,
      "rewards/rejected": -0.12202148139476776,
      "step": 3330
    },
    {
      "epoch": 0.2534527242373653,
      "grad_norm": 1.9884853306075043,
      "learning_rate": 1.3842570804119655e-06,
      "log_odds_chosen": 1.0947265625,
      "log_odds_ratio": -0.44355469942092896,
      "logits/chosen": -0.9673827886581421,
      "logits/rejected": -0.832226574420929,
      "logps/chosen": -0.6509765386581421,
      "logps/rejected": -1.3839843273162842,
      "loss": 0.9472,
      "nll_loss": 0.8648437261581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06513671576976776,
      "rewards/margins": 0.07327880710363388,
      "rewards/rejected": -0.13840332627296448,
      "step": 3340
    },
    {
      "epoch": 0.25421156472909395,
      "grad_norm": 3.7716690402223643,
      "learning_rate": 1.3821894809301763e-06,
      "log_odds_chosen": 1.4836914539337158,
      "log_odds_ratio": -0.3805175721645355,
      "logits/chosen": -0.8980468511581421,
      "logits/rejected": -0.7972656488418579,
      "logps/chosen": -0.64111328125,
      "logps/rejected": -1.6281249523162842,
      "loss": 0.9458,
      "nll_loss": 0.8804687261581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06407471001148224,
      "rewards/margins": 0.09869384765625,
      "rewards/rejected": -0.16264648735523224,
      "step": 3350
    },
    {
      "epoch": 0.2549704052208226,
      "grad_norm": 1.9417352866678919,
      "learning_rate": 1.3801311186847081e-06,
      "log_odds_chosen": 1.279882788658142,
      "log_odds_ratio": -0.42607420682907104,
      "logits/chosen": -0.94921875,
      "logits/rejected": -0.835156261920929,
      "logps/chosen": -0.6883789300918579,
      "logps/rejected": -1.540624976158142,
      "loss": 0.9421,
      "nll_loss": 0.9169921875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06882324069738388,
      "rewards/margins": 0.08518066257238388,
      "rewards/rejected": -0.15407714247703552,
      "step": 3360
    },
    {
      "epoch": 0.25572924571255123,
      "grad_norm": 1.9677525270182894,
      "learning_rate": 1.378081925099109e-06,
      "log_odds_chosen": 1.1848633289337158,
      "log_odds_ratio": -0.41083985567092896,
      "logits/chosen": -0.86328125,
      "logits/rejected": -0.802734375,
      "logps/chosen": -0.6630859375,
      "logps/rejected": -1.443945288658142,
      "loss": 0.9459,
      "nll_loss": 0.881640613079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06630859524011612,
      "rewards/margins": 0.07810668647289276,
      "rewards/rejected": -0.1444091796875,
      "step": 3370
    },
    {
      "epoch": 0.2564880862042799,
      "grad_norm": 1.7058936097763349,
      "learning_rate": 1.376041832307563e-06,
      "log_odds_chosen": 1.240234375,
      "log_odds_ratio": -0.4556640684604645,
      "logits/chosen": -0.948437511920929,
      "logits/rejected": -0.8101562261581421,
      "logps/chosen": -0.694042980670929,
      "logps/rejected": -1.537109375,
      "loss": 0.9528,
      "nll_loss": 0.9375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06947021186351776,
      "rewards/margins": 0.08425293117761612,
      "rewards/rejected": -0.15361328423023224,
      "step": 3380
    },
    {
      "epoch": 0.2572469266960085,
      "grad_norm": 2.2082044702359465,
      "learning_rate": 1.3740107731454524e-06,
      "log_odds_chosen": 1.232031226158142,
      "log_odds_ratio": -0.4321533143520355,
      "logits/chosen": -0.9410156011581421,
      "logits/rejected": -0.8345702886581421,
      "logps/chosen": -0.6962890625,
      "logps/rejected": -1.5615234375,
      "loss": 0.9379,
      "nll_loss": 0.9957031011581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06964111328125,
      "rewards/margins": 0.08657226711511612,
      "rewards/rejected": -0.15615233778953552,
      "step": 3390
    },
    {
      "epoch": 0.25800576718773716,
      "grad_norm": 2.1820637460198142,
      "learning_rate": 1.3719886811400705e-06,
      "log_odds_chosen": 1.3191406726837158,
      "log_odds_ratio": -0.4033203125,
      "logits/chosen": -0.981640636920929,
      "logits/rejected": -0.876953125,
      "logps/chosen": -0.6343749761581421,
      "logps/rejected": -1.5291016101837158,
      "loss": 0.922,
      "nll_loss": 0.8228515386581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06345214694738388,
      "rewards/margins": 0.0894775390625,
      "rewards/rejected": -0.15297850966453552,
      "step": 3400
    },
    {
      "epoch": 0.2587646076794658,
      "grad_norm": 1.8723787448291065,
      "learning_rate": 1.3699754905014834e-06,
      "log_odds_chosen": 1.3383300304412842,
      "log_odds_ratio": -0.427001953125,
      "logits/chosen": -0.9251953363418579,
      "logits/rejected": -0.8294922113418579,
      "logps/chosen": -0.6695312261581421,
      "logps/rejected": -1.616796851158142,
      "loss": 0.9542,
      "nll_loss": 0.871874988079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06691894680261612,
      "rewards/margins": 0.09471359103918076,
      "rewards/rejected": -0.16169433295726776,
      "step": 3410
    },
    {
      "epoch": 0.25952344817119444,
      "grad_norm": 1.909954286522631,
      "learning_rate": 1.3679711361135388e-06,
      "log_odds_chosen": 1.097021460533142,
      "log_odds_ratio": -0.4974609315395355,
      "logits/chosen": -0.9791015386581421,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.7193359136581421,
      "logps/rejected": -1.520898461341858,
      "loss": 0.9201,
      "nll_loss": 0.8841797113418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07192382961511612,
      "rewards/margins": 0.08006133884191513,
      "rewards/rejected": -0.152099609375,
      "step": 3420
    },
    {
      "epoch": 0.2602822886629231,
      "grad_norm": 1.8424699803048412,
      "learning_rate": 1.3659755535250212e-06,
      "log_odds_chosen": 1.4440429210662842,
      "log_odds_ratio": -0.39545899629592896,
      "logits/chosen": -0.941601574420929,
      "logits/rejected": -0.8285156488418579,
      "logps/chosen": -0.63427734375,
      "logps/rejected": -1.6384766101837158,
      "loss": 0.9418,
      "nll_loss": 0.841015636920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06340332329273224,
      "rewards/margins": 0.10048218071460724,
      "rewards/rejected": -0.16384276747703552,
      "step": 3430
    },
    {
      "epoch": 0.2610411291546517,
      "grad_norm": 1.9198743792335182,
      "learning_rate": 1.3639886789409469e-06,
      "log_odds_chosen": 1.4453125,
      "log_odds_ratio": -0.38801270723342896,
      "logits/chosen": -0.9693359136581421,
      "logits/rejected": -0.830859363079071,
      "logps/chosen": -0.6615234613418579,
      "logps/rejected": -1.659765601158142,
      "loss": 0.9337,
      "nll_loss": 0.9193359613418579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06612548977136612,
      "rewards/margins": 0.09989013522863388,
      "rewards/rejected": -0.16608886420726776,
      "step": 3440
    },
    {
      "epoch": 0.2617999696463803,
      "grad_norm": 1.8882359433159133,
      "learning_rate": 1.3620104492139977e-06,
      "log_odds_chosen": 1.552832007408142,
      "log_odds_ratio": -0.38276368379592896,
      "logits/chosen": -0.944531261920929,
      "logits/rejected": -0.8271484375,
      "logps/chosen": -0.650195300579071,
      "logps/rejected": -1.726953148841858,
      "loss": 0.9616,
      "nll_loss": 0.879687488079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06500244140625,
      "rewards/margins": 0.10766907036304474,
      "rewards/rejected": -0.17258301377296448,
      "step": 3450
    },
    {
      "epoch": 0.26255881013810894,
      "grad_norm": 1.9301623087868962,
      "learning_rate": 1.3600408018360918e-06,
      "log_odds_chosen": 1.2575194835662842,
      "log_odds_ratio": -0.4151855409145355,
      "logits/chosen": -0.938671886920929,
      "logits/rejected": -0.852343738079071,
      "logps/chosen": -0.65234375,
      "logps/rejected": -1.4910156726837158,
      "loss": 0.9257,
      "nll_loss": 0.842968761920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06523437798023224,
      "rewards/margins": 0.08380126953125,
      "rewards/rejected": -0.14899902045726776,
      "step": 3460
    },
    {
      "epoch": 0.2633176506298376,
      "grad_norm": 1.9454646415269345,
      "learning_rate": 1.3580796749300878e-06,
      "log_odds_chosen": 1.2312500476837158,
      "log_odds_ratio": -0.4684081971645355,
      "logits/chosen": -0.9351562261581421,
      "logits/rejected": -0.8656250238418579,
      "logps/chosen": -0.7134765386581421,
      "logps/rejected": -1.5994141101837158,
      "loss": 0.9468,
      "nll_loss": 0.8939453363418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07143554836511612,
      "rewards/margins": 0.08856506645679474,
      "rewards/rejected": -0.15996094048023224,
      "step": 3470
    },
    {
      "epoch": 0.2640764911215662,
      "grad_norm": 1.8682090773735338,
      "learning_rate": 1.3561270072416209e-06,
      "log_odds_chosen": 1.100976586341858,
      "log_odds_ratio": -0.4833984375,
      "logits/chosen": -0.9535156488418579,
      "logits/rejected": -0.8607422113418579,
      "logps/chosen": -0.6826171875,
      "logps/rejected": -1.4677734375,
      "loss": 0.9192,
      "nll_loss": 0.895703136920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06829833984375,
      "rewards/margins": 0.07857055962085724,
      "rewards/rejected": -0.14677734673023224,
      "step": 3480
    },
    {
      "epoch": 0.26483533161329487,
      "grad_norm": 2.031528833234933,
      "learning_rate": 1.3541827381310652e-06,
      "log_odds_chosen": 1.175878882408142,
      "log_odds_ratio": -0.46772462129592896,
      "logits/chosen": -0.931445300579071,
      "logits/rejected": -0.8638671636581421,
      "logps/chosen": -0.707324206829071,
      "logps/rejected": -1.5478515625,
      "loss": 0.9382,
      "nll_loss": 0.87890625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07070312649011612,
      "rewards/margins": 0.08412780612707138,
      "rewards/rejected": -0.15488281846046448,
      "step": 3490
    },
    {
      "epoch": 0.2655941721050235,
      "grad_norm": 2.0922319319691103,
      "learning_rate": 1.3522468075656264e-06,
      "log_odds_chosen": 1.394921898841858,
      "log_odds_ratio": -0.44111329317092896,
      "logits/chosen": -0.9380859136581421,
      "logits/rejected": -0.8345702886581421,
      "logps/chosen": -0.717480480670929,
      "logps/rejected": -1.7511718273162842,
      "loss": 0.9151,
      "nll_loss": 0.8675781488418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07181396335363388,
      "rewards/margins": 0.10313721001148224,
      "rewards/rejected": -0.17502442002296448,
      "step": 3500
    },
    {
      "epoch": 0.26635301259675215,
      "grad_norm": 2.038393716254202,
      "learning_rate": 1.3503191561115553e-06,
      "log_odds_chosen": 1.2841796875,
      "log_odds_ratio": -0.4270996153354645,
      "logits/chosen": -0.954882800579071,
      "logits/rejected": -0.860546886920929,
      "logps/chosen": -0.656054675579071,
      "logps/rejected": -1.5242187976837158,
      "loss": 0.9097,
      "nll_loss": 0.872265636920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06560058891773224,
      "rewards/margins": 0.08687744289636612,
      "rewards/rejected": -0.15261229872703552,
      "step": 3510
    },
    {
      "epoch": 0.2671118530884808,
      "grad_norm": 2.113088406087,
      "learning_rate": 1.348399724926484e-06,
      "log_odds_chosen": 1.34375,
      "log_odds_ratio": -0.3807128965854645,
      "logits/chosen": -0.9605468511581421,
      "logits/rejected": -0.8394531011581421,
      "logps/chosen": -0.6573241949081421,
      "logps/rejected": -1.5763671398162842,
      "loss": 0.938,
      "nll_loss": 0.9312499761581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06573486328125,
      "rewards/margins": 0.09195556491613388,
      "rewards/rejected": -0.15773925185203552,
      "step": 3520
    },
    {
      "epoch": 0.26787069358020943,
      "grad_norm": 2.206269408386097,
      "learning_rate": 1.346488455751882e-06,
      "log_odds_chosen": 1.100976586341858,
      "log_odds_ratio": -0.45136719942092896,
      "logits/chosen": -0.981249988079071,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.708300769329071,
      "logps/rejected": -1.443359375,
      "loss": 0.9309,
      "nll_loss": 0.9097656011581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07087402045726776,
      "rewards/margins": 0.07344970852136612,
      "rewards/rejected": -0.14433594048023224,
      "step": 3530
    },
    {
      "epoch": 0.26862953407193807,
      "grad_norm": 1.9530518034425253,
      "learning_rate": 1.3445852909056286e-06,
      "log_odds_chosen": 1.2160155773162842,
      "log_odds_ratio": -0.40668946504592896,
      "logits/chosen": -0.971875011920929,
      "logits/rejected": -0.860546886920929,
      "logps/chosen": -0.6458984613418579,
      "logps/rejected": -1.461328148841858,
      "loss": 0.9214,
      "nll_loss": 0.887890636920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06459961086511612,
      "rewards/margins": 0.08159179985523224,
      "rewards/rejected": -0.1461181640625,
      "step": 3540
    },
    {
      "epoch": 0.2693883745636667,
      "grad_norm": 1.9384344977682093,
      "learning_rate": 1.3426901732747024e-06,
      "log_odds_chosen": 1.2693359851837158,
      "log_odds_ratio": -0.4195312559604645,
      "logits/chosen": -0.9878906011581421,
      "logits/rejected": -0.8871093988418579,
      "logps/chosen": -0.684863269329071,
      "logps/rejected": -1.568945288658142,
      "loss": 0.9331,
      "nll_loss": 0.9654296636581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06842041015625,
      "rewards/margins": 0.0883331298828125,
      "rewards/rejected": -0.15693359076976776,
      "step": 3550
    },
    {
      "epoch": 0.27014721505539535,
      "grad_norm": 2.1127083794668517,
      "learning_rate": 1.3408030463079818e-06,
      "log_odds_chosen": 1.28515625,
      "log_odds_ratio": -0.4278808534145355,
      "logits/chosen": -0.9398437738418579,
      "logits/rejected": -0.8525390625,
      "logps/chosen": -0.697265625,
      "logps/rejected": -1.605078101158142,
      "loss": 0.9248,
      "nll_loss": 0.8998047113418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06967773288488388,
      "rewards/margins": 0.09077148139476776,
      "rewards/rejected": -0.16044922173023224,
      "step": 3560
    },
    {
      "epoch": 0.270906055547124,
      "grad_norm": 2.0473745290079193,
      "learning_rate": 1.3389238540091568e-06,
      "log_odds_chosen": 1.4441406726837158,
      "log_odds_ratio": -0.4049316346645355,
      "logits/chosen": -1.032812476158142,
      "logits/rejected": -0.912109375,
      "logps/chosen": -0.709179699420929,
      "logps/rejected": -1.760156273841858,
      "loss": 0.9217,
      "nll_loss": 0.885546863079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07088623195886612,
      "rewards/margins": 0.10515137016773224,
      "rewards/rejected": -0.176025390625,
      "step": 3570
    },
    {
      "epoch": 0.27166489603885263,
      "grad_norm": 2.0116456378081438,
      "learning_rate": 1.337052540929751e-06,
      "log_odds_chosen": 1.521875023841858,
      "log_odds_ratio": -0.3868164122104645,
      "logits/chosen": -0.9634765386581421,
      "logits/rejected": -0.8515625,
      "logps/chosen": -0.721484363079071,
      "logps/rejected": -1.8093750476837158,
      "loss": 0.914,
      "nll_loss": 0.939257800579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0721435546875,
      "rewards/margins": 0.10887451469898224,
      "rewards/rejected": -0.18093261122703552,
      "step": 3580
    },
    {
      "epoch": 0.2724237365305813,
      "grad_norm": 2.0250387953551834,
      "learning_rate": 1.33518905216225e-06,
      "log_odds_chosen": 1.1780273914337158,
      "log_odds_ratio": -0.4666503965854645,
      "logits/chosen": -0.9242187738418579,
      "logits/rejected": -0.8359375,
      "logps/chosen": -0.71484375,
      "logps/rejected": -1.535546898841858,
      "loss": 0.9198,
      "nll_loss": 0.9498046636581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07145996391773224,
      "rewards/margins": 0.0820465087890625,
      "rewards/rejected": -0.15351562201976776,
      "step": 3590
    },
    {
      "epoch": 0.2731825770223099,
      "grad_norm": 1.8833860642550362,
      "learning_rate": 1.3333333333333332e-06,
      "log_odds_chosen": 1.3043944835662842,
      "log_odds_ratio": -0.4422851502895355,
      "logits/chosen": -0.8955078125,
      "logits/rejected": -0.804492175579071,
      "logps/chosen": -0.66796875,
      "logps/rejected": -1.592382788658142,
      "loss": 0.9095,
      "nll_loss": 0.8248046636581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06680908054113388,
      "rewards/margins": 0.09229431301355362,
      "rewards/rejected": -0.15915527939796448,
      "step": 3600
    },
    {
      "epoch": 0.27394141751403855,
      "grad_norm": 1.860998223645059,
      "learning_rate": 1.3314853305972122e-06,
      "log_odds_chosen": 1.147851586341858,
      "log_odds_ratio": -0.4798339903354645,
      "logits/chosen": -1.007226586341858,
      "logits/rejected": -0.916015625,
      "logps/chosen": -0.6913086175918579,
      "logps/rejected": -1.504296898841858,
      "loss": 0.9262,
      "nll_loss": 0.9146484136581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06912841647863388,
      "rewards/margins": 0.08133240044116974,
      "rewards/rejected": -0.15043945610523224,
      "step": 3610
    },
    {
      "epoch": 0.2747002580057672,
      "grad_norm": 3.411752689796777,
      "learning_rate": 1.3296449906290671e-06,
      "log_odds_chosen": 1.2785155773162842,
      "log_odds_ratio": -0.427490234375,
      "logits/chosen": -0.944531261920929,
      "logits/rejected": -0.8177734613418579,
      "logps/chosen": -0.6968749761581421,
      "logps/rejected": -1.5828125476837158,
      "loss": 0.9241,
      "nll_loss": 0.933789074420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06971435248851776,
      "rewards/margins": 0.08861084282398224,
      "rewards/rejected": -0.15817871689796448,
      "step": 3620
    },
    {
      "epoch": 0.27545909849749584,
      "grad_norm": 1.7878113783437786,
      "learning_rate": 1.3278122606185844e-06,
      "log_odds_chosen": 1.1787598133087158,
      "log_odds_ratio": -0.4697265625,
      "logits/chosen": -0.930859386920929,
      "logits/rejected": -0.851367175579071,
      "logps/chosen": -0.6568359136581421,
      "logps/rejected": -1.4609375,
      "loss": 0.9242,
      "nll_loss": 0.8294922113418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.065673828125,
      "rewards/margins": 0.080413818359375,
      "rewards/rejected": -0.1461181640625,
      "step": 3630
    },
    {
      "epoch": 0.2762179389892245,
      "grad_norm": 2.04196840241614,
      "learning_rate": 1.3259870882635918e-06,
      "log_odds_chosen": 1.085351586341858,
      "log_odds_ratio": -0.43281251192092896,
      "logits/chosen": -0.958203136920929,
      "logits/rejected": -0.855664074420929,
      "logps/chosen": -0.705859363079071,
      "logps/rejected": -1.427343726158142,
      "loss": 0.9117,
      "nll_loss": 0.894726574420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07058105617761612,
      "rewards/margins": 0.07220458984375,
      "rewards/rejected": -0.14274902641773224,
      "step": 3640
    },
    {
      "epoch": 0.2769767794809531,
      "grad_norm": 2.0541773707451956,
      "learning_rate": 1.3241694217637886e-06,
      "log_odds_chosen": 1.3690917491912842,
      "log_odds_ratio": -0.41411131620407104,
      "logits/chosen": -0.9505859613418579,
      "logits/rejected": -0.845898449420929,
      "logps/chosen": -0.6644531488418579,
      "logps/rejected": -1.6238281726837158,
      "loss": 0.8978,
      "nll_loss": 0.8138672113418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06641845405101776,
      "rewards/margins": 0.09598693996667862,
      "rewards/rejected": -0.16249999403953552,
      "step": 3650
    },
    {
      "epoch": 0.27773561997268176,
      "grad_norm": 1.85220670548097,
      "learning_rate": 1.3223592098145723e-06,
      "log_odds_chosen": 1.144677758216858,
      "log_odds_ratio": -0.46337890625,
      "logits/chosen": -0.990039050579071,
      "logits/rejected": -0.8544921875,
      "logps/chosen": -0.693359375,
      "logps/rejected": -1.516210913658142,
      "loss": 0.9182,
      "nll_loss": 0.889453113079071,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0693359375,
      "rewards/margins": 0.08226318657398224,
      "rewards/rejected": -0.15158692002296448,
      "step": 3660
    },
    {
      "epoch": 0.2784944604644104,
      "grad_norm": 2.0232170372823512,
      "learning_rate": 1.3205564016009555e-06,
      "log_odds_chosen": 1.1967284679412842,
      "log_odds_ratio": -0.44624024629592896,
      "logits/chosen": -0.9720703363418579,
      "logits/rejected": -0.866015613079071,
      "logps/chosen": -0.638671875,
      "logps/rejected": -1.4365234375,
      "loss": 0.9047,
      "nll_loss": 0.837695300579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0638427734375,
      "rewards/margins": 0.07980956882238388,
      "rewards/rejected": -0.14372558891773224,
      "step": 3670
    },
    {
      "epoch": 0.27925330095613904,
      "grad_norm": 2.0496707214558834,
      "learning_rate": 1.318760946791574e-06,
      "log_odds_chosen": 1.037841796875,
      "log_odds_ratio": -0.4786132872104645,
      "logits/chosen": -0.9595702886581421,
      "logits/rejected": -0.8451172113418579,
      "logps/chosen": -0.758007824420929,
      "logps/rejected": -1.464453101158142,
      "loss": 0.9316,
      "nll_loss": 0.9769531488418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07581786811351776,
      "rewards/margins": 0.07055969536304474,
      "rewards/rejected": -0.1463623046875,
      "step": 3680
    },
    {
      "epoch": 0.2800121414478677,
      "grad_norm": 1.9833909441957962,
      "learning_rate": 1.316972795532786e-06,
      "log_odds_chosen": 1.102392554283142,
      "log_odds_ratio": -0.44877928495407104,
      "logits/chosen": -0.970898449420929,
      "logits/rejected": -0.8890625238418579,
      "logps/chosen": -0.6924804449081421,
      "logps/rejected": -1.426367163658142,
      "loss": 0.9288,
      "nll_loss": 0.881640613079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06929931789636612,
      "rewards/margins": 0.07329712063074112,
      "rewards/rejected": -0.14252929389476776,
      "step": 3690
    },
    {
      "epoch": 0.2807709819395963,
      "grad_norm": 1.8332235239722916,
      "learning_rate": 1.3151918984428582e-06,
      "log_odds_chosen": 1.079687476158142,
      "log_odds_ratio": -0.474365234375,
      "logits/chosen": -0.947460949420929,
      "logits/rejected": -0.8470703363418579,
      "logps/chosen": -0.6900390386581421,
      "logps/rejected": -1.4333984851837158,
      "loss": 0.9166,
      "nll_loss": 0.875781238079071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06898193061351776,
      "rewards/margins": 0.07448425143957138,
      "rewards/rejected": -0.14338378608226776,
      "step": 3700
    },
    {
      "epoch": 0.28152982243132496,
      "grad_norm": 2.1768807174108207,
      "learning_rate": 1.313418206606237e-06,
      "log_odds_chosen": 1.1287109851837158,
      "log_odds_ratio": -0.4718261659145355,
      "logits/chosen": -0.9134765863418579,
      "logits/rejected": -0.8324218988418579,
      "logps/chosen": -0.703320324420929,
      "logps/rejected": -1.4970703125,
      "loss": 0.9236,
      "nll_loss": 0.8587890863418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07032470405101776,
      "rewards/margins": 0.07926025241613388,
      "rewards/rejected": -0.14970703423023224,
      "step": 3710
    },
    {
      "epoch": 0.2822886629230536,
      "grad_norm": 1.69205488461686,
      "learning_rate": 1.3116516715679057e-06,
      "log_odds_chosen": 1.261816382408142,
      "log_odds_ratio": -0.42744141817092896,
      "logits/chosen": -0.9498046636581421,
      "logits/rejected": -0.8570312261581421,
      "logps/chosen": -0.66796875,
      "logps/rejected": -1.5369141101837158,
      "loss": 0.8861,
      "nll_loss": 0.8794921636581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06682129204273224,
      "rewards/margins": 0.086822509765625,
      "rewards/rejected": -0.15363769233226776,
      "step": 3720
    },
    {
      "epoch": 0.2830475034147822,
      "grad_norm": 2.172458859919438,
      "learning_rate": 1.3098922453278258e-06,
      "log_odds_chosen": 1.377343773841858,
      "log_odds_ratio": -0.38911134004592896,
      "logits/chosen": -0.9644531011581421,
      "logits/rejected": -0.8539062738418579,
      "logps/chosen": -0.676074206829071,
      "logps/rejected": -1.6179687976837158,
      "loss": 0.954,
      "nll_loss": 0.8990234136581421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06757812201976776,
      "rewards/margins": 0.09407959133386612,
      "rewards/rejected": -0.16171875596046448,
      "step": 3730
    },
    {
      "epoch": 0.2838063439065108,
      "grad_norm": 1.993112848217689,
      "learning_rate": 1.3081398803354573e-06,
      "log_odds_chosen": 1.075585961341858,
      "log_odds_ratio": -0.4784179627895355,
      "logits/chosen": -0.939257800579071,
      "logits/rejected": -0.861328125,
      "logps/chosen": -0.7232421636581421,
      "logps/rejected": -1.469140648841858,
      "loss": 0.914,
      "nll_loss": 0.815234363079071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07233886420726776,
      "rewards/margins": 0.074615478515625,
      "rewards/rejected": -0.14699706435203552,
      "step": 3740
    },
    {
      "epoch": 0.28456518439823947,
      "grad_norm": 2.044150243486001,
      "learning_rate": 1.3063945294843617e-06,
      "log_odds_chosen": 1.310449242591858,
      "log_odds_ratio": -0.39531248807907104,
      "logits/chosen": -0.927929699420929,
      "logits/rejected": -0.8052734136581421,
      "logps/chosen": -0.637402355670929,
      "logps/rejected": -1.5,
      "loss": 0.9004,
      "nll_loss": 0.8472656011581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06375732272863388,
      "rewards/margins": 0.08640136569738388,
      "rewards/rejected": -0.15000000596046448,
      "step": 3750
    },
    {
      "epoch": 0.2853240248899681,
      "grad_norm": 2.1374911680271174,
      "learning_rate": 1.3046561461068843e-06,
      "log_odds_chosen": 1.138281226158142,
      "log_odds_ratio": -0.43183594942092896,
      "logits/chosen": -0.947070300579071,
      "logits/rejected": -0.845507800579071,
      "logps/chosen": -0.651171863079071,
      "logps/rejected": -1.3779296875,
      "loss": 0.907,
      "nll_loss": 0.8470703363418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06514892727136612,
      "rewards/margins": 0.07267455756664276,
      "rewards/rejected": -0.13779297471046448,
      "step": 3760
    },
    {
      "epoch": 0.28608286538169675,
      "grad_norm": 2.1722206297390354,
      "learning_rate": 1.3029246839689124e-06,
      "log_odds_chosen": 1.3113524913787842,
      "log_odds_ratio": -0.43989259004592896,
      "logits/chosen": -0.980664074420929,
      "logits/rejected": -0.8580077886581421,
      "logps/chosen": -0.712695300579071,
      "logps/rejected": -1.656835913658142,
      "loss": 0.8966,
      "nll_loss": 0.8304687738418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07132568210363388,
      "rewards/margins": 0.09425659477710724,
      "rewards/rejected": -0.16567382216453552,
      "step": 3770
    },
    {
      "epoch": 0.2868417058734254,
      "grad_norm": 2.1742138914896745,
      "learning_rate": 1.3012000972647109e-06,
      "log_odds_chosen": 1.387109398841858,
      "log_odds_ratio": -0.42460936307907104,
      "logits/chosen": -0.951367199420929,
      "logits/rejected": -0.8472656011581421,
      "logps/chosen": -0.6595703363418579,
      "logps/rejected": -1.5859375,
      "loss": 0.8949,
      "nll_loss": 0.8529297113418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06594238430261612,
      "rewards/margins": 0.09256591647863388,
      "rewards/rejected": -0.15861816704273224,
      "step": 3780
    },
    {
      "epoch": 0.28760054636515403,
      "grad_norm": 2.155127876083863,
      "learning_rate": 1.299482340611832e-06,
      "log_odds_chosen": 1.07763671875,
      "log_odds_ratio": -0.45654296875,
      "logits/chosen": -0.9107421636581421,
      "logits/rejected": -0.841796875,
      "logps/chosen": -0.62109375,
      "logps/rejected": -1.2839844226837158,
      "loss": 0.9269,
      "nll_loss": 0.898632824420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06208496168255806,
      "rewards/margins": 0.0662841796875,
      "rewards/rejected": -0.12841796875,
      "step": 3790
    },
    {
      "epoch": 0.28835938685688267,
      "grad_norm": 1.9850754100896,
      "learning_rate": 1.2977713690461003e-06,
      "log_odds_chosen": 0.9205077886581421,
      "log_odds_ratio": -0.5121093988418579,
      "logits/chosen": -0.9189453125,
      "logits/rejected": -0.8374999761581421,
      "logps/chosen": -0.702343761920929,
      "logps/rejected": -1.354101538658142,
      "loss": 0.9102,
      "nll_loss": 0.8666015863418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07027588039636612,
      "rewards/margins": 0.06508179008960724,
      "rewards/rejected": -0.13547363877296448,
      "step": 3800
    },
    {
      "epoch": 0.2891182273486113,
      "grad_norm": 3.6873578144144328,
      "learning_rate": 1.296067138016669e-06,
      "log_odds_chosen": 1.4038574695587158,
      "log_odds_ratio": -0.42290037870407104,
      "logits/chosen": -0.9017578363418579,
      "logits/rejected": -0.8236328363418579,
      "logps/chosen": -0.65576171875,
      "logps/rejected": -1.642968773841858,
      "loss": 0.9158,
      "nll_loss": 0.9505859613418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06558837741613388,
      "rewards/margins": 0.0986480712890625,
      "rewards/rejected": -0.16423340141773224,
      "step": 3810
    },
    {
      "epoch": 0.28987706784033995,
      "grad_norm": 1.7797642030913543,
      "learning_rate": 1.294369603381147e-06,
      "log_odds_chosen": 1.154687523841858,
      "log_odds_ratio": -0.4600585997104645,
      "logits/chosen": -0.940234363079071,
      "logits/rejected": -0.8539062738418579,
      "logps/chosen": -0.6429687738418579,
      "logps/rejected": -1.458398461341858,
      "loss": 0.9202,
      "nll_loss": 0.8675781488418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06423339992761612,
      "rewards/margins": 0.08149413764476776,
      "rewards/rejected": -0.14577636122703552,
      "step": 3820
    },
    {
      "epoch": 0.2906359083320686,
      "grad_norm": 2.069119773208419,
      "learning_rate": 1.2926787214007981e-06,
      "log_odds_chosen": 1.3681640625,
      "log_odds_ratio": -0.40424805879592896,
      "logits/chosen": -0.929492175579071,
      "logits/rejected": -0.828906238079071,
      "logps/chosen": -0.6714843511581421,
      "logps/rejected": -1.6003906726837158,
      "loss": 0.9061,
      "nll_loss": 0.9058593511581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06715087592601776,
      "rewards/margins": 0.092864990234375,
      "rewards/rejected": -0.16010741889476776,
      "step": 3830
    },
    {
      "epoch": 0.29139474882379723,
      "grad_norm": 2.1430312635959963,
      "learning_rate": 1.2909944487358056e-06,
      "log_odds_chosen": 1.35888671875,
      "log_odds_ratio": -0.42060548067092896,
      "logits/chosen": -0.9111328125,
      "logits/rejected": -0.826367199420929,
      "logps/chosen": -0.666699230670929,
      "logps/rejected": -1.606835961341858,
      "loss": 0.9176,
      "nll_loss": 0.8921874761581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06668701022863388,
      "rewards/margins": 0.09390868991613388,
      "rewards/rejected": -0.16062012314796448,
      "step": 3840
    },
    {
      "epoch": 0.2921535893155259,
      "grad_norm": 1.8334780666623056,
      "learning_rate": 1.2893167424406084e-06,
      "log_odds_chosen": 1.49951171875,
      "log_odds_ratio": -0.3844238221645355,
      "logits/chosen": -0.9283202886581421,
      "logits/rejected": -0.83203125,
      "logps/chosen": -0.6283203363418579,
      "logps/rejected": -1.661523461341858,
      "loss": 0.9123,
      "nll_loss": 0.8617187738418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06282959133386612,
      "rewards/margins": 0.10329589992761612,
      "rewards/rejected": -0.1661376953125,
      "step": 3850
    },
    {
      "epoch": 0.2929124298072545,
      "grad_norm": 3.226465128998707,
      "learning_rate": 1.2876455599593008e-06,
      "log_odds_chosen": 1.295019507408142,
      "log_odds_ratio": -0.4169921875,
      "logits/chosen": -0.9349609613418579,
      "logits/rejected": -0.8218749761581421,
      "logps/chosen": -0.687695324420929,
      "logps/rejected": -1.5730469226837158,
      "loss": 0.9304,
      "nll_loss": 0.884960949420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06887207180261612,
      "rewards/margins": 0.08841247856616974,
      "rewards/rejected": -0.15725097060203552,
      "step": 3860
    },
    {
      "epoch": 0.29367127029898316,
      "grad_norm": 1.8748341791580312,
      "learning_rate": 1.285980859121099e-06,
      "log_odds_chosen": 1.3193359375,
      "log_odds_ratio": -0.3985839784145355,
      "logits/chosen": -0.947460949420929,
      "logits/rejected": -0.849414050579071,
      "logps/chosen": -0.6934570074081421,
      "logps/rejected": -1.58984375,
      "loss": 0.8835,
      "nll_loss": 0.816210925579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06939697265625,
      "rewards/margins": 0.089569091796875,
      "rewards/rejected": -0.15903320908546448,
      "step": 3870
    },
    {
      "epoch": 0.2944301107907118,
      "grad_norm": 2.0849897268039674,
      "learning_rate": 1.2843225981358712e-06,
      "log_odds_chosen": 1.206884741783142,
      "log_odds_ratio": -0.4189453125,
      "logits/chosen": -0.894726574420929,
      "logits/rejected": -0.818359375,
      "logps/chosen": -0.6875,
      "logps/rejected": -1.5177733898162842,
      "loss": 0.8965,
      "nll_loss": 0.895703136920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06877441704273224,
      "rewards/margins": 0.08306427299976349,
      "rewards/rejected": -0.15168456733226776,
      "step": 3880
    },
    {
      "epoch": 0.29518895128244044,
      "grad_norm": 2.096660042103269,
      "learning_rate": 1.2826707355897317e-06,
      "log_odds_chosen": 1.1281249523162842,
      "log_odds_ratio": -0.4625488221645355,
      "logits/chosen": -0.9380859136581421,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.712890625,
      "logps/rejected": -1.5154297351837158,
      "loss": 0.9078,
      "nll_loss": 0.9111328125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0712890625,
      "rewards/margins": 0.08025512844324112,
      "rewards/rejected": -0.15156249701976776,
      "step": 3890
    },
    {
      "epoch": 0.2959477917741691,
      "grad_norm": 2.037013139295946,
      "learning_rate": 1.281025230440697e-06,
      "log_odds_chosen": 1.2263672351837158,
      "log_odds_ratio": -0.4684081971645355,
      "logits/chosen": -0.949023425579071,
      "logits/rejected": -0.8646484613418579,
      "logps/chosen": -0.738964855670929,
      "logps/rejected": -1.6062500476837158,
      "loss": 0.9104,
      "nll_loss": 0.9224609136581421,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07393798977136612,
      "rewards/margins": 0.086822509765625,
      "rewards/rejected": -0.16074219346046448,
      "step": 3900
    },
    {
      "epoch": 0.2967066322658977,
      "grad_norm": 2.027263988966167,
      "learning_rate": 1.2793860420144025e-06,
      "log_odds_chosen": 1.141992211341858,
      "log_odds_ratio": -0.46015626192092896,
      "logits/chosen": -0.947070300579071,
      "logits/rejected": -0.85546875,
      "logps/chosen": -0.693359375,
      "logps/rejected": -1.4386718273162842,
      "loss": 0.9024,
      "nll_loss": 0.832226574420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06937255710363388,
      "rewards/margins": 0.07454834133386612,
      "rewards/rejected": -0.14394530653953552,
      "step": 3910
    },
    {
      "epoch": 0.29746547275762636,
      "grad_norm": 2.401330558412098,
      "learning_rate": 1.2777531299998798e-06,
      "log_odds_chosen": 1.37109375,
      "log_odds_ratio": -0.405517578125,
      "logits/chosen": -1.0207030773162842,
      "logits/rejected": -0.8681640625,
      "logps/chosen": -0.656542956829071,
      "logps/rejected": -1.558203101158142,
      "loss": 0.9375,
      "nll_loss": 0.9273437261581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.065673828125,
      "rewards/margins": 0.09022827446460724,
      "rewards/rejected": -0.15598145127296448,
      "step": 3920
    },
    {
      "epoch": 0.298224313249355,
      "grad_norm": 2.0230385894292295,
      "learning_rate": 1.2761264544453928e-06,
      "log_odds_chosen": 1.116113305091858,
      "log_odds_ratio": -0.4505371153354645,
      "logits/chosen": -0.9798828363418579,
      "logits/rejected": -0.9058593511581421,
      "logps/chosen": -0.734570324420929,
      "logps/rejected": -1.5070312023162842,
      "loss": 0.9004,
      "nll_loss": 0.866992175579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07340087741613388,
      "rewards/margins": 0.07725830376148224,
      "rewards/rejected": -0.15078124403953552,
      "step": 3930
    },
    {
      "epoch": 0.29898315374108364,
      "grad_norm": 2.4045696372396175,
      "learning_rate": 1.2745059757543324e-06,
      "log_odds_chosen": 1.170654296875,
      "log_odds_ratio": -0.45654296875,
      "logits/chosen": -0.9185546636581421,
      "logits/rejected": -0.806640625,
      "logps/chosen": -0.7037109136581421,
      "logps/rejected": -1.4855468273162842,
      "loss": 0.8907,
      "nll_loss": 0.8763672113418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07038573920726776,
      "rewards/margins": 0.07807006686925888,
      "rewards/rejected": -0.1485595703125,
      "step": 3940
    },
    {
      "epoch": 0.2997419942328123,
      "grad_norm": 2.38418116530518,
      "learning_rate": 1.272891654681168e-06,
      "log_odds_chosen": 1.058984398841858,
      "log_odds_ratio": -0.475830078125,
      "logits/chosen": -0.8978515863418579,
      "logits/rejected": -0.828320324420929,
      "logps/chosen": -0.6246093511581421,
      "logps/rejected": -1.282617211341858,
      "loss": 0.9058,
      "nll_loss": 0.878125011920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06243896484375,
      "rewards/margins": 0.06582947075366974,
      "rewards/rejected": -0.12822265923023224,
      "step": 3950
    },
    {
      "epoch": 0.3005008347245409,
      "grad_norm": 2.2719917645534053,
      "learning_rate": 1.2712834523274563e-06,
      "log_odds_chosen": 1.1682617664337158,
      "log_odds_ratio": -0.43828123807907104,
      "logits/chosen": -0.954296886920929,
      "logits/rejected": -0.8714843988418579,
      "logps/chosen": -0.62060546875,
      "logps/rejected": -1.387109398841858,
      "loss": 0.9131,
      "nll_loss": 0.8638671636581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06207275390625,
      "rewards/margins": 0.07665405422449112,
      "rewards/rejected": -0.138671875,
      "step": 3960
    },
    {
      "epoch": 0.30125967521626956,
      "grad_norm": 2.6488593849334277,
      "learning_rate": 1.2696813301379032e-06,
      "log_odds_chosen": 1.3332030773162842,
      "log_odds_ratio": -0.4345703125,
      "logits/chosen": -0.9886718988418579,
      "logits/rejected": -0.884960949420929,
      "logps/chosen": -0.663867175579071,
      "logps/rejected": -1.55859375,
      "loss": 0.9141,
      "nll_loss": 0.8414062261581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06636963039636612,
      "rewards/margins": 0.08956298977136612,
      "rewards/rejected": -0.15590819716453552,
      "step": 3970
    },
    {
      "epoch": 0.3020185157079982,
      "grad_norm": 2.10235152985044,
      "learning_rate": 1.2680852498964829e-06,
      "log_odds_chosen": 1.461523413658142,
      "log_odds_ratio": -0.3689941465854645,
      "logits/chosen": -0.960156261920929,
      "logits/rejected": -0.847460925579071,
      "logps/chosen": -0.628710925579071,
      "logps/rejected": -1.6316406726837158,
      "loss": 0.9188,
      "nll_loss": 0.794921875,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06287841498851776,
      "rewards/margins": 0.10035400092601776,
      "rewards/rejected": -0.16318359971046448,
      "step": 3980
    },
    {
      "epoch": 0.30277735619972684,
      "grad_norm": 2.270931300000564,
      "learning_rate": 1.266495173722607e-06,
      "log_odds_chosen": 1.0371582508087158,
      "log_odds_ratio": -0.4625000059604645,
      "logits/chosen": -0.907031238079071,
      "logits/rejected": -0.8451172113418579,
      "logps/chosen": -0.685351550579071,
      "logps/rejected": -1.3771483898162842,
      "loss": 0.9084,
      "nll_loss": 0.817187488079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06854248046875,
      "rewards/margins": 0.06908264011144638,
      "rewards/rejected": -0.13762207329273224,
      "step": 3990
    },
    {
      "epoch": 0.30353619669145543,
      "grad_norm": 2.1861920101261725,
      "learning_rate": 1.2649110640673517e-06,
      "log_odds_chosen": 1.5475585460662842,
      "log_odds_ratio": -0.3888183534145355,
      "logits/chosen": -0.92578125,
      "logits/rejected": -0.838085949420929,
      "logps/chosen": -0.691113293170929,
      "logps/rejected": -1.7919921875,
      "loss": 0.9114,
      "nll_loss": 0.8662109375,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06910400092601776,
      "rewards/margins": 0.11008910834789276,
      "rewards/rejected": -0.17917481064796448,
      "step": 4000
    },
    {
      "epoch": 0.30429503718318407,
      "grad_norm": 1.8779121675796782,
      "learning_rate": 1.2633328837097308e-06,
      "log_odds_chosen": 1.2890625,
      "log_odds_ratio": -0.422119140625,
      "logits/chosen": -0.9326171875,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.65625,
      "logps/rejected": -1.5244140625,
      "loss": 0.903,
      "nll_loss": 0.890820324420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06556396186351776,
      "rewards/margins": 0.08682861179113388,
      "rewards/rejected": -0.1524658203125,
      "step": 4010
    },
    {
      "epoch": 0.3050538776749127,
      "grad_norm": 2.01813428939538,
      "learning_rate": 1.2617605957530233e-06,
      "log_odds_chosen": 1.365332007408142,
      "log_odds_ratio": -0.3958496153354645,
      "logits/chosen": -0.9375,
      "logits/rejected": -0.833984375,
      "logps/chosen": -0.6661132574081421,
      "logps/rejected": -1.6046874523162842,
      "loss": 0.9001,
      "nll_loss": 0.8521484136581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06668701022863388,
      "rewards/margins": 0.09389648586511612,
      "rewards/rejected": -0.16057129204273224,
      "step": 4020
    },
    {
      "epoch": 0.30581271816664135,
      "grad_norm": 2.4756880333260267,
      "learning_rate": 1.2601941636211516e-06,
      "log_odds_chosen": 1.533593773841858,
      "log_odds_ratio": -0.35151368379592896,
      "logits/chosen": -0.97265625,
      "logits/rejected": -0.8453124761581421,
      "logps/chosen": -0.6441406011581421,
      "logps/rejected": -1.714453101158142,
      "loss": 0.8684,
      "nll_loss": 0.843945324420929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06441650539636612,
      "rewards/margins": 0.10716553032398224,
      "rewards/rejected": -0.17145995795726776,
      "step": 4030
    },
    {
      "epoch": 0.30657155865837,
      "grad_norm": 2.1960269276331688,
      "learning_rate": 1.2586335510551052e-06,
      "log_odds_chosen": 1.3595702648162842,
      "log_odds_ratio": -0.40283203125,
      "logits/chosen": -0.9234374761581421,
      "logits/rejected": -0.826367199420929,
      "logps/chosen": -0.689746081829071,
      "logps/rejected": -1.6552734375,
      "loss": 0.9035,
      "nll_loss": 0.8949218988418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06898193061351776,
      "rewards/margins": 0.09656982123851776,
      "rewards/rejected": -0.16552734375,
      "step": 4040
    },
    {
      "epoch": 0.30733039915009863,
      "grad_norm": 4.778497260855035,
      "learning_rate": 1.2570787221094177e-06,
      "log_odds_chosen": 1.20697021484375,
      "log_odds_ratio": -0.40961915254592896,
      "logits/chosen": -0.929882824420929,
      "logits/rejected": -0.8060547113418579,
      "logps/chosen": -0.642871081829071,
      "logps/rejected": -1.4294922351837158,
      "loss": 0.9063,
      "nll_loss": 0.8509765863418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06430663913488388,
      "rewards/margins": 0.07865753024816513,
      "rewards/rejected": -0.14291992783546448,
      "step": 4050
    },
    {
      "epoch": 0.3080892396418273,
      "grad_norm": 2.007858230707382,
      "learning_rate": 1.255529641148689e-06,
      "log_odds_chosen": 1.3185546398162842,
      "log_odds_ratio": -0.406005859375,
      "logits/chosen": -0.9306640625,
      "logits/rejected": -0.8427734375,
      "logps/chosen": -0.6982421875,
      "logps/rejected": -1.6140625476837158,
      "loss": 0.8963,
      "nll_loss": 0.87109375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06978759914636612,
      "rewards/margins": 0.09173583984375,
      "rewards/rejected": -0.16159668564796448,
      "step": 4060
    },
    {
      "epoch": 0.3088480801335559,
      "grad_norm": 2.0422660259092082,
      "learning_rate": 1.2539862728441536e-06,
      "log_odds_chosen": 1.2222168445587158,
      "log_odds_ratio": -0.46635740995407104,
      "logits/chosen": -0.9429687261581421,
      "logits/rejected": -0.8658202886581421,
      "logps/chosen": -0.699999988079071,
      "logps/rejected": -1.578710913658142,
      "loss": 0.9181,
      "nll_loss": 0.825976550579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0699462890625,
      "rewards/margins": 0.0879364013671875,
      "rewards/rejected": -0.157958984375,
      "step": 4070
    },
    {
      "epoch": 0.30960692062528455,
      "grad_norm": 2.2480501927541967,
      "learning_rate": 1.252448582170299e-06,
      "log_odds_chosen": 1.4699218273162842,
      "log_odds_ratio": -0.43339842557907104,
      "logits/chosen": -0.9517577886581421,
      "logits/rejected": -0.873828113079071,
      "logps/chosen": -0.7090820074081421,
      "logps/rejected": -1.7548828125,
      "loss": 0.9077,
      "nll_loss": 0.899609386920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07089843600988388,
      "rewards/margins": 0.10444335639476776,
      "rewards/rejected": -0.17539063096046448,
      "step": 4080
    },
    {
      "epoch": 0.3103657611170132,
      "grad_norm": 2.2268249219221,
      "learning_rate": 1.2509165344015243e-06,
      "log_odds_chosen": 1.2501952648162842,
      "log_odds_ratio": -0.46025389432907104,
      "logits/chosen": -0.967968761920929,
      "logits/rejected": -0.870312511920929,
      "logps/chosen": -0.673632800579071,
      "logps/rejected": -1.5476562976837158,
      "loss": 0.9388,
      "nll_loss": 0.914843738079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06741943210363388,
      "rewards/margins": 0.08745117485523224,
      "rewards/rejected": -0.15473632514476776,
      "step": 4090
    },
    {
      "epoch": 0.31112460160874184,
      "grad_norm": 2.27005999070546,
      "learning_rate": 1.2493900951088486e-06,
      "log_odds_chosen": 1.28955078125,
      "log_odds_ratio": -0.42329102754592896,
      "logits/chosen": -0.951171875,
      "logits/rejected": -0.8755859136581421,
      "logps/chosen": -0.691601574420929,
      "logps/rejected": -1.557031273841858,
      "loss": 0.9157,
      "nll_loss": 0.8306640386581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06904296576976776,
      "rewards/margins": 0.0865631103515625,
      "rewards/rejected": -0.1556396484375,
      "step": 4100
    },
    {
      "epoch": 0.3118834421004705,
      "grad_norm": 2.1754099073977238,
      "learning_rate": 1.2478692301566601e-06,
      "log_odds_chosen": 1.5134766101837158,
      "log_odds_ratio": -0.4137206971645355,
      "logits/chosen": -0.948046863079071,
      "logits/rejected": -0.8490234613418579,
      "logps/chosen": -0.667773425579071,
      "logps/rejected": -1.7433593273162842,
      "loss": 0.902,
      "nll_loss": 0.8935546875,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06674804538488388,
      "rewards/margins": 0.10740967094898224,
      "rewards/rejected": -0.17424316704273224,
      "step": 4110
    },
    {
      "epoch": 0.3126422825921991,
      "grad_norm": 1.8991756706793939,
      "learning_rate": 1.2463539056995116e-06,
      "log_odds_chosen": 1.2667968273162842,
      "log_odds_ratio": -0.44287109375,
      "logits/chosen": -0.9554687738418579,
      "logits/rejected": -0.882617175579071,
      "logps/chosen": -0.703125,
      "logps/rejected": -1.630859375,
      "loss": 0.9017,
      "nll_loss": 0.8999999761581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07025146484375,
      "rewards/margins": 0.09304885566234589,
      "rewards/rejected": -0.16325683891773224,
      "step": 4120
    },
    {
      "epoch": 0.31340112308392776,
      "grad_norm": 1.960326000266427,
      "learning_rate": 1.2448440881789541e-06,
      "log_odds_chosen": 1.252832055091858,
      "log_odds_ratio": -0.4205078184604645,
      "logits/chosen": -0.9300781488418579,
      "logits/rejected": -0.866406261920929,
      "logps/chosen": -0.6532226800918579,
      "logps/rejected": -1.4568359851837158,
      "loss": 0.8844,
      "nll_loss": 0.8388671875,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06529541313648224,
      "rewards/margins": 0.08049926906824112,
      "rewards/rejected": -0.14572754502296448,
      "step": 4130
    },
    {
      "epoch": 0.3141599635756564,
      "grad_norm": 1.9958829149266735,
      "learning_rate": 1.2433397443204184e-06,
      "log_odds_chosen": 1.1348145008087158,
      "log_odds_ratio": -0.4706054627895355,
      "logits/chosen": -0.9605468511581421,
      "logits/rejected": -0.8822265863418579,
      "logps/chosen": -0.6656249761581421,
      "logps/rejected": -1.4658203125,
      "loss": 0.9019,
      "nll_loss": 0.8388671875,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06654052436351776,
      "rewards/margins": 0.07998809963464737,
      "rewards/rejected": -0.14653320610523224,
      "step": 4140
    },
    {
      "epoch": 0.31491880406738504,
      "grad_norm": 4.091569889843401,
      "learning_rate": 1.2418408411301324e-06,
      "log_odds_chosen": 1.275976538658142,
      "log_odds_ratio": -0.400634765625,
      "logits/chosen": -0.9775390625,
      "logits/rejected": -0.8160156011581421,
      "logps/chosen": -0.6201171875,
      "logps/rejected": -1.4548828601837158,
      "loss": 0.901,
      "nll_loss": 0.78271484375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06202392652630806,
      "rewards/margins": 0.08343505859375,
      "rewards/rejected": -0.14548340439796448,
      "step": 4150
    },
    {
      "epoch": 0.3156776445591137,
      "grad_norm": 2.2497340997960076,
      "learning_rate": 1.2403473458920844e-06,
      "log_odds_chosen": 1.372656226158142,
      "log_odds_ratio": -0.43525391817092896,
      "logits/chosen": -0.9624999761581421,
      "logits/rejected": -0.8412109613418579,
      "logps/chosen": -0.692187488079071,
      "logps/rejected": -1.6554687023162842,
      "loss": 0.8977,
      "nll_loss": 0.8486328125,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06931152194738388,
      "rewards/margins": 0.09630127251148224,
      "rewards/rejected": -0.16550293564796448,
      "step": 4160
    },
    {
      "epoch": 0.3164364850508423,
      "grad_norm": 2.038350353584351,
      "learning_rate": 1.2388592261650217e-06,
      "log_odds_chosen": 1.199072241783142,
      "log_odds_ratio": -0.4593749940395355,
      "logits/chosen": -0.9271484613418579,
      "logits/rejected": -0.8607422113418579,
      "logps/chosen": -0.6734374761581421,
      "logps/rejected": -1.5021483898162842,
      "loss": 0.8952,
      "nll_loss": 0.833984375,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06735839694738388,
      "rewards/margins": 0.08281860500574112,
      "rewards/rejected": -0.15019531548023224,
      "step": 4170
    },
    {
      "epoch": 0.31719532554257096,
      "grad_norm": 2.1135411907978474,
      "learning_rate": 1.2373764497794918e-06,
      "log_odds_chosen": 1.4931640625,
      "log_odds_ratio": -0.38886719942092896,
      "logits/chosen": -0.953320324420929,
      "logits/rejected": -0.8460937738418579,
      "logps/chosen": -0.63427734375,
      "logps/rejected": -1.657812476158142,
      "loss": 0.8894,
      "nll_loss": 0.837890625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06341552734375,
      "rewards/margins": 0.10235595703125,
      "rewards/rejected": -0.16586914658546448,
      "step": 4180
    },
    {
      "epoch": 0.3179541660342996,
      "grad_norm": 2.0657817064350033,
      "learning_rate": 1.2358989848349217e-06,
      "log_odds_chosen": 1.474707007408142,
      "log_odds_ratio": -0.37358397245407104,
      "logits/chosen": -0.8857421875,
      "logits/rejected": -0.806445300579071,
      "logps/chosen": -0.6400390863418579,
      "logps/rejected": -1.6335937976837158,
      "loss": 0.8925,
      "nll_loss": 0.833789050579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06398925930261612,
      "rewards/margins": 0.09937743842601776,
      "rewards/rejected": -0.16340331733226776,
      "step": 4190
    },
    {
      "epoch": 0.31871300652602824,
      "grad_norm": 2.182532950524458,
      "learning_rate": 1.2344267996967353e-06,
      "log_odds_chosen": 1.465917944908142,
      "log_odds_ratio": -0.3947509825229645,
      "logits/chosen": -0.9203125238418579,
      "logits/rejected": -0.852734386920929,
      "logps/chosen": -0.662402331829071,
      "logps/rejected": -1.6550781726837158,
      "loss": 0.908,
      "nll_loss": 0.9375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06618652492761612,
      "rewards/margins": 0.09916381537914276,
      "rewards/rejected": -0.16538086533546448,
      "step": 4200
    },
    {
      "epoch": 0.3194718470177569,
      "grad_norm": 1.9590389783181057,
      "learning_rate": 1.2329598629935076e-06,
      "log_odds_chosen": 1.366308569908142,
      "log_odds_ratio": -0.4139648377895355,
      "logits/chosen": -0.9091796875,
      "logits/rejected": -0.8013671636581421,
      "logps/chosen": -0.6680663824081421,
      "logps/rejected": -1.6267578601837158,
      "loss": 0.8914,
      "nll_loss": 0.8695312738418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06683349609375,
      "rewards/margins": 0.09581909328699112,
      "rewards/rejected": -0.16259765625,
      "step": 4210
    },
    {
      "epoch": 0.3202306875094855,
      "grad_norm": 2.1182170742214326,
      "learning_rate": 1.2314981436141583e-06,
      "log_odds_chosen": 1.3098633289337158,
      "log_odds_ratio": -0.4046386778354645,
      "logits/chosen": -0.9716796875,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.619335949420929,
      "logps/rejected": -1.520117163658142,
      "loss": 0.8848,
      "nll_loss": 0.7918945550918579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06196289137005806,
      "rewards/margins": 0.09002685546875,
      "rewards/rejected": -0.15190429985523224,
      "step": 4220
    },
    {
      "epoch": 0.32098952800121416,
      "grad_norm": 2.1681970591127184,
      "learning_rate": 1.2300416107051802e-06,
      "log_odds_chosen": 1.2394530773162842,
      "log_odds_ratio": -0.45927733182907104,
      "logits/chosen": -0.9613281488418579,
      "logits/rejected": -0.832812488079071,
      "logps/chosen": -0.72607421875,
      "logps/rejected": -1.5896484851837158,
      "loss": 0.9274,
      "nll_loss": 0.852343738079071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07264403998851776,
      "rewards/margins": 0.08648681640625,
      "rewards/rejected": -0.15913085639476776,
      "step": 4230
    },
    {
      "epoch": 0.3217483684929428,
      "grad_norm": 2.046706232589475,
      "learning_rate": 1.2285902336679024e-06,
      "log_odds_chosen": 1.12548828125,
      "log_odds_ratio": -0.4857421815395355,
      "logits/chosen": -0.963085949420929,
      "logits/rejected": -0.8150390386581421,
      "logps/chosen": -0.7162109613418579,
      "logps/rejected": -1.516210913658142,
      "loss": 0.9045,
      "nll_loss": 0.8978515863418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07159423828125,
      "rewards/margins": 0.08000488579273224,
      "rewards/rejected": -0.151611328125,
      "step": 4240
    },
    {
      "epoch": 0.32250720898467145,
      "grad_norm": 2.291442872750335,
      "learning_rate": 1.2271439821557926e-06,
      "log_odds_chosen": 1.22265625,
      "log_odds_ratio": -0.43427735567092896,
      "logits/chosen": -0.926953136920929,
      "logits/rejected": -0.859179675579071,
      "logps/chosen": -0.6561523675918579,
      "logps/rejected": -1.5001952648162842,
      "loss": 0.901,
      "nll_loss": 0.830859363079071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06550292670726776,
      "rewards/margins": 0.08442382514476776,
      "rewards/rejected": -0.14992675185203552,
      "step": 4250
    },
    {
      "epoch": 0.3232660494764001,
      "grad_norm": 2.3934126129829516,
      "learning_rate": 1.225702826071791e-06,
      "log_odds_chosen": 1.6476562023162842,
      "log_odds_ratio": -0.3468261659145355,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.870898425579071,
      "logps/chosen": -0.622851550579071,
      "logps/rejected": -1.7775390148162842,
      "loss": 0.8859,
      "nll_loss": 0.803417980670929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06228027492761612,
      "rewards/margins": 0.11549071967601776,
      "rewards/rejected": -0.17783203721046448,
      "step": 4260
    },
    {
      "epoch": 0.3240248899681287,
      "grad_norm": 2.2978147982794916,
      "learning_rate": 1.2242667355656797e-06,
      "log_odds_chosen": 1.464257836341858,
      "log_odds_ratio": -0.3810058534145355,
      "logits/chosen": -0.9378906488418579,
      "logits/rejected": -0.811328113079071,
      "logps/chosen": -0.6431640386581421,
      "logps/rejected": -1.6599609851837158,
      "loss": 0.862,
      "nll_loss": 0.8465820550918579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06434325873851776,
      "rewards/margins": 0.1016845703125,
      "rewards/rejected": -0.16606445610523224,
      "step": 4270
    },
    {
      "epoch": 0.3247837304598573,
      "grad_norm": 2.2231775385940775,
      "learning_rate": 1.2228356810314862e-06,
      "log_odds_chosen": 1.059661865234375,
      "log_odds_ratio": -0.49833983182907104,
      "logits/chosen": -0.930859386920929,
      "logits/rejected": -0.8203125,
      "logps/chosen": -0.7120116949081421,
      "logps/rejected": -1.4490234851837158,
      "loss": 0.9366,
      "nll_loss": 0.8724609613418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07120361179113388,
      "rewards/margins": 0.07371368259191513,
      "rewards/rejected": -0.14499512314796448,
      "step": 4280
    },
    {
      "epoch": 0.32554257095158595,
      "grad_norm": 2.2345119533491937,
      "learning_rate": 1.2214096331049186e-06,
      "log_odds_chosen": 1.5973632335662842,
      "log_odds_ratio": -0.401611328125,
      "logits/chosen": -0.9457031488418579,
      "logits/rejected": -0.8681640625,
      "logps/chosen": -0.6317383050918579,
      "logps/rejected": -1.779687523841858,
      "loss": 0.8845,
      "nll_loss": 0.7544921636581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06319580227136612,
      "rewards/margins": 0.11481933295726776,
      "rewards/rejected": -0.17795410752296448,
      "step": 4290
    },
    {
      "epoch": 0.3263014114433146,
      "grad_norm": 2.5710473923093504,
      "learning_rate": 1.2199885626608373e-06,
      "log_odds_chosen": 1.3624999523162842,
      "log_odds_ratio": -0.4041503965854645,
      "logits/chosen": -0.9730468988418579,
      "logits/rejected": -0.8931640386581421,
      "logps/chosen": -0.6630859375,
      "logps/rejected": -1.584375023841858,
      "loss": 0.902,
      "nll_loss": 0.8310546875,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06634521484375,
      "rewards/margins": 0.09211425483226776,
      "rewards/rejected": -0.15842285752296448,
      "step": 4300
    },
    {
      "epoch": 0.32706025193504323,
      "grad_norm": 1.9495590217172698,
      "learning_rate": 1.2185724408107546e-06,
      "log_odds_chosen": 1.344335913658142,
      "log_odds_ratio": -0.4015136659145355,
      "logits/chosen": -0.976367175579071,
      "logits/rejected": -0.8623046875,
      "logps/chosen": -0.651171863079071,
      "logps/rejected": -1.5617187023162842,
      "loss": 0.9226,
      "nll_loss": 0.8587890863418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06514892727136612,
      "rewards/margins": 0.09112548828125,
      "rewards/rejected": -0.15617676079273224,
      "step": 4310
    },
    {
      "epoch": 0.3278190924267719,
      "grad_norm": 2.0775732902118866,
      "learning_rate": 1.2171612389003689e-06,
      "log_odds_chosen": 1.409570336341858,
      "log_odds_ratio": -0.37944334745407104,
      "logits/chosen": -0.9593750238418579,
      "logits/rejected": -0.8255859613418579,
      "logps/chosen": -0.671679675579071,
      "logps/rejected": -1.689453125,
      "loss": 0.8989,
      "nll_loss": 0.8636718988418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06711425632238388,
      "rewards/margins": 0.101898193359375,
      "rewards/rejected": -0.16921386122703552,
      "step": 4320
    },
    {
      "epoch": 0.3285779329185005,
      "grad_norm": 2.1901595897704644,
      "learning_rate": 1.2157549285071297e-06,
      "log_odds_chosen": 1.056640625,
      "log_odds_ratio": -0.494873046875,
      "logits/chosen": -0.951367199420929,
      "logits/rejected": -0.8515625,
      "logps/chosen": -0.708789050579071,
      "logps/rejected": -1.460546851158142,
      "loss": 0.9165,
      "nll_loss": 0.8740234375,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07082519680261612,
      "rewards/margins": 0.07513427734375,
      "rewards/rejected": -0.14604492485523224,
      "step": 4330
    },
    {
      "epoch": 0.32933677341022916,
      "grad_norm": 1.9604302213993103,
      "learning_rate": 1.2143534814378327e-06,
      "log_odds_chosen": 1.382421851158142,
      "log_odds_ratio": -0.41289061307907104,
      "logits/chosen": -0.914843738079071,
      "logits/rejected": -0.8330078125,
      "logps/chosen": -0.682910144329071,
      "logps/rejected": -1.63671875,
      "loss": 0.8791,
      "nll_loss": 0.846386730670929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06829833984375,
      "rewards/margins": 0.095367431640625,
      "rewards/rejected": -0.16352538764476776,
      "step": 4340
    },
    {
      "epoch": 0.3300956139019578,
      "grad_norm": 2.156153319128,
      "learning_rate": 1.2129568697262454e-06,
      "log_odds_chosen": 1.302148461341858,
      "log_odds_ratio": -0.41865235567092896,
      "logits/chosen": -0.9525390863418579,
      "logits/rejected": -0.8365234136581421,
      "logps/chosen": -0.6341797113418579,
      "logps/rejected": -1.4474608898162842,
      "loss": 0.8645,
      "nll_loss": 0.8104492425918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06348876655101776,
      "rewards/margins": 0.08118285983800888,
      "rewards/rejected": -0.1446533203125,
      "step": 4350
    },
    {
      "epoch": 0.33085445439368644,
      "grad_norm": 2.5408462766533515,
      "learning_rate": 1.2115650656307653e-06,
      "log_odds_chosen": 1.381445288658142,
      "log_odds_ratio": -0.3592773377895355,
      "logits/chosen": -0.9085937738418579,
      "logits/rejected": -0.8330078125,
      "logps/chosen": -0.6597656011581421,
      "logps/rejected": -1.552343726158142,
      "loss": 0.8761,
      "nll_loss": 0.7962890863418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06594238430261612,
      "rewards/margins": 0.08934326469898224,
      "rewards/rejected": -0.15520019829273224,
      "step": 4360
    },
    {
      "epoch": 0.3316132948854151,
      "grad_norm": 2.755454510311766,
      "learning_rate": 1.210178041632103e-06,
      "log_odds_chosen": 1.542578101158142,
      "log_odds_ratio": -0.3629394471645355,
      "logits/chosen": -0.909375011920929,
      "logits/rejected": -0.854296863079071,
      "logps/chosen": -0.62451171875,
      "logps/rejected": -1.678125023841858,
      "loss": 0.9076,
      "nll_loss": 0.865039050579071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06241454929113388,
      "rewards/margins": 0.10528564453125,
      "rewards/rejected": -0.16774901747703552,
      "step": 4370
    },
    {
      "epoch": 0.3323721353771437,
      "grad_norm": 2.043167141768478,
      "learning_rate": 1.2087957704309988e-06,
      "log_odds_chosen": 1.2589843273162842,
      "log_odds_ratio": -0.42744141817092896,
      "logits/chosen": -0.9208984375,
      "logits/rejected": -0.8177734613418579,
      "logps/chosen": -0.638476550579071,
      "logps/rejected": -1.4943358898162842,
      "loss": 0.8986,
      "nll_loss": 0.804492175579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0638427734375,
      "rewards/margins": 0.08549194037914276,
      "rewards/rejected": -0.14938965439796448,
      "step": 4380
    },
    {
      "epoch": 0.33313097586887236,
      "grad_norm": 2.174469428614346,
      "learning_rate": 1.2074182249459642e-06,
      "log_odds_chosen": 1.512841820716858,
      "log_odds_ratio": -0.40498048067092896,
      "logits/chosen": -0.9583984613418579,
      "logits/rejected": -0.875781238079071,
      "logps/chosen": -0.666210949420929,
      "logps/rejected": -1.726171851158142,
      "loss": 0.8872,
      "nll_loss": 0.817675769329071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06667480617761612,
      "rewards/margins": 0.10597991943359375,
      "rewards/rejected": -0.17250975966453552,
      "step": 4390
    },
    {
      "epoch": 0.333889816360601,
      "grad_norm": 2.220872324929471,
      "learning_rate": 1.2060453783110545e-06,
      "log_odds_chosen": 1.1480224132537842,
      "log_odds_ratio": -0.46357423067092896,
      "logits/chosen": -0.9693359136581421,
      "logits/rejected": -0.8525390625,
      "logps/chosen": -0.6807616949081421,
      "logps/rejected": -1.4871094226837158,
      "loss": 0.8837,
      "nll_loss": 0.805468738079071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06805419921875,
      "rewards/margins": 0.08069610595703125,
      "rewards/rejected": -0.14877930283546448,
      "step": 4400
    },
    {
      "epoch": 0.33464865685232964,
      "grad_norm": 2.2372440653759513,
      "learning_rate": 1.2046772038736682e-06,
      "log_odds_chosen": 1.3009765148162842,
      "log_odds_ratio": -0.434326171875,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.8521484136581421,
      "logps/chosen": -0.6429687738418579,
      "logps/rejected": -1.5300781726837158,
      "loss": 0.9141,
      "nll_loss": 0.8697265386581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06434325873851776,
      "rewards/margins": 0.08879394829273224,
      "rewards/rejected": -0.15317383408546448,
      "step": 4410
    },
    {
      "epoch": 0.3354074973440583,
      "grad_norm": 2.2973654270322967,
      "learning_rate": 1.2033136751923736e-06,
      "log_odds_chosen": 1.101904273033142,
      "log_odds_ratio": -0.45087891817092896,
      "logits/chosen": -0.919726550579071,
      "logits/rejected": -0.7900390625,
      "logps/chosen": -0.700976550579071,
      "logps/rejected": -1.473242163658142,
      "loss": 0.9005,
      "nll_loss": 0.849414050579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07012939453125,
      "rewards/margins": 0.07722167670726776,
      "rewards/rejected": -0.14738769829273224,
      "step": 4420
    },
    {
      "epoch": 0.3361663378357869,
      "grad_norm": 2.247061985239726,
      "learning_rate": 1.201954766034762e-06,
      "log_odds_chosen": 1.4426758289337158,
      "log_odds_ratio": -0.42011719942092896,
      "logits/chosen": -0.9330078363418579,
      "logits/rejected": -0.845507800579071,
      "logps/chosen": -0.661425769329071,
      "logps/rejected": -1.6591796875,
      "loss": 0.8758,
      "nll_loss": 0.8121093511581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.066162109375,
      "rewards/margins": 0.09980468451976776,
      "rewards/rejected": -0.16584472358226776,
      "step": 4430
    },
    {
      "epoch": 0.33692517832751556,
      "grad_norm": 2.2570856208655163,
      "learning_rate": 1.2006004503753285e-06,
      "log_odds_chosen": 1.5849609375,
      "log_odds_ratio": -0.4055419862270355,
      "logits/chosen": -0.9097656011581421,
      "logits/rejected": -0.8033202886581421,
      "logps/chosen": -0.640917956829071,
      "logps/rejected": -1.806054711341858,
      "loss": 0.903,
      "nll_loss": 0.824999988079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06409911811351776,
      "rewards/margins": 0.11650238186120987,
      "rewards/rejected": -0.1805419921875,
      "step": 4440
    },
    {
      "epoch": 0.3376840188192442,
      "grad_norm": 2.2141600181166363,
      "learning_rate": 1.1992507023933782e-06,
      "log_odds_chosen": 1.526513695716858,
      "log_odds_ratio": -0.3704589903354645,
      "logits/chosen": -0.939648449420929,
      "logits/rejected": -0.822265625,
      "logps/chosen": -0.69140625,
      "logps/rejected": -1.748437523841858,
      "loss": 0.8751,
      "nll_loss": 0.888476550579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06917724758386612,
      "rewards/margins": 0.10583877563476562,
      "rewards/rejected": -0.1751708984375,
      "step": 4450
    },
    {
      "epoch": 0.33844285931097284,
      "grad_norm": 2.0103647610137925,
      "learning_rate": 1.1979054964709597e-06,
      "log_odds_chosen": 1.1826171875,
      "log_odds_ratio": -0.4234375059604645,
      "logits/chosen": -0.941210925579071,
      "logits/rejected": -0.809374988079071,
      "logps/chosen": -0.6451171636581421,
      "logps/rejected": -1.4169921875,
      "loss": 0.8857,
      "nll_loss": 0.794140636920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06455077975988388,
      "rewards/margins": 0.07726440578699112,
      "rewards/rejected": -0.14177246391773224,
      "step": 4460
    },
    {
      "epoch": 0.3392016998027015,
      "grad_norm": 2.344873760987262,
      "learning_rate": 1.1965648071908207e-06,
      "log_odds_chosen": 1.375,
      "log_odds_ratio": -0.39765626192092896,
      "logits/chosen": -0.9437500238418579,
      "logits/rejected": -0.8138672113418579,
      "logps/chosen": -0.626953125,
      "logps/rejected": -1.572265625,
      "loss": 0.8756,
      "nll_loss": 0.788769543170929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06269530951976776,
      "rewards/margins": 0.09456787258386612,
      "rewards/rejected": -0.15725097060203552,
      "step": 4470
    },
    {
      "epoch": 0.3399605402944301,
      "grad_norm": 1.9603069157506832,
      "learning_rate": 1.1952286093343935e-06,
      "log_odds_chosen": 1.318945288658142,
      "log_odds_ratio": -0.42138671875,
      "logits/chosen": -0.9644531011581421,
      "logits/rejected": -0.848437488079071,
      "logps/chosen": -0.660449206829071,
      "logps/rejected": -1.5525391101837158,
      "loss": 0.8975,
      "nll_loss": 0.8497070074081421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06605224311351776,
      "rewards/margins": 0.08906249701976776,
      "rewards/rejected": -0.15517577528953552,
      "step": 4480
    },
    {
      "epoch": 0.34071938078615877,
      "grad_norm": 2.3007253003330135,
      "learning_rate": 1.1938968778798005e-06,
      "log_odds_chosen": 1.3662109375,
      "log_odds_ratio": -0.3960937559604645,
      "logits/chosen": -0.921875,
      "logits/rejected": -0.7919921875,
      "logps/chosen": -0.637890636920929,
      "logps/rejected": -1.585546851158142,
      "loss": 0.8729,
      "nll_loss": 0.8238281011581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06375732272863388,
      "rewards/margins": 0.09487304836511612,
      "rewards/rejected": -0.15864257514476776,
      "step": 4490
    },
    {
      "epoch": 0.3414782212778874,
      "grad_norm": 2.0606137302048273,
      "learning_rate": 1.1925695879998878e-06,
      "log_odds_chosen": 1.36328125,
      "log_odds_ratio": -0.41728514432907104,
      "logits/chosen": -0.9349609613418579,
      "logits/rejected": -0.8619140386581421,
      "logps/chosen": -0.6795898675918579,
      "logps/rejected": -1.6300780773162842,
      "loss": 0.8698,
      "nll_loss": 0.849609375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0679931640625,
      "rewards/margins": 0.09499511867761612,
      "rewards/rejected": -0.16298827528953552,
      "step": 4500
    },
    {
      "epoch": 0.34223706176961605,
      "grad_norm": 2.2176914597724995,
      "learning_rate": 1.1912467150602794e-06,
      "log_odds_chosen": 1.429296851158142,
      "log_odds_ratio": -0.3785644471645355,
      "logits/chosen": -0.946484386920929,
      "logits/rejected": -0.8521484136581421,
      "logps/chosen": -0.6634765863418579,
      "logps/rejected": -1.657812476158142,
      "loss": 0.8836,
      "nll_loss": 0.82763671875,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06641845405101776,
      "rewards/margins": 0.09949950873851776,
      "rewards/rejected": -0.16584472358226776,
      "step": 4510
    },
    {
      "epoch": 0.3429959022613447,
      "grad_norm": 2.588685801085802,
      "learning_rate": 1.189928234617459e-06,
      "log_odds_chosen": 1.237280249595642,
      "log_odds_ratio": -0.4388671815395355,
      "logits/chosen": -0.9232422113418579,
      "logits/rejected": -0.801953136920929,
      "logps/chosen": -0.6878906488418579,
      "logps/rejected": -1.5712890625,
      "loss": 0.8666,
      "nll_loss": 0.8218749761581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06878662109375,
      "rewards/margins": 0.08840103447437286,
      "rewards/rejected": -0.15731200575828552,
      "step": 4520
    },
    {
      "epoch": 0.34375474275307333,
      "grad_norm": 2.320935556141627,
      "learning_rate": 1.1886141224168716e-06,
      "log_odds_chosen": 1.211328148841858,
      "log_odds_ratio": -0.43071287870407104,
      "logits/chosen": -0.8705078363418579,
      "logits/rejected": -0.8150390386581421,
      "logps/chosen": -0.641406238079071,
      "logps/rejected": -1.460546851158142,
      "loss": 0.8651,
      "nll_loss": 0.822070300579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06413574516773224,
      "rewards/margins": 0.08183594048023224,
      "rewards/rejected": -0.14589843153953552,
      "step": 4530
    },
    {
      "epoch": 0.34451358324480197,
      "grad_norm": 2.1049123689733786,
      "learning_rate": 1.1873043543910495e-06,
      "log_odds_chosen": 1.21533203125,
      "log_odds_ratio": -0.42182618379592896,
      "logits/chosen": -0.920117199420929,
      "logits/rejected": -0.8255859613418579,
      "logps/chosen": -0.618945300579071,
      "logps/rejected": -1.412500023841858,
      "loss": 0.8874,
      "nll_loss": 0.83203125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06195068359375,
      "rewards/margins": 0.07939453423023224,
      "rewards/rejected": -0.14145508408546448,
      "step": 4540
    },
    {
      "epoch": 0.34527242373653055,
      "grad_norm": 2.1976254676362554,
      "learning_rate": 1.1859989066577617e-06,
      "log_odds_chosen": 1.4050781726837158,
      "log_odds_ratio": -0.388671875,
      "logits/chosen": -0.910351574420929,
      "logits/rejected": -0.80078125,
      "logps/chosen": -0.6504882574081421,
      "logps/rejected": -1.6105468273162842,
      "loss": 0.8843,
      "nll_loss": 0.800976574420929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06500244140625,
      "rewards/margins": 0.0960693359375,
      "rewards/rejected": -0.16103515028953552,
      "step": 4550
    },
    {
      "epoch": 0.3460312642282592,
      "grad_norm": 2.1822551995963404,
      "learning_rate": 1.1846977555181846e-06,
      "log_odds_chosen": 1.103369116783142,
      "log_odds_ratio": -0.4722656309604645,
      "logits/chosen": -0.8988281488418579,
      "logits/rejected": -0.824999988079071,
      "logps/chosen": -0.7012695074081421,
      "logps/rejected": -1.4494140148162842,
      "loss": 0.8828,
      "nll_loss": 0.890625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07011719048023224,
      "rewards/margins": 0.07481994479894638,
      "rewards/rejected": -0.14504393935203552,
      "step": 4560
    },
    {
      "epoch": 0.34679010471998784,
      "grad_norm": 2.3308594142253143,
      "learning_rate": 1.1834008774550946e-06,
      "log_odds_chosen": 1.40966796875,
      "log_odds_ratio": -0.40717774629592896,
      "logits/chosen": -0.936328113079071,
      "logits/rejected": -0.8021484613418579,
      "logps/chosen": -0.637011706829071,
      "logps/rejected": -1.5871093273162842,
      "loss": 0.8749,
      "nll_loss": 0.81494140625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06374511867761612,
      "rewards/margins": 0.09494628757238388,
      "rewards/rejected": -0.15874023735523224,
      "step": 4570
    },
    {
      "epoch": 0.3475489452117165,
      "grad_norm": 2.4823474237824605,
      "learning_rate": 1.1821082491310835e-06,
      "log_odds_chosen": 1.223535180091858,
      "log_odds_ratio": -0.4647460877895355,
      "logits/chosen": -0.913281261920929,
      "logits/rejected": -0.8218749761581421,
      "logps/chosen": -0.6629883050918579,
      "logps/rejected": -1.484375,
      "loss": 0.858,
      "nll_loss": 0.824414074420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06619872897863388,
      "rewards/margins": 0.08223648369312286,
      "rewards/rejected": -0.14836426079273224,
      "step": 4580
    },
    {
      "epoch": 0.3483077857034451,
      "grad_norm": 2.0314177948784358,
      "learning_rate": 1.1808198473867937e-06,
      "log_odds_chosen": 1.179785132408142,
      "log_odds_ratio": -0.46484375,
      "logits/chosen": -1.0146484375,
      "logits/rejected": -0.906445324420929,
      "logps/chosen": -0.69921875,
      "logps/rejected": -1.537500023841858,
      "loss": 0.8742,
      "nll_loss": 0.8388671875,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06987304985523224,
      "rewards/margins": 0.08375243842601776,
      "rewards/rejected": -0.15375976264476776,
      "step": 4590
    },
    {
      "epoch": 0.34906662619517376,
      "grad_norm": 1.9804242538388541,
      "learning_rate": 1.179535649239177e-06,
      "log_odds_chosen": 1.2795898914337158,
      "log_odds_ratio": -0.4310058653354645,
      "logits/chosen": -0.903515636920929,
      "logits/rejected": -0.843554675579071,
      "logps/chosen": -0.655957043170929,
      "logps/rejected": -1.542578101158142,
      "loss": 0.8787,
      "nll_loss": 0.8548828363418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06556396186351776,
      "rewards/margins": 0.08857116848230362,
      "rewards/rejected": -0.15422363579273224,
      "step": 4600
    },
    {
      "epoch": 0.3498254666869024,
      "grad_norm": 2.1676793117363498,
      "learning_rate": 1.178255631879771e-06,
      "log_odds_chosen": 1.33642578125,
      "log_odds_ratio": -0.406982421875,
      "logits/chosen": -0.940234363079071,
      "logits/rejected": -0.830273449420929,
      "logps/chosen": -0.685351550579071,
      "logps/rejected": -1.626953125,
      "loss": 0.886,
      "nll_loss": 0.8482421636581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06854248046875,
      "rewards/margins": 0.09415893256664276,
      "rewards/rejected": -0.16267089545726776,
      "step": 4610
    },
    {
      "epoch": 0.35058430717863104,
      "grad_norm": 2.0364662326117187,
      "learning_rate": 1.1769797726729992e-06,
      "log_odds_chosen": 0.963574230670929,
      "log_odds_ratio": -0.4800781309604645,
      "logits/chosen": -0.9091796875,
      "logits/rejected": -0.803515613079071,
      "logps/chosen": -0.69482421875,
      "logps/rejected": -1.3136718273162842,
      "loss": 0.8694,
      "nll_loss": 0.900195300579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06943359225988388,
      "rewards/margins": 0.06182251125574112,
      "rewards/rejected": -0.13129882514476776,
      "step": 4620
    },
    {
      "epoch": 0.3513431476703597,
      "grad_norm": 2.286515715845322,
      "learning_rate": 1.1757080491544881e-06,
      "log_odds_chosen": 1.3899657726287842,
      "log_odds_ratio": -0.4193359315395355,
      "logits/chosen": -0.9341796636581421,
      "logits/rejected": -0.8255859613418579,
      "logps/chosen": -0.683789074420929,
      "logps/rejected": -1.6462891101837158,
      "loss": 0.8888,
      "nll_loss": 0.846972644329071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06842041015625,
      "rewards/margins": 0.09616546332836151,
      "rewards/rejected": -0.16464844346046448,
      "step": 4630
    },
    {
      "epoch": 0.3521019881620883,
      "grad_norm": 2.1123173473303924,
      "learning_rate": 1.1744404390294068e-06,
      "log_odds_chosen": 1.2043945789337158,
      "log_odds_ratio": -0.45966798067092896,
      "logits/chosen": -0.9322265386581421,
      "logits/rejected": -0.8619140386581421,
      "logps/chosen": -0.688183605670929,
      "logps/rejected": -1.525390625,
      "loss": 0.853,
      "nll_loss": 0.8505859375,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06875000149011612,
      "rewards/margins": 0.08389892429113388,
      "rewards/rejected": -0.15251465141773224,
      "step": 4640
    },
    {
      "epoch": 0.35286082865381696,
      "grad_norm": 2.1002667873030645,
      "learning_rate": 1.1731769201708264e-06,
      "log_odds_chosen": 1.4666016101837158,
      "log_odds_ratio": -0.45659178495407104,
      "logits/chosen": -0.916210949420929,
      "logits/rejected": -0.8314453363418579,
      "logps/chosen": -0.671875,
      "logps/rejected": -1.7628905773162842,
      "loss": 0.8795,
      "nll_loss": 0.8500000238418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06715087592601776,
      "rewards/margins": 0.10904540866613388,
      "rewards/rejected": -0.17622070014476776,
      "step": 4650
    },
    {
      "epoch": 0.3536196691455456,
      "grad_norm": 2.5257160808205312,
      "learning_rate": 1.1719174706180952e-06,
      "log_odds_chosen": 1.6716797351837158,
      "log_odds_ratio": -0.3566650450229645,
      "logits/chosen": -0.924609363079071,
      "logits/rejected": -0.7876952886581421,
      "logps/chosen": -0.6436523199081421,
      "logps/rejected": -1.8214843273162842,
      "loss": 0.8786,
      "nll_loss": 0.822265625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06437988579273224,
      "rewards/margins": 0.11765136569738388,
      "rewards/rejected": -0.18222656846046448,
      "step": 4660
    },
    {
      "epoch": 0.35437850963727424,
      "grad_norm": 2.1474539080309687,
      "learning_rate": 1.1706620685752386e-06,
      "log_odds_chosen": 1.5402343273162842,
      "log_odds_ratio": -0.36308592557907104,
      "logits/chosen": -0.9468749761581421,
      "logits/rejected": -0.8091796636581421,
      "logps/chosen": -0.646679699420929,
      "logps/rejected": -1.7273437976837158,
      "loss": 0.8829,
      "nll_loss": 0.829882800579071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06467285007238388,
      "rewards/margins": 0.10795898735523224,
      "rewards/rejected": -0.17270508408546448,
      "step": 4670
    },
    {
      "epoch": 0.3551373501290029,
      "grad_norm": 2.2943867159959916,
      "learning_rate": 1.1694106924093723e-06,
      "log_odds_chosen": 1.168481469154358,
      "log_odds_ratio": -0.4696289002895355,
      "logits/chosen": -0.9359375238418579,
      "logits/rejected": -0.836718738079071,
      "logps/chosen": -0.728515625,
      "logps/rejected": -1.5451171398162842,
      "loss": 0.8728,
      "nll_loss": 0.8746093511581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07291259616613388,
      "rewards/margins": 0.08176727592945099,
      "rewards/rejected": -0.15458984673023224,
      "step": 4680
    },
    {
      "epoch": 0.3558961906207315,
      "grad_norm": 2.6928939956255524,
      "learning_rate": 1.1681633206491381e-06,
      "log_odds_chosen": 1.176171898841858,
      "log_odds_ratio": -0.4358886778354645,
      "logits/chosen": -0.9214843511581421,
      "logits/rejected": -0.7984374761581421,
      "logps/chosen": -0.6650390625,
      "logps/rejected": -1.473242163658142,
      "loss": 0.9016,
      "nll_loss": 0.853515625,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06654052436351776,
      "rewards/margins": 0.08079834282398224,
      "rewards/rejected": -0.147216796875,
      "step": 4690
    },
    {
      "epoch": 0.35665503111246016,
      "grad_norm": 2.6738779587849146,
      "learning_rate": 1.1669199319831564e-06,
      "log_odds_chosen": 1.326562523841858,
      "log_odds_ratio": -0.416015625,
      "logits/chosen": -0.915820300579071,
      "logits/rejected": -0.813671886920929,
      "logps/chosen": -0.674023449420929,
      "logps/rejected": -1.578515648841858,
      "loss": 0.8946,
      "nll_loss": 0.8443359136581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0673828125,
      "rewards/margins": 0.09053955227136612,
      "rewards/rejected": -0.15786132216453552,
      "step": 4700
    },
    {
      "epoch": 0.3574138716041888,
      "grad_norm": 2.400721974422087,
      "learning_rate": 1.1656805052584958e-06,
      "log_odds_chosen": 1.5525391101837158,
      "log_odds_ratio": -0.41228026151657104,
      "logits/chosen": -0.9332031011581421,
      "logits/rejected": -0.82421875,
      "logps/chosen": -0.6337890625,
      "logps/rejected": -1.6853516101837158,
      "loss": 0.8857,
      "nll_loss": 0.789257824420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06333007663488388,
      "rewards/margins": 0.10518798977136612,
      "rewards/rejected": -0.16850586235523224,
      "step": 4710
    },
    {
      "epoch": 0.35817271209591744,
      "grad_norm": 2.78621158336074,
      "learning_rate": 1.164445019479164e-06,
      "log_odds_chosen": 1.36474609375,
      "log_odds_ratio": -0.4100097715854645,
      "logits/chosen": -0.942578136920929,
      "logits/rejected": -0.8260742425918579,
      "logps/chosen": -0.6576172113418579,
      "logps/rejected": -1.57421875,
      "loss": 0.8657,
      "nll_loss": 0.802929699420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06569824367761612,
      "rewards/margins": 0.09161376953125,
      "rewards/rejected": -0.15742187201976776,
      "step": 4720
    },
    {
      "epoch": 0.3589315525876461,
      "grad_norm": 2.1486906743867236,
      "learning_rate": 1.1632134538046105e-06,
      "log_odds_chosen": 1.153161644935608,
      "log_odds_ratio": -0.4811035096645355,
      "logits/chosen": -0.9345703125,
      "logits/rejected": -0.806835949420929,
      "logps/chosen": -0.6722656488418579,
      "logps/rejected": -1.4718749523162842,
      "loss": 0.8575,
      "nll_loss": 0.855664074420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06721191108226776,
      "rewards/margins": 0.07992400974035263,
      "rewards/rejected": -0.14705809950828552,
      "step": 4730
    },
    {
      "epoch": 0.3596903930793747,
      "grad_norm": 2.365164784115729,
      "learning_rate": 1.1619857875482536e-06,
      "log_odds_chosen": 1.594384789466858,
      "log_odds_ratio": -0.3369140625,
      "logits/chosen": -0.931640625,
      "logits/rejected": -0.832226574420929,
      "logps/chosen": -0.630664050579071,
      "logps/rejected": -1.719140648841858,
      "loss": 0.8798,
      "nll_loss": 0.808300793170929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0631103515625,
      "rewards/margins": 0.10862121731042862,
      "rewards/rejected": -0.17164306342601776,
      "step": 4740
    },
    {
      "epoch": 0.36044923357110337,
      "grad_norm": 2.1740905865131164,
      "learning_rate": 1.1607620001760185e-06,
      "log_odds_chosen": 1.189550757408142,
      "log_odds_ratio": -0.46137696504592896,
      "logits/chosen": -0.9443359375,
      "logits/rejected": -0.8707031011581421,
      "logps/chosen": -0.6524413824081421,
      "logps/rejected": -1.4599609375,
      "loss": 0.8714,
      "nll_loss": 0.7978515625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06529541313648224,
      "rewards/margins": 0.08076782524585724,
      "rewards/rejected": -0.14604492485523224,
      "step": 4750
    },
    {
      "epoch": 0.361208074062832,
      "grad_norm": 2.1663302947053142,
      "learning_rate": 1.1595420713048968e-06,
      "log_odds_chosen": 1.485742211341858,
      "log_odds_ratio": -0.3995117247104645,
      "logits/chosen": -0.9716796875,
      "logits/rejected": -0.8482421636581421,
      "logps/chosen": -0.7230468988418579,
      "logps/rejected": -1.788671851158142,
      "loss": 0.8855,
      "nll_loss": 0.843554675579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07232666015625,
      "rewards/margins": 0.10650940239429474,
      "rewards/rejected": -0.17890624701976776,
      "step": 4760
    },
    {
      "epoch": 0.36196691455456065,
      "grad_norm": 3.308691876993286,
      "learning_rate": 1.1583259807015182e-06,
      "log_odds_chosen": 1.368554711341858,
      "log_odds_ratio": -0.4066406190395355,
      "logits/chosen": -0.9326171875,
      "logits/rejected": -0.822070300579071,
      "logps/chosen": -0.694531261920929,
      "logps/rejected": -1.6535155773162842,
      "loss": 0.88,
      "nll_loss": 0.863085925579071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06947021186351776,
      "rewards/margins": 0.09576416015625,
      "rewards/rejected": -0.16523437201976776,
      "step": 4770
    },
    {
      "epoch": 0.3627257550462893,
      "grad_norm": 2.192361081732721,
      "learning_rate": 1.1571137082807434e-06,
      "log_odds_chosen": 1.4275391101837158,
      "log_odds_ratio": -0.38886719942092896,
      "logits/chosen": -0.965039074420929,
      "logits/rejected": -0.8587890863418579,
      "logps/chosen": -0.6673828363418579,
      "logps/rejected": -1.667578101158142,
      "loss": 0.8549,
      "nll_loss": 0.8187500238418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06673584133386612,
      "rewards/margins": 0.10012207180261612,
      "rewards/rejected": -0.16679687798023224,
      "step": 4780
    },
    {
      "epoch": 0.36348459553801793,
      "grad_norm": 3.3996780477885356,
      "learning_rate": 1.155905234104269e-06,
      "log_odds_chosen": 1.564660668373108,
      "log_odds_ratio": -0.3798828125,
      "logits/chosen": -0.943554699420929,
      "logits/rejected": -0.795703113079071,
      "logps/chosen": -0.6434570550918579,
      "logps/rejected": -1.7576172351837158,
      "loss": 0.8883,
      "nll_loss": 0.8423827886581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06425781548023224,
      "rewards/margins": 0.11158599704504013,
      "rewards/rejected": -0.17592772841453552,
      "step": 4790
    },
    {
      "epoch": 0.36424343602974657,
      "grad_norm": 2.185774065398957,
      "learning_rate": 1.1547005383792516e-06,
      "log_odds_chosen": 1.408593773841858,
      "log_odds_ratio": -0.4390625059604645,
      "logits/chosen": -0.964062511920929,
      "logits/rejected": -0.8746093511581421,
      "logps/chosen": -0.65283203125,
      "logps/rejected": -1.615234375,
      "loss": 0.8545,
      "nll_loss": 0.817187488079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06531982123851776,
      "rewards/margins": 0.09609375149011612,
      "rewards/rejected": -0.16140136122703552,
      "step": 4800
    },
    {
      "epoch": 0.3650022765214752,
      "grad_norm": 2.3827746388722084,
      "learning_rate": 1.1534996014569446e-06,
      "log_odds_chosen": 1.5846679210662842,
      "log_odds_ratio": -0.39116209745407104,
      "logits/chosen": -0.992382824420929,
      "logits/rejected": -0.8853515386581421,
      "logps/chosen": -0.662792980670929,
      "logps/rejected": -1.7734375,
      "loss": 0.8416,
      "nll_loss": 0.82470703125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06625976413488388,
      "rewards/margins": 0.11094971001148224,
      "rewards/rejected": -0.17724609375,
      "step": 4810
    },
    {
      "epoch": 0.36576111701320385,
      "grad_norm": 2.22383281870511,
      "learning_rate": 1.1523024038313547e-06,
      "log_odds_chosen": 1.4586913585662842,
      "log_odds_ratio": -0.3944335877895355,
      "logits/chosen": -0.96875,
      "logits/rejected": -0.8539062738418579,
      "logps/chosen": -0.692675769329071,
      "logps/rejected": -1.717187523841858,
      "loss": 0.8684,
      "nll_loss": 0.8714843988418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06920166313648224,
      "rewards/margins": 0.10234222561120987,
      "rewards/rejected": -0.17170409858226776,
      "step": 4820
    },
    {
      "epoch": 0.36651995750493244,
      "grad_norm": 2.405173511680066,
      "learning_rate": 1.1511089261379083e-06,
      "log_odds_chosen": 1.4882080554962158,
      "log_odds_ratio": -0.40507811307907104,
      "logits/chosen": -0.91796875,
      "logits/rejected": -0.8207031488418579,
      "logps/chosen": -0.664257824420929,
      "logps/rejected": -1.7685546875,
      "loss": 0.874,
      "nll_loss": 0.831250011920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06644286960363388,
      "rewards/margins": 0.11036987602710724,
      "rewards/rejected": -0.17680664360523224,
      "step": 4830
    },
    {
      "epoch": 0.3672787979966611,
      "grad_norm": 2.3117284638928,
      "learning_rate": 1.149919149152138e-06,
      "log_odds_chosen": 1.329003930091858,
      "log_odds_ratio": -0.41816407442092896,
      "logits/chosen": -0.9652343988418579,
      "logits/rejected": -0.866992175579071,
      "logps/chosen": -0.6732422113418579,
      "logps/rejected": -1.618749976158142,
      "loss": 0.8594,
      "nll_loss": 0.854687511920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06730957329273224,
      "rewards/margins": 0.09454345703125,
      "rewards/rejected": -0.16188964247703552,
      "step": 4840
    },
    {
      "epoch": 0.3680376384883897,
      "grad_norm": 2.2721615455443147,
      "learning_rate": 1.148733053788381e-06,
      "log_odds_chosen": 1.3947265148162842,
      "log_odds_ratio": -0.4109863340854645,
      "logits/chosen": -0.950390636920929,
      "logits/rejected": -0.8623046875,
      "logps/chosen": -0.6473633050918579,
      "logps/rejected": -1.59375,
      "loss": 0.8642,
      "nll_loss": 0.8150390386581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.064697265625,
      "rewards/margins": 0.09471587836742401,
      "rewards/rejected": -0.159423828125,
      "step": 4850
    },
    {
      "epoch": 0.36879647898011836,
      "grad_norm": 2.2251074013039718,
      "learning_rate": 1.1475506210984938e-06,
      "log_odds_chosen": 1.3505859375,
      "log_odds_ratio": -0.40800780057907104,
      "logits/chosen": -0.9847656488418579,
      "logits/rejected": -0.8919922113418579,
      "logps/chosen": -0.66650390625,
      "logps/rejected": -1.58984375,
      "loss": 0.835,
      "nll_loss": 0.805468738079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06657715141773224,
      "rewards/margins": 0.09235839545726776,
      "rewards/rejected": -0.15900878608226776,
      "step": 4860
    },
    {
      "epoch": 0.369555319471847,
      "grad_norm": 1.9517990530874827,
      "learning_rate": 1.1463718322705807e-06,
      "log_odds_chosen": 1.147802710533142,
      "log_odds_ratio": -0.43701171875,
      "logits/chosen": -0.955273449420929,
      "logits/rejected": -0.8939453363418579,
      "logps/chosen": -0.6595703363418579,
      "logps/rejected": -1.411718726158142,
      "loss": 0.8602,
      "nll_loss": 0.8057616949081421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06586913764476776,
      "rewards/margins": 0.07521972805261612,
      "rewards/rejected": -0.14113768935203552,
      "step": 4870
    },
    {
      "epoch": 0.37031415996357564,
      "grad_norm": 2.4321003116514985,
      "learning_rate": 1.1451966686277364e-06,
      "log_odds_chosen": 1.424414038658142,
      "log_odds_ratio": -0.38740235567092896,
      "logits/chosen": -0.9224609136581421,
      "logits/rejected": -0.7994140386581421,
      "logps/chosen": -0.6337890625,
      "logps/rejected": -1.6320312023162842,
      "loss": 0.859,
      "nll_loss": 0.802734375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0633544921875,
      "rewards/margins": 0.09980468451976776,
      "rewards/rejected": -0.16313476860523224,
      "step": 4880
    },
    {
      "epoch": 0.3710730004553043,
      "grad_norm": 2.3392296736339198,
      "learning_rate": 1.1440251116268034e-06,
      "log_odds_chosen": 1.458593726158142,
      "log_odds_ratio": -0.4004882872104645,
      "logits/chosen": -0.9595702886581421,
      "logits/rejected": -0.8148437738418579,
      "logps/chosen": -0.670703113079071,
      "logps/rejected": -1.6671874523162842,
      "loss": 0.8698,
      "nll_loss": 0.810742199420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06708984076976776,
      "rewards/margins": 0.09968261420726776,
      "rewards/rejected": -0.166748046875,
      "step": 4890
    },
    {
      "epoch": 0.3718318409470329,
      "grad_norm": 2.0434121903510243,
      "learning_rate": 1.1428571428571428e-06,
      "log_odds_chosen": 1.3869140148162842,
      "log_odds_ratio": -0.425048828125,
      "logits/chosen": -0.9623047113418579,
      "logits/rejected": -0.8697265386581421,
      "logps/chosen": -0.664843738079071,
      "logps/rejected": -1.6328125,
      "loss": 0.8433,
      "nll_loss": 0.8255859613418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06645508110523224,
      "rewards/margins": 0.09688720852136612,
      "rewards/rejected": -0.1634521484375,
      "step": 4900
    },
    {
      "epoch": 0.37259068143876156,
      "grad_norm": 2.1097305955384544,
      "learning_rate": 1.14169274403942e-06,
      "log_odds_chosen": 1.365576148033142,
      "log_odds_ratio": -0.4039550721645355,
      "logits/chosen": -0.9173828363418579,
      "logits/rejected": -0.8062499761581421,
      "logps/chosen": -0.664257824420929,
      "logps/rejected": -1.6240234375,
      "loss": 0.836,
      "nll_loss": 0.849804699420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06641845405101776,
      "rewards/margins": 0.09599609673023224,
      "rewards/rejected": -0.162353515625,
      "step": 4910
    },
    {
      "epoch": 0.3733495219304902,
      "grad_norm": 2.096457299618778,
      "learning_rate": 1.140531897024402e-06,
      "log_odds_chosen": 1.4734375476837158,
      "log_odds_ratio": -0.4014648497104645,
      "logits/chosen": -0.982226550579071,
      "logits/rejected": -0.8636718988418579,
      "logps/chosen": -0.63330078125,
      "logps/rejected": -1.6238281726837158,
      "loss": 0.8636,
      "nll_loss": 0.770703136920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06337890774011612,
      "rewards/margins": 0.09904785454273224,
      "rewards/rejected": -0.16240234673023224,
      "step": 4920
    },
    {
      "epoch": 0.37410836242221884,
      "grad_norm": 2.625565690686019,
      "learning_rate": 1.13937458379177e-06,
      "log_odds_chosen": 1.322167992591858,
      "log_odds_ratio": -0.45234376192092896,
      "logits/chosen": -0.924609363079071,
      "logits/rejected": -0.819140613079071,
      "logps/chosen": -0.68115234375,
      "logps/rejected": -1.6484375,
      "loss": 0.856,
      "nll_loss": 0.820996105670929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06809081882238388,
      "rewards/margins": 0.09653320163488388,
      "rewards/rejected": -0.16477051377296448,
      "step": 4930
    },
    {
      "epoch": 0.3748672029139475,
      "grad_norm": 2.203218780916155,
      "learning_rate": 1.1382207864489444e-06,
      "log_odds_chosen": 1.2156250476837158,
      "log_odds_ratio": -0.43525391817092896,
      "logits/chosen": -0.9537109136581421,
      "logits/rejected": -0.828320324420929,
      "logps/chosen": -0.624804675579071,
      "logps/rejected": -1.423437476158142,
      "loss": 0.8619,
      "nll_loss": 0.8041015863418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06251220405101776,
      "rewards/margins": 0.07990722358226776,
      "rewards/rejected": -0.14243164658546448,
      "step": 4940
    },
    {
      "epoch": 0.3756260434056761,
      "grad_norm": 2.7540164581142594,
      "learning_rate": 1.1370704872299223e-06,
      "log_odds_chosen": 1.2926757335662842,
      "log_odds_ratio": -0.41874998807907104,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.891406238079071,
      "logps/chosen": -0.6659179925918579,
      "logps/rejected": -1.556640625,
      "loss": 0.8565,
      "nll_loss": 0.809765636920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06658935546875,
      "rewards/margins": 0.08905029296875,
      "rewards/rejected": -0.15556640923023224,
      "step": 4950
    },
    {
      "epoch": 0.37638488389740477,
      "grad_norm": 2.025269380602589,
      "learning_rate": 1.1359236684941295e-06,
      "log_odds_chosen": 1.349511742591858,
      "log_odds_ratio": -0.41899412870407104,
      "logits/chosen": -0.9544922113418579,
      "logits/rejected": -0.8314453363418579,
      "logps/chosen": -0.6807616949081421,
      "logps/rejected": -1.603515625,
      "loss": 0.8806,
      "nll_loss": 0.882617175579071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06804199516773224,
      "rewards/margins": 0.09243164211511612,
      "rewards/rejected": -0.16044922173023224,
      "step": 4960
    },
    {
      "epoch": 0.3771437243891334,
      "grad_norm": 2.2485565467037207,
      "learning_rate": 1.1347803127252839e-06,
      "log_odds_chosen": 1.569238305091858,
      "log_odds_ratio": -0.3885253965854645,
      "logits/chosen": -1.0224609375,
      "logits/rejected": -0.893359363079071,
      "logps/chosen": -0.633496105670929,
      "logps/rejected": -1.7433593273162842,
      "loss": 0.8377,
      "nll_loss": 0.8037109375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06339111179113388,
      "rewards/margins": 0.11101684719324112,
      "rewards/rejected": -0.17434081435203552,
      "step": 4970
    },
    {
      "epoch": 0.37790256488086205,
      "grad_norm": 2.165794505306308,
      "learning_rate": 1.1336404025302715e-06,
      "log_odds_chosen": 1.4425780773162842,
      "log_odds_ratio": -0.40278321504592896,
      "logits/chosen": -0.9632812738418579,
      "logits/rejected": -0.84375,
      "logps/chosen": -0.6806640625,
      "logps/rejected": -1.71484375,
      "loss": 0.8718,
      "nll_loss": 0.8779296875,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06804199516773224,
      "rewards/margins": 0.10341186821460724,
      "rewards/rejected": -0.17160645127296448,
      "step": 4980
    },
    {
      "epoch": 0.3786614053725907,
      "grad_norm": 2.164096377102248,
      "learning_rate": 1.1325039206380352e-06,
      "log_odds_chosen": 1.0779297351837158,
      "log_odds_ratio": -0.47822266817092896,
      "logits/chosen": -0.9638671875,
      "logits/rejected": -0.884570300579071,
      "logps/chosen": -0.641894519329071,
      "logps/rejected": -1.3117187023162842,
      "loss": 0.8722,
      "nll_loss": 0.8316406011581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06416015326976776,
      "rewards/margins": 0.0670166015625,
      "rewards/rejected": -0.13124999403953552,
      "step": 4990
    },
    {
      "epoch": 0.3794202458643193,
      "grad_norm": 2.4681602129978293,
      "learning_rate": 1.131370849898476e-06,
      "log_odds_chosen": 1.2675902843475342,
      "log_odds_ratio": -0.4202636778354645,
      "logits/chosen": -0.980273425579071,
      "logits/rejected": -0.873046875,
      "logps/chosen": -0.6693359613418579,
      "logps/rejected": -1.5099608898162842,
      "loss": 0.8537,
      "nll_loss": 0.8560546636581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06689453125,
      "rewards/margins": 0.08423157036304474,
      "rewards/rejected": -0.15097656846046448,
      "step": 5000
    },
    {
      "epoch": 0.3794202458643193,
      "eval_log_odds_chosen": 1.0485382080078125,
      "eval_log_odds_ratio": -0.5006263852119446,
      "eval_logits/chosen": -0.8404242396354675,
      "eval_logits/rejected": -0.757588803768158,
      "eval_logps/chosen": -0.7668778300285339,
      "eval_logps/rejected": -1.5307999849319458,
      "eval_loss": 1.1802747249603271,
      "eval_nll_loss": 1.1388089656829834,
      "eval_rewards/accuracies": 0.7127641439437866,
      "eval_rewards/chosen": -0.07668878138065338,
      "eval_rewards/margins": 0.07640629261732101,
      "eval_rewards/rejected": -0.15308316051959991,
      "eval_runtime": 1322.2772,
      "eval_samples_per_second": 71.001,
      "eval_steps_per_second": 1.109,
      "step": 5000
    }
  ],
  "logging_steps": 10,
  "max_steps": 13178,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 5000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}