{
  "best_metric": 1.2826783657073975,
  "best_model_checkpoint": "models/llama3.2-3b-orpo-mini/checkpoint-5000",
  "epoch": 1.0,
  "eval_steps": 5000,
  "global_step": 7749,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0012904890953671442,
      "grad_norm": 16.999913281366116,
      "learning_rate": 8e-07,
      "log_odds_chosen": 0.679980456829071,
      "log_odds_ratio": -0.634570300579071,
      "logits/chosen": -1.0515625476837158,
      "logits/rejected": -1.062890648841858,
      "logps/chosen": -2.380078077316284,
      "logps/rejected": -3.0171875953674316,
      "loss": 2.6235,
      "nll_loss": 2.483203172683716,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.23818358778953552,
      "rewards/margins": 0.06354980170726776,
      "rewards/rejected": -0.30180662870407104,
      "step": 10
    },
    {
      "epoch": 0.0025809781907342884,
      "grad_norm": 9.565778103394054,
      "learning_rate": 1.6e-06,
      "log_odds_chosen": 0.594287097454071,
      "log_odds_ratio": -0.7671874761581421,
      "logits/chosen": -1.169921875,
      "logits/rejected": -1.1298828125,
      "logps/chosen": -2.4136719703674316,
      "logps/rejected": -2.9671874046325684,
      "loss": 2.4974,
      "nll_loss": 2.360546827316284,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.24155274033546448,
      "rewards/margins": 0.05530090257525444,
      "rewards/rejected": -0.2967285215854645,
      "step": 20
    },
    {
      "epoch": 0.0038714672861014324,
      "grad_norm": 3.9136251197922145,
      "learning_rate": 2.4e-06,
      "log_odds_chosen": 0.607128918170929,
      "log_odds_ratio": -0.58154296875,
      "logits/chosen": -1.32421875,
      "logits/rejected": -1.2667968273162842,
      "logps/chosen": -1.776953101158142,
      "logps/rejected": -2.3128905296325684,
      "loss": 2.0588,
      "nll_loss": 2.059375047683716,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.17734375596046448,
      "rewards/margins": 0.05376587063074112,
      "rewards/rejected": -0.23129883408546448,
      "step": 30
    },
    {
      "epoch": 0.005161956381468577,
      "grad_norm": 2.898943978823639,
      "learning_rate": 3.2e-06,
      "log_odds_chosen": 0.571044921875,
      "log_odds_ratio": -0.6006835699081421,
      "logits/chosen": -1.0539062023162842,
      "logits/rejected": -0.981249988079071,
      "logps/chosen": -1.6183593273162842,
      "logps/rejected": -2.096874952316284,
      "loss": 1.8951,
      "nll_loss": 1.8679687976837158,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.16181640326976776,
      "rewards/margins": 0.04776611179113388,
      "rewards/rejected": -0.20957031846046448,
      "step": 40
    },
    {
      "epoch": 0.0064524454768357204,
      "grad_norm": 2.9265981814461894,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.503143310546875,
      "log_odds_ratio": -0.633984386920929,
      "logits/chosen": -0.9986327886581421,
      "logits/rejected": -0.9150390625,
      "logps/chosen": -1.4988281726837158,
      "logps/rejected": -1.9386718273162842,
      "loss": 1.8313,
      "nll_loss": 1.7687499523162842,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.14992675185203552,
      "rewards/margins": 0.04414062574505806,
      "rewards/rejected": -0.19414062798023224,
      "step": 50
    },
    {
      "epoch": 0.007742934572202865,
      "grad_norm": 2.717291268996869,
      "learning_rate": 4.8e-06,
      "log_odds_chosen": 0.563427746295929,
      "log_odds_ratio": -0.570507824420929,
      "logits/chosen": -1.0320312976837158,
      "logits/rejected": -0.9408203363418579,
      "logps/chosen": -1.348046898841858,
      "logps/rejected": -1.8046875,
      "loss": 1.7306,
      "nll_loss": 1.617578148841858,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.13481445610523224,
      "rewards/margins": 0.04554443433880806,
      "rewards/rejected": -0.18046875298023224,
      "step": 60
    },
    {
      "epoch": 0.009033423667570008,
      "grad_norm": 2.8399549099448613,
      "learning_rate": 5.6e-06,
      "log_odds_chosen": 0.5670410394668579,
      "log_odds_ratio": -0.587109386920929,
      "logits/chosen": -1.1511719226837158,
      "logits/rejected": -1.054101586341858,
      "logps/chosen": -1.2833983898162842,
      "logps/rejected": -1.7550780773162842,
      "loss": 1.7214,
      "nll_loss": 1.6066405773162842,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.12834472954273224,
      "rewards/margins": 0.04714050143957138,
      "rewards/rejected": -0.175537109375,
      "step": 70
    },
    {
      "epoch": 0.010323912762937154,
      "grad_norm": 2.611682433823492,
      "learning_rate": 6.4e-06,
      "log_odds_chosen": 0.5081787109375,
      "log_odds_ratio": -0.589648425579071,
      "logits/chosen": -1.1337890625,
      "logits/rejected": -1.046484351158142,
      "logps/chosen": -1.2878906726837158,
      "logps/rejected": -1.7136719226837158,
      "loss": 1.6566,
      "nll_loss": 1.6574218273162842,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.12868651747703552,
      "rewards/margins": 0.04265747219324112,
      "rewards/rejected": -0.17145995795726776,
      "step": 80
    },
    {
      "epoch": 0.011614401858304297,
      "grad_norm": 2.5560068706226295,
      "learning_rate": 7.2e-06,
      "log_odds_chosen": 0.44941407442092896,
      "log_odds_ratio": -0.6102539300918579,
      "logits/chosen": -1.154882788658142,
      "logits/rejected": -1.091210961341858,
      "logps/chosen": -1.216406226158142,
      "logps/rejected": -1.5867187976837158,
      "loss": 1.6146,
      "nll_loss": 1.5089843273162842,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.12163086235523224,
      "rewards/margins": 0.03702545166015625,
      "rewards/rejected": -0.15852050483226776,
      "step": 90
    },
    {
      "epoch": 0.012904890953671441,
      "grad_norm": 2.2454822411218425,
      "learning_rate": 8e-06,
      "log_odds_chosen": 0.5118652582168579,
      "log_odds_ratio": -0.583789050579071,
      "logits/chosen": -1.10546875,
      "logits/rejected": -1.005859375,
      "logps/chosen": -1.197265625,
      "logps/rejected": -1.613671898841858,
      "loss": 1.5452,
      "nll_loss": 1.5167968273162842,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.11977539211511612,
      "rewards/margins": 0.04156799241900444,
      "rewards/rejected": -0.1611328125,
      "step": 100
    },
    {
      "epoch": 0.014195380049038586,
      "grad_norm": 2.2152737471228177,
      "learning_rate": 7.627700713964738e-06,
      "log_odds_chosen": 0.5327392816543579,
      "log_odds_ratio": -0.570996105670929,
      "logits/chosen": -1.0710937976837158,
      "logits/rejected": -0.9955078363418579,
      "logps/chosen": -1.146093726158142,
      "logps/rejected": -1.560937523841858,
      "loss": 1.5196,
      "nll_loss": 1.521484375,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11467285454273224,
      "rewards/margins": 0.04147491604089737,
      "rewards/rejected": -0.15615233778953552,
      "step": 110
    },
    {
      "epoch": 0.01548586914440573,
      "grad_norm": 1.759447238527514,
      "learning_rate": 7.3029674334022146e-06,
      "log_odds_chosen": 0.43010252714157104,
      "log_odds_ratio": -0.6280273199081421,
      "logits/chosen": -1.0740234851837158,
      "logits/rejected": -1.010156273841858,
      "logps/chosen": -1.089453101158142,
      "logps/rejected": -1.432031273841858,
      "loss": 1.5054,
      "nll_loss": 1.419921875,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.10898437350988388,
      "rewards/margins": 0.03422699123620987,
      "rewards/rejected": -0.14313964545726776,
      "step": 120
    },
    {
      "epoch": 0.016776358239772873,
      "grad_norm": 1.7820356883941126,
      "learning_rate": 7.016464154456233e-06,
      "log_odds_chosen": 0.7165771722793579,
      "log_odds_ratio": -0.515429675579071,
      "logits/chosen": -1.0480468273162842,
      "logits/rejected": -0.9751952886581421,
      "logps/chosen": -1.0460937023162842,
      "logps/rejected": -1.6105468273162842,
      "loss": 1.4591,
      "nll_loss": 1.4132812023162842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1046142578125,
      "rewards/margins": 0.05650634691119194,
      "rewards/rejected": -0.1611328125,
      "step": 130
    },
    {
      "epoch": 0.018066847335140017,
      "grad_norm": 1.814261370304629,
      "learning_rate": 6.7612340378281325e-06,
      "log_odds_chosen": 0.5973144769668579,
      "log_odds_ratio": -0.551074206829071,
      "logits/chosen": -0.9820312261581421,
      "logits/rejected": -0.926953136920929,
      "logps/chosen": -1.0166015625,
      "logps/rejected": -1.46484375,
      "loss": 1.4674,
      "nll_loss": 1.374414086341858,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1016845703125,
      "rewards/margins": 0.04479675367474556,
      "rewards/rejected": -0.146484375,
      "step": 140
    },
    {
      "epoch": 0.019357336430507164,
      "grad_norm": 1.7617502134980012,
      "learning_rate": 6.531972647421809e-06,
      "log_odds_chosen": 0.609375,
      "log_odds_ratio": -0.5322265625,
      "logits/chosen": -1.0107421875,
      "logits/rejected": -0.9390624761581421,
      "logps/chosen": -1.0427734851837158,
      "logps/rejected": -1.509374976158142,
      "loss": 1.4479,
      "nll_loss": 1.420312523841858,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.10434570163488388,
      "rewards/margins": 0.04667968675494194,
      "rewards/rejected": -0.15087890625,
      "step": 150
    },
    {
      "epoch": 0.020647825525874308,
      "grad_norm": 2.126663282636822,
      "learning_rate": 6.3245553203367575e-06,
      "log_odds_chosen": 0.50140380859375,
      "log_odds_ratio": -0.5757812261581421,
      "logits/chosen": -0.943164050579071,
      "logits/rejected": -0.8433593511581421,
      "logps/chosen": -1.019140601158142,
      "logps/rejected": -1.4167969226837158,
      "loss": 1.4116,
      "nll_loss": 1.387109398841858,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10190429538488388,
      "rewards/margins": 0.03973083570599556,
      "rewards/rejected": -0.14167480170726776,
      "step": 160
    },
    {
      "epoch": 0.02193831462124145,
      "grad_norm": 1.7741560769312266,
      "learning_rate": 6.135719910778963e-06,
      "log_odds_chosen": 0.614306628704071,
      "log_odds_ratio": -0.540478527545929,
      "logits/chosen": -0.923828125,
      "logits/rejected": -0.828320324420929,
      "logps/chosen": -0.982617199420929,
      "logps/rejected": -1.4513671398162842,
      "loss": 1.3886,
      "nll_loss": 1.2941405773162842,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.09836425632238388,
      "rewards/margins": 0.0467681884765625,
      "rewards/rejected": -0.14501953125,
      "step": 170
    },
    {
      "epoch": 0.023228803716608595,
      "grad_norm": 2.12319096121112,
      "learning_rate": 5.962847939999439e-06,
      "log_odds_chosen": 0.5820678472518921,
      "log_odds_ratio": -0.55029296875,
      "logits/chosen": -0.891406238079071,
      "logits/rejected": -0.817187488079071,
      "logps/chosen": -1.014062523841858,
      "logps/rejected": -1.4542968273162842,
      "loss": 1.4575,
      "nll_loss": 1.3859374523162842,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.10158691555261612,
      "rewards/margins": 0.04384765774011612,
      "rewards/rejected": -0.14543457329273224,
      "step": 180
    },
    {
      "epoch": 0.024519292811975738,
      "grad_norm": 1.9249787489004448,
      "learning_rate": 5.803810000880094e-06,
      "log_odds_chosen": 0.4267334043979645,
      "log_odds_ratio": -0.59130859375,
      "logits/chosen": -0.9203125238418579,
      "logits/rejected": -0.8333984613418579,
      "logps/chosen": -1.0105469226837158,
      "logps/rejected": -1.3347656726837158,
      "loss": 1.3584,
      "nll_loss": 1.294921875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.10104980319738388,
      "rewards/margins": 0.03241882473230362,
      "rewards/rejected": -0.13347168266773224,
      "step": 190
    },
    {
      "epoch": 0.025809781907342882,
      "grad_norm": 25.214922233680923,
      "learning_rate": 5.65685424949238e-06,
      "log_odds_chosen": 0.551403820514679,
      "log_odds_ratio": -0.5633789300918579,
      "logits/chosen": -0.9150390625,
      "logits/rejected": -0.8525390625,
      "logps/chosen": -0.984375,
      "logps/rejected": -1.4072265625,
      "loss": 1.3821,
      "nll_loss": 1.3093750476837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09848632663488388,
      "rewards/margins": 0.04229431226849556,
      "rewards/rejected": -0.14082030951976776,
      "step": 200
    },
    {
      "epoch": 0.02710027100271003,
      "grad_norm": 2.3550027578690917,
      "learning_rate": 5.5205244747388325e-06,
      "log_odds_chosen": 0.610302746295929,
      "log_odds_ratio": -0.5941406488418579,
      "logits/chosen": -0.892382800579071,
      "logits/rejected": -0.8062499761581421,
      "logps/chosen": -0.9521484375,
      "logps/rejected": -1.4326171875,
      "loss": 1.3412,
      "nll_loss": 1.3125,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.09521484375,
      "rewards/margins": 0.04804839938879013,
      "rewards/rejected": -0.14323730766773224,
      "step": 210
    },
    {
      "epoch": 0.028390760098077172,
      "grad_norm": 1.4889952185004816,
      "learning_rate": 5.393598899705936e-06,
      "log_odds_chosen": 0.6220947504043579,
      "log_odds_ratio": -0.544726550579071,
      "logits/chosen": -0.902148425579071,
      "logits/rejected": -0.83203125,
      "logps/chosen": -0.9488281011581421,
      "logps/rejected": -1.427343726158142,
      "loss": 1.3482,
      "nll_loss": 1.314453125,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09479980170726776,
      "rewards/margins": 0.04799041897058487,
      "rewards/rejected": -0.14284667372703552,
      "step": 220
    },
    {
      "epoch": 0.029681249193444316,
      "grad_norm": 1.703926654060642,
      "learning_rate": 5.275043787166296e-06,
      "log_odds_chosen": 0.609619140625,
      "log_odds_ratio": -0.5557616949081421,
      "logits/chosen": -0.951367199420929,
      "logits/rejected": -0.8658202886581421,
      "logps/chosen": -0.9423828125,
      "logps/rejected": -1.404687523841858,
      "loss": 1.2949,
      "nll_loss": 1.216796875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.09414062649011612,
      "rewards/margins": 0.04624023288488388,
      "rewards/rejected": -0.14040526747703552,
      "step": 230
    },
    {
      "epoch": 0.03097173828881146,
      "grad_norm": 1.6528161594026065,
      "learning_rate": 5.163977794943223e-06,
      "log_odds_chosen": 0.680041491985321,
      "log_odds_ratio": -0.5626465082168579,
      "logits/chosen": -0.888867199420929,
      "logits/rejected": -0.81201171875,
      "logps/chosen": -0.932812511920929,
      "logps/rejected": -1.429296851158142,
      "loss": 1.3159,
      "nll_loss": 1.326562523841858,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.09331054985523224,
      "rewards/margins": 0.04965972900390625,
      "rewards/rejected": -0.14296874403953552,
      "step": 240
    },
    {
      "epoch": 0.0322622273841786,
      "grad_norm": 1.734361304795382,
      "learning_rate": 5.059644256269407e-06,
      "log_odds_chosen": 0.6534668207168579,
      "log_odds_ratio": -0.5233398675918579,
      "logits/chosen": -0.9048827886581421,
      "logits/rejected": -0.805859386920929,
      "logps/chosen": -0.8851562738418579,
      "logps/rejected": -1.3544921875,
      "loss": 1.3052,
      "nll_loss": 1.3318359851837158,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.08854980766773224,
      "rewards/margins": 0.04707183688879013,
      "rewards/rejected": -0.13557128608226776,
      "step": 250
    },
    {
      "epoch": 0.03355271647954575,
      "grad_norm": 1.6367374273839836,
      "learning_rate": 4.961389383568338e-06,
      "log_odds_chosen": 0.5970458984375,
      "log_odds_ratio": -0.560742199420929,
      "logits/chosen": -0.9330078363418579,
      "logits/rejected": -0.867968738079071,
      "logps/chosen": -0.864062488079071,
      "logps/rejected": -1.2861328125,
      "loss": 1.2934,
      "nll_loss": 1.1640625,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.08637695014476776,
      "rewards/margins": 0.04220123216509819,
      "rewards/rejected": -0.12856444716453552,
      "step": 260
    },
    {
      "epoch": 0.03484320557491289,
      "grad_norm": 1.3938624286392924,
      "learning_rate": 4.8686449556014755e-06,
      "log_odds_chosen": 0.863085925579071,
      "log_odds_ratio": -0.48710936307907104,
      "logits/chosen": -0.9058593511581421,
      "logits/rejected": -0.809374988079071,
      "logps/chosen": -0.856640636920929,
      "logps/rejected": -1.482421875,
      "loss": 1.2738,
      "nll_loss": 1.2326171398162842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08564452826976776,
      "rewards/margins": 0.06248169019818306,
      "rewards/rejected": -0.14824219048023224,
      "step": 270
    },
    {
      "epoch": 0.036133694670280034,
      "grad_norm": 1.5136559490069907,
      "learning_rate": 4.780914437337574e-06,
      "log_odds_chosen": 0.649462878704071,
      "log_odds_ratio": -0.5328124761581421,
      "logits/chosen": -0.900195300579071,
      "logits/rejected": -0.8599609136581421,
      "logps/chosen": -0.843945324420929,
      "logps/rejected": -1.280664086341858,
      "loss": 1.2864,
      "nll_loss": 1.252539038658142,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.08439941704273224,
      "rewards/margins": 0.04367981106042862,
      "rewards/rejected": -0.1280517578125,
      "step": 280
    },
    {
      "epoch": 0.03742418376564718,
      "grad_norm": 1.5794727258843329,
      "learning_rate": 4.697761756117627e-06,
      "log_odds_chosen": 0.814257800579071,
      "log_odds_ratio": -0.526074230670929,
      "logits/chosen": -0.9175781011581421,
      "logits/rejected": -0.8392578363418579,
      "logps/chosen": -0.8304687738418579,
      "logps/rejected": -1.4373047351837158,
      "loss": 1.2624,
      "nll_loss": 1.172265648841858,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.08305664360523224,
      "rewards/margins": 0.06073608249425888,
      "rewards/rejected": -0.143798828125,
      "step": 290
    },
    {
      "epoch": 0.03871467286101433,
      "grad_norm": 1.460624199950962,
      "learning_rate": 4.618802153517006e-06,
      "log_odds_chosen": 0.642382800579071,
      "log_odds_ratio": -0.547607421875,
      "logits/chosen": -0.8814452886581421,
      "logits/rejected": -0.802539050579071,
      "logps/chosen": -0.8267577886581421,
      "logps/rejected": -1.258398413658142,
      "loss": 1.2695,
      "nll_loss": 1.285742163658142,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.08260498195886612,
      "rewards/margins": 0.04326171800494194,
      "rewards/rejected": -0.12604980170726776,
      "step": 300
    },
    {
      "epoch": 0.04000516195638147,
      "grad_norm": 1.4774677608762785,
      "learning_rate": 4.543694673976518e-06,
      "log_odds_chosen": 0.8711913824081421,
      "log_odds_ratio": -0.4677734375,
      "logits/chosen": -0.896289050579071,
      "logits/rejected": -0.803906261920929,
      "logps/chosen": -0.7894531488418579,
      "logps/rejected": -1.4089844226837158,
      "loss": 1.2553,
      "nll_loss": 1.2179687023162842,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07893066108226776,
      "rewards/margins": 0.06187744066119194,
      "rewards/rejected": -0.14077147841453552,
      "step": 310
    },
    {
      "epoch": 0.041295651051748615,
      "grad_norm": 1.6892251839683603,
      "learning_rate": 4.472135954999579e-06,
      "log_odds_chosen": 0.837268054485321,
      "log_odds_ratio": -0.4810546934604645,
      "logits/chosen": -0.9341796636581421,
      "logits/rejected": -0.8472656011581421,
      "logps/chosen": -0.7777343988418579,
      "logps/rejected": -1.352929711341858,
      "loss": 1.2467,
      "nll_loss": 1.189843773841858,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07773437350988388,
      "rewards/margins": 0.05760955810546875,
      "rewards/rejected": -0.1353759765625,
      "step": 320
    },
    {
      "epoch": 0.04258614014711576,
      "grad_norm": 1.4446402598649475,
      "learning_rate": 4.403855060505443e-06,
      "log_odds_chosen": 0.6492675542831421,
      "log_odds_ratio": -0.5450195074081421,
      "logits/chosen": -0.930859386920929,
      "logits/rejected": -0.857421875,
      "logps/chosen": -0.7933593988418579,
      "logps/rejected": -1.2236328125,
      "loss": 1.2698,
      "nll_loss": 1.2234375476837158,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.079345703125,
      "rewards/margins": 0.04298705980181694,
      "rewards/rejected": -0.12233886867761612,
      "step": 330
    },
    {
      "epoch": 0.0438766292424829,
      "grad_norm": 1.4869685664991195,
      "learning_rate": 4.338609156373123e-06,
      "log_odds_chosen": 0.719970703125,
      "log_odds_ratio": -0.5347656011581421,
      "logits/chosen": -0.972460925579071,
      "logits/rejected": -0.919921875,
      "logps/chosen": -0.7669922113418579,
      "logps/rejected": -1.2722656726837158,
      "loss": 1.2213,
      "nll_loss": 1.1511719226837158,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07675781100988388,
      "rewards/margins": 0.05054626613855362,
      "rewards/rejected": -0.1273193359375,
      "step": 340
    },
    {
      "epoch": 0.045167118337850046,
      "grad_norm": 1.4203268907708364,
      "learning_rate": 4.27617987059879e-06,
      "log_odds_chosen": 0.61309814453125,
      "log_odds_ratio": -0.57666015625,
      "logits/chosen": -0.967968761920929,
      "logits/rejected": -0.8871093988418579,
      "logps/chosen": -0.900390625,
      "logps/rejected": -1.333593726158142,
      "loss": 1.237,
      "nll_loss": 1.1912109851837158,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0899658203125,
      "rewards/margins": 0.043502043932676315,
      "rewards/rejected": -0.13334961235523224,
      "step": 350
    },
    {
      "epoch": 0.04645760743321719,
      "grad_norm": 1.6264373465820539,
      "learning_rate": 4.216370213557839e-06,
      "log_odds_chosen": 0.7685546875,
      "log_odds_ratio": -0.522167980670929,
      "logits/chosen": -0.9574218988418579,
      "logits/rejected": -0.854296863079071,
      "logps/chosen": -0.8001953363418579,
      "logps/rejected": -1.3439452648162842,
      "loss": 1.2191,
      "nll_loss": 1.187109351158142,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.08000488579273224,
      "rewards/margins": 0.05424194410443306,
      "rewards/rejected": -0.13430175185203552,
      "step": 360
    },
    {
      "epoch": 0.04774809652858433,
      "grad_norm": 1.4514023870554544,
      "learning_rate": 4.15900195928029e-06,
      "log_odds_chosen": 0.706250011920929,
      "log_odds_ratio": -0.533984363079071,
      "logits/chosen": -0.8896484375,
      "logits/rejected": -0.791210949420929,
      "logps/chosen": -0.790820300579071,
      "logps/rejected": -1.281835913658142,
      "loss": 1.2356,
      "nll_loss": 1.149999976158142,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07911376655101776,
      "rewards/margins": 0.04909820482134819,
      "rewards/rejected": -0.12819823622703552,
      "step": 370
    },
    {
      "epoch": 0.049038585623951476,
      "grad_norm": 1.5408258542869078,
      "learning_rate": 4.103913408340617e-06,
      "log_odds_chosen": 0.7417968511581421,
      "log_odds_ratio": -0.5160156488418579,
      "logits/chosen": -0.902148425579071,
      "logits/rejected": -0.8199218511581421,
      "logps/chosen": -0.834179699420929,
      "logps/rejected": -1.361328125,
      "loss": 1.2395,
      "nll_loss": 1.2197265625,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0833740234375,
      "rewards/margins": 0.05266723781824112,
      "rewards/rejected": -0.13608399033546448,
      "step": 380
    },
    {
      "epoch": 0.05032907471931862,
      "grad_norm": 1.3953803820685304,
      "learning_rate": 4.050957468334666e-06,
      "log_odds_chosen": 0.67633056640625,
      "log_odds_ratio": -0.568359375,
      "logits/chosen": -0.937304675579071,
      "logits/rejected": -0.8798828125,
      "logps/chosen": -0.826953113079071,
      "logps/rejected": -1.3054687976837158,
      "loss": 1.2153,
      "nll_loss": 1.1720702648162842,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08273925632238388,
      "rewards/margins": 0.04777374118566513,
      "rewards/rejected": -0.13046875596046448,
      "step": 390
    },
    {
      "epoch": 0.051619563814685764,
      "grad_norm": 1.6181728160109372,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.7749999761581421,
      "log_odds_ratio": -0.54345703125,
      "logits/chosen": -0.9193359613418579,
      "logits/rejected": -0.8257812261581421,
      "logps/chosen": -0.7904297113418579,
      "logps/rejected": -1.325781226158142,
      "loss": 1.2613,
      "nll_loss": 1.288671851158142,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07900390774011612,
      "rewards/margins": 0.05355072021484375,
      "rewards/rejected": -0.13259276747703552,
      "step": 400
    },
    {
      "epoch": 0.05291005291005291,
      "grad_norm": 1.4575468785178762,
      "learning_rate": 3.950918386598359e-06,
      "log_odds_chosen": 0.745068371295929,
      "log_odds_ratio": -0.5352538824081421,
      "logits/chosen": -0.925585925579071,
      "logits/rejected": -0.850781261920929,
      "logps/chosen": -0.8037109375,
      "logps/rejected": -1.300390601158142,
      "loss": 1.2109,
      "nll_loss": 1.133203148841858,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.08041992038488388,
      "rewards/margins": 0.0496063232421875,
      "rewards/rejected": -0.13005371391773224,
      "step": 410
    },
    {
      "epoch": 0.05420054200542006,
      "grad_norm": 1.4956364954751828,
      "learning_rate": 3.903600291794132e-06,
      "log_odds_chosen": 0.76318359375,
      "log_odds_ratio": -0.519238293170929,
      "logits/chosen": -0.968554675579071,
      "logits/rejected": -0.889843761920929,
      "logps/chosen": -0.7783203125,
      "logps/rejected": -1.312890648841858,
      "loss": 1.2401,
      "nll_loss": 1.1912109851837158,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07789306342601776,
      "rewards/margins": 0.05345764011144638,
      "rewards/rejected": -0.13120117783546448,
      "step": 420
    },
    {
      "epoch": 0.0554910311007872,
      "grad_norm": 1.458771767228927,
      "learning_rate": 3.857942577363297e-06,
      "log_odds_chosen": 0.8623046875,
      "log_odds_ratio": -0.4755859375,
      "logits/chosen": -0.9839843511581421,
      "logits/rejected": -0.910351574420929,
      "logps/chosen": -0.740234375,
      "logps/rejected": -1.333398461341858,
      "loss": 1.1944,
      "nll_loss": 1.129492163658142,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07399902492761612,
      "rewards/margins": 0.05929260328412056,
      "rewards/rejected": -0.13334961235523224,
      "step": 430
    },
    {
      "epoch": 0.056781520196154345,
      "grad_norm": 1.4302837778419895,
      "learning_rate": 3.813850356982369e-06,
      "log_odds_chosen": 0.916308581829071,
      "log_odds_ratio": -0.48779296875,
      "logits/chosen": -0.935742199420929,
      "logits/rejected": -0.8349609375,
      "logps/chosen": -0.7640625238418579,
      "logps/rejected": -1.412500023841858,
      "loss": 1.2151,
      "nll_loss": 1.1892578601837158,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07640381157398224,
      "rewards/margins": 0.06473388522863388,
      "rewards/rejected": -0.1412353515625,
      "step": 440
    },
    {
      "epoch": 0.05807200929152149,
      "grad_norm": 1.5297074047561452,
      "learning_rate": 3.7712361663282537e-06,
      "log_odds_chosen": 0.6771484613418579,
      "log_odds_ratio": -0.5936523675918579,
      "logits/chosen": -0.9408203363418579,
      "logits/rejected": -0.8716796636581421,
      "logps/chosen": -0.838574230670929,
      "logps/rejected": -1.326757788658142,
      "loss": 1.1924,
      "nll_loss": 1.147070288658142,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.08378906548023224,
      "rewards/margins": 0.04883880540728569,
      "rewards/rejected": -0.13266602158546448,
      "step": 450
    },
    {
      "epoch": 0.05936249838688863,
      "grad_norm": 1.475600823986248,
      "learning_rate": 3.730019232961255e-06,
      "log_odds_chosen": 0.790637195110321,
      "log_odds_ratio": -0.52099609375,
      "logits/chosen": -0.928906261920929,
      "logits/rejected": -0.8515625,
      "logps/chosen": -0.76953125,
      "logps/rejected": -1.332421898841858,
      "loss": 1.2237,
      "nll_loss": 1.1708984375,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07700195163488388,
      "rewards/margins": 0.0562744140625,
      "rewards/rejected": -0.13334961235523224,
      "step": 460
    },
    {
      "epoch": 0.060652987482255775,
      "grad_norm": 1.4843411621666365,
      "learning_rate": 3.6901248321155403e-06,
      "log_odds_chosen": 0.906445324420929,
      "log_odds_ratio": -0.4745117127895355,
      "logits/chosen": -0.9126952886581421,
      "logits/rejected": -0.8218749761581421,
      "logps/chosen": -0.7464843988418579,
      "logps/rejected": -1.3699219226837158,
      "loss": 1.2121,
      "nll_loss": 1.1710937023162842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07465820014476776,
      "rewards/margins": 0.062408447265625,
      "rewards/rejected": -0.13698729872703552,
      "step": 470
    },
    {
      "epoch": 0.06194347657762292,
      "grad_norm": 1.4233455341009487,
      "learning_rate": 3.6514837167011073e-06,
      "log_odds_chosen": 0.859692394733429,
      "log_odds_ratio": -0.506152331829071,
      "logits/chosen": -0.949023425579071,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.779101550579071,
      "logps/rejected": -1.3562500476837158,
      "loss": 1.2006,
      "nll_loss": 1.0849609375,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07783202826976776,
      "rewards/margins": 0.05777130275964737,
      "rewards/rejected": -0.135498046875,
      "step": 480
    },
    {
      "epoch": 0.06323396567299007,
      "grad_norm": 1.4724809948849447,
      "learning_rate": 3.6140316116210052e-06,
      "log_odds_chosen": 0.8216797113418579,
      "log_odds_ratio": -0.4976562559604645,
      "logits/chosen": -0.9742187261581421,
      "logits/rejected": -0.882617175579071,
      "logps/chosen": -0.74609375,
      "logps/rejected": -1.326171875,
      "loss": 1.1938,
      "nll_loss": 1.125390648841858,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07463379204273224,
      "rewards/margins": 0.05808410793542862,
      "rewards/rejected": -0.13271483778953552,
      "step": 490
    },
    {
      "epoch": 0.0645244547683572,
      "grad_norm": 2.5332769497429672,
      "learning_rate": 3.5777087639996634e-06,
      "log_odds_chosen": 0.86865234375,
      "log_odds_ratio": -0.502880871295929,
      "logits/chosen": -0.954882800579071,
      "logits/rejected": -0.8646484613418579,
      "logps/chosen": -0.77734375,
      "logps/rejected": -1.411523461341858,
      "loss": 1.2081,
      "nll_loss": 1.123046875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0777587890625,
      "rewards/margins": 0.06325836479663849,
      "rewards/rejected": -0.1409912109375,
      "step": 500
    },
    {
      "epoch": 0.06581494386372436,
      "grad_norm": 1.481387574580784,
      "learning_rate": 3.5424595421603814e-06,
      "log_odds_chosen": 0.82666015625,
      "log_odds_ratio": -0.49677735567092896,
      "logits/chosen": -0.9525390863418579,
      "logits/rejected": -0.8646484613418579,
      "logps/chosen": -0.7689453363418579,
      "logps/rejected": -1.3429687023162842,
      "loss": 1.1859,
      "nll_loss": 1.153710961341858,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07686767727136612,
      "rewards/margins": 0.05735168606042862,
      "rewards/rejected": -0.13432617485523224,
      "step": 510
    },
    {
      "epoch": 0.0671054329590915,
      "grad_norm": 1.5802122059801613,
      "learning_rate": 3.5082320772281165e-06,
      "log_odds_chosen": 0.730725109577179,
      "log_odds_ratio": -0.5574706792831421,
      "logits/chosen": -0.9955078363418579,
      "logits/rejected": -0.869140625,
      "logps/chosen": -0.804492175579071,
      "logps/rejected": -1.3126952648162842,
      "loss": 1.2016,
      "nll_loss": 1.167382836341858,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0804443359375,
      "rewards/margins": 0.05084533616900444,
      "rewards/rejected": -0.13124999403953552,
      "step": 520
    },
    {
      "epoch": 0.06839592205445864,
      "grad_norm": 2.4797024617102554,
      "learning_rate": 3.474977942104555e-06,
      "log_odds_chosen": 0.801440417766571,
      "log_odds_ratio": -0.5602051019668579,
      "logits/chosen": -0.9638671875,
      "logits/rejected": -0.8701171875,
      "logps/chosen": -0.8734375238418579,
      "logps/rejected": -1.441796898841858,
      "loss": 1.1788,
      "nll_loss": 1.203125,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.08737792819738388,
      "rewards/margins": 0.05670928955078125,
      "rewards/rejected": -0.14404296875,
      "step": 530
    },
    {
      "epoch": 0.06968641114982578,
      "grad_norm": 1.5526397393405487,
      "learning_rate": 3.442651863295481e-06,
      "log_odds_chosen": 0.7856689691543579,
      "log_odds_ratio": -0.5472167730331421,
      "logits/chosen": -0.936328113079071,
      "logits/rejected": -0.8238281011581421,
      "logps/chosen": -0.7630859613418579,
      "logps/rejected": -1.336523413658142,
      "loss": 1.1852,
      "nll_loss": 1.110742211341858,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07625732570886612,
      "rewards/margins": 0.0573272705078125,
      "rewards/rejected": -0.1336669921875,
      "step": 540
    },
    {
      "epoch": 0.07097690024519293,
      "grad_norm": 1.4668926499094914,
      "learning_rate": 3.4112114616897665e-06,
      "log_odds_chosen": 0.7321532964706421,
      "log_odds_ratio": -0.5518554449081421,
      "logits/chosen": -0.9595702886581421,
      "logits/rejected": -0.869140625,
      "logps/chosen": -0.771484375,
      "logps/rejected": -1.2802734375,
      "loss": 1.1917,
      "nll_loss": 1.1121094226837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07706298679113388,
      "rewards/margins": 0.05086059495806694,
      "rewards/rejected": -0.12795409560203552,
      "step": 550
    },
    {
      "epoch": 0.07226738934056007,
      "grad_norm": 1.4658115621754695,
      "learning_rate": 3.3806170189140663e-06,
      "log_odds_chosen": 0.9544922113418579,
      "log_odds_ratio": -0.4916015565395355,
      "logits/chosen": -0.94921875,
      "logits/rejected": -0.8773437738418579,
      "logps/chosen": -0.7767578363418579,
      "logps/rejected": -1.451171875,
      "loss": 1.1762,
      "nll_loss": 1.1593749523162842,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07770995795726776,
      "rewards/margins": 0.06751098483800888,
      "rewards/rejected": -0.14519043266773224,
      "step": 560
    },
    {
      "epoch": 0.07355787843592722,
      "grad_norm": 1.4346845344366215,
      "learning_rate": 3.350831266333564e-06,
      "log_odds_chosen": 0.9032226800918579,
      "log_odds_ratio": -0.5068359375,
      "logits/chosen": -0.9384765625,
      "logits/rejected": -0.840624988079071,
      "logps/chosen": -0.791015625,
      "logps/rejected": -1.46484375,
      "loss": 1.1741,
      "nll_loss": 1.1447265148162842,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07911376655101776,
      "rewards/margins": 0.06731567531824112,
      "rewards/rejected": -0.14643554389476776,
      "step": 570
    },
    {
      "epoch": 0.07484836753129435,
      "grad_norm": 1.6142638363788113,
      "learning_rate": 3.3218191941495984e-06,
      "log_odds_chosen": 0.880322277545929,
      "log_odds_ratio": -0.5274902582168579,
      "logits/chosen": -0.9849609136581421,
      "logits/rejected": -0.9037109613418579,
      "logps/chosen": -0.764843761920929,
      "logps/rejected": -1.3798828125,
      "loss": 1.1695,
      "nll_loss": 1.077539086341858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07646484673023224,
      "rewards/margins": 0.061557769775390625,
      "rewards/rejected": -0.13798828423023224,
      "step": 580
    },
    {
      "epoch": 0.0761388566266615,
      "grad_norm": 1.5610252577367603,
      "learning_rate": 3.293547878370473e-06,
      "log_odds_chosen": 0.767993152141571,
      "log_odds_ratio": -0.5763183832168579,
      "logits/chosen": -0.9603515863418579,
      "logits/rejected": -0.873242199420929,
      "logps/chosen": -0.8189452886581421,
      "logps/rejected": -1.3898437023162842,
      "loss": 1.1805,
      "nll_loss": 1.168359398841858,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.08193359524011612,
      "rewards/margins": 0.05710601806640625,
      "rewards/rejected": -0.13906249403953552,
      "step": 590
    },
    {
      "epoch": 0.07742934572202866,
      "grad_norm": 1.4358796294480831,
      "learning_rate": 3.2659863237109044e-06,
      "log_odds_chosen": 0.864147961139679,
      "log_odds_ratio": -0.49365234375,
      "logits/chosen": -0.9644531011581421,
      "logits/rejected": -0.875,
      "logps/chosen": -0.7476562261581421,
      "logps/rejected": -1.368554711341858,
      "loss": 1.1566,
      "nll_loss": 1.1501953601837158,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07476806640625,
      "rewards/margins": 0.06196746975183487,
      "rewards/rejected": -0.13671875,
      "step": 600
    },
    {
      "epoch": 0.07871983481739579,
      "grad_norm": 1.480161966340777,
      "learning_rate": 3.239105320715664e-06,
      "log_odds_chosen": 0.8868408203125,
      "log_odds_ratio": -0.46772462129592896,
      "logits/chosen": -0.943164050579071,
      "logits/rejected": -0.862500011920929,
      "logps/chosen": -0.7496093511581421,
      "logps/rejected": -1.3410155773162842,
      "loss": 1.1746,
      "nll_loss": 1.095312476158142,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07493896782398224,
      "rewards/margins": 0.05926513671875,
      "rewards/rejected": -0.13422851264476776,
      "step": 610
    },
    {
      "epoch": 0.08001032391276294,
      "grad_norm": 1.6285636478037393,
      "learning_rate": 3.2128773156099956e-06,
      "log_odds_chosen": 1.0142090320587158,
      "log_odds_ratio": -0.48369139432907104,
      "logits/chosen": -1.0222656726837158,
      "logits/rejected": -0.9375,
      "logps/chosen": -0.8080078363418579,
      "logps/rejected": -1.543359398841858,
      "loss": 1.197,
      "nll_loss": 1.1287109851837158,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.08082275092601776,
      "rewards/margins": 0.07362060248851776,
      "rewards/rejected": -0.15432128310203552,
      "step": 620
    },
    {
      "epoch": 0.08130081300813008,
      "grad_norm": 1.6545609934781655,
      "learning_rate": 3.187276291558383e-06,
      "log_odds_chosen": 0.9271484613418579,
      "log_odds_ratio": -0.49311524629592896,
      "logits/chosen": -0.9488281011581421,
      "logits/rejected": -0.877148449420929,
      "logps/chosen": -0.740429699420929,
      "logps/rejected": -1.350976586341858,
      "loss": 1.1582,
      "nll_loss": 1.16796875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07402344048023224,
      "rewards/margins": 0.06111602857708931,
      "rewards/rejected": -0.1351318359375,
      "step": 630
    },
    {
      "epoch": 0.08259130210349723,
      "grad_norm": 1.4518076811964788,
      "learning_rate": 3.1622776601683788e-06,
      "log_odds_chosen": 1.045996069908142,
      "log_odds_ratio": -0.4473632872104645,
      "logits/chosen": -1.0007812976837158,
      "logits/rejected": -0.8765624761581421,
      "logps/chosen": -0.774609386920929,
      "logps/rejected": -1.4777343273162842,
      "loss": 1.1388,
      "nll_loss": 1.081640601158142,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07741699367761612,
      "rewards/margins": 0.07030029594898224,
      "rewards/rejected": -0.14780274033546448,
      "step": 640
    },
    {
      "epoch": 0.08388179119886437,
      "grad_norm": 1.4547591472011996,
      "learning_rate": 3.1378581622109444e-06,
      "log_odds_chosen": 1.095483422279358,
      "log_odds_ratio": -0.45356446504592896,
      "logits/chosen": -1.013281226158142,
      "logits/rejected": -0.902539074420929,
      "logps/chosen": -0.7333984375,
      "logps/rejected": -1.5070312023162842,
      "loss": 1.1736,
      "nll_loss": 1.0656249523162842,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0733642578125,
      "rewards/margins": 0.0772705078125,
      "rewards/rejected": -0.15061035752296448,
      "step": 650
    },
    {
      "epoch": 0.08517228029423152,
      "grad_norm": 1.7038830348578193,
      "learning_rate": 3.113995776646092e-06,
      "log_odds_chosen": 0.95263671875,
      "log_odds_ratio": -0.4935546815395355,
      "logits/chosen": -1.0138671398162842,
      "logits/rejected": -0.903515636920929,
      "logps/chosen": -0.741015613079071,
      "logps/rejected": -1.3914062976837158,
      "loss": 1.16,
      "nll_loss": 1.15625,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07408447563648224,
      "rewards/margins": 0.0650634765625,
      "rewards/rejected": -0.13916015625,
      "step": 660
    },
    {
      "epoch": 0.08646276938959865,
      "grad_norm": 1.486654711429888,
      "learning_rate": 3.090669637145023e-06,
      "log_odds_chosen": 0.748974621295929,
      "log_odds_ratio": -0.540722668170929,
      "logits/chosen": -0.991406261920929,
      "logits/rejected": -0.8792968988418579,
      "logps/chosen": -0.7710937261581421,
      "logps/rejected": -1.301171898841858,
      "loss": 1.158,
      "nll_loss": 1.055273413658142,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07709960639476776,
      "rewards/margins": 0.05306396633386612,
      "rewards/rejected": -0.13015136122703552,
      "step": 670
    },
    {
      "epoch": 0.0877532584849658,
      "grad_norm": 1.6760372926415095,
      "learning_rate": 3.0678599553894814e-06,
      "log_odds_chosen": 0.93798828125,
      "log_odds_ratio": -0.45393067598342896,
      "logits/chosen": -0.9525390863418579,
      "logits/rejected": -0.833789050579071,
      "logps/chosen": -0.751171886920929,
      "logps/rejected": -1.374414086341858,
      "loss": 1.1514,
      "nll_loss": 1.1261718273162842,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07514648139476776,
      "rewards/margins": 0.06225891038775444,
      "rewards/rejected": -0.13737793266773224,
      "step": 680
    },
    {
      "epoch": 0.08904374758033294,
      "grad_norm": 1.6617341343815,
      "learning_rate": 3.0455479505075235e-06,
      "log_odds_chosen": 0.9442383050918579,
      "log_odds_ratio": -0.47607421875,
      "logits/chosen": -1.0037109851837158,
      "logits/rejected": -0.9164062738418579,
      "logps/chosen": -0.7650390863418579,
      "logps/rejected": -1.42578125,
      "loss": 1.1652,
      "nll_loss": 1.097070336341858,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07650146633386612,
      "rewards/margins": 0.06599731743335724,
      "rewards/rejected": -0.1424560546875,
      "step": 690
    },
    {
      "epoch": 0.09033423667570009,
      "grad_norm": 1.4630879034217015,
      "learning_rate": 3.0237157840738173e-06,
      "log_odds_chosen": 0.9931396245956421,
      "log_odds_ratio": -0.4849609434604645,
      "logits/chosen": -1.0324218273162842,
      "logits/rejected": -0.9150390625,
      "logps/chosen": -0.786914050579071,
      "logps/rejected": -1.4796874523162842,
      "loss": 1.1437,
      "nll_loss": 1.105078101158142,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07860107719898224,
      "rewards/margins": 0.06933288276195526,
      "rewards/rejected": -0.1480712890625,
      "step": 700
    },
    {
      "epoch": 0.09162472577106723,
      "grad_norm": 1.4470973522078117,
      "learning_rate": 3.002346500163206e-06,
      "log_odds_chosen": 1.23388671875,
      "log_odds_ratio": -0.3971191346645355,
      "logits/chosen": -1.0486328601837158,
      "logits/rejected": -0.985546886920929,
      "logps/chosen": -0.725390613079071,
      "logps/rejected": -1.591406226158142,
      "loss": 1.1588,
      "nll_loss": 1.0691406726837158,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.07257080078125,
      "rewards/margins": 0.08654174953699112,
      "rewards/rejected": -0.15908202528953552,
      "step": 710
    },
    {
      "epoch": 0.09291521486643438,
      "grad_norm": 1.4200065338833956,
      "learning_rate": 2.9814239699997195e-06,
      "log_odds_chosen": 1.0605590343475342,
      "log_odds_ratio": -0.4505859315395355,
      "logits/chosen": -1.109765648841858,
      "logits/rejected": -1.027734398841858,
      "logps/chosen": -0.732226550579071,
      "logps/rejected": -1.4503905773162842,
      "loss": 1.1422,
      "nll_loss": 1.079687476158142,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07325439155101776,
      "rewards/margins": 0.07174072414636612,
      "rewards/rejected": -0.14497070014476776,
      "step": 720
    },
    {
      "epoch": 0.09420570396180153,
      "grad_norm": 1.5980794730706656,
      "learning_rate": 2.9609328407904207e-06,
      "log_odds_chosen": 0.692456066608429,
      "log_odds_ratio": -0.5464843511581421,
      "logits/chosen": -1.062890648841858,
      "logits/rejected": -0.9976562261581421,
      "logps/chosen": -0.7466796636581421,
      "logps/rejected": -1.2275390625,
      "loss": 1.1336,
      "nll_loss": 1.080468773841858,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07459716498851776,
      "rewards/margins": 0.04819030687212944,
      "rewards/rejected": -0.12285156548023224,
      "step": 730
    },
    {
      "epoch": 0.09549619305716867,
      "grad_norm": 1.7207335380048365,
      "learning_rate": 2.940858488375231e-06,
      "log_odds_chosen": 0.710742175579071,
      "log_odds_ratio": -0.540234386920929,
      "logits/chosen": -1.0068359375,
      "logits/rejected": -0.953906238079071,
      "logps/chosen": -0.7542968988418579,
      "logps/rejected": -1.257226586341858,
      "loss": 1.1286,
      "nll_loss": 1.063085913658142,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.075439453125,
      "rewards/margins": 0.05036010593175888,
      "rewards/rejected": -0.12583008408546448,
      "step": 740
    },
    {
      "epoch": 0.09678668215253582,
      "grad_norm": 1.6424125036882053,
      "learning_rate": 2.9211869733608857e-06,
      "log_odds_chosen": 0.831005871295929,
      "log_odds_ratio": -0.5052245855331421,
      "logits/chosen": -0.9873046875,
      "logits/rejected": -0.908203125,
      "logps/chosen": -0.7880859375,
      "logps/rejected": -1.322265625,
      "loss": 1.1597,
      "nll_loss": 1.140234351158142,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07877197116613388,
      "rewards/margins": 0.05340881273150444,
      "rewards/rejected": -0.1322021484375,
      "step": 750
    },
    {
      "epoch": 0.09807717124790295,
      "grad_norm": 1.5258866173338657,
      "learning_rate": 2.901905000440047e-06,
      "log_odds_chosen": 0.87939453125,
      "log_odds_ratio": -0.4961914122104645,
      "logits/chosen": -0.987500011920929,
      "logits/rejected": -0.8941406011581421,
      "logps/chosen": -0.767578125,
      "logps/rejected": -1.3738281726837158,
      "loss": 1.1391,
      "nll_loss": 1.0646483898162842,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07670898735523224,
      "rewards/margins": 0.06062011793255806,
      "rewards/rejected": -0.13742676377296448,
      "step": 760
    },
    {
      "epoch": 0.0993676603432701,
      "grad_norm": 1.5625298989181904,
      "learning_rate": 2.8829998806257885e-06,
      "log_odds_chosen": 1.093115210533142,
      "log_odds_ratio": -0.4473632872104645,
      "logits/chosen": -1.0302734375,
      "logits/rejected": -0.9384765625,
      "logps/chosen": -0.7699218988418579,
      "logps/rejected": -1.5734374523162842,
      "loss": 1.1452,
      "nll_loss": 1.123046875,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07703857123851776,
      "rewards/margins": 0.08021850883960724,
      "rewards/rejected": -0.15725097060203552,
      "step": 770
    },
    {
      "epoch": 0.10065814943863724,
      "grad_norm": 1.7154687344647153,
      "learning_rate": 2.8644594961577314e-06,
      "log_odds_chosen": 0.8791869878768921,
      "log_odds_ratio": -0.49628907442092896,
      "logits/chosen": -1.041406273841858,
      "logits/rejected": -0.962109386920929,
      "logps/chosen": -0.743359386920929,
      "logps/rejected": -1.344335913658142,
      "loss": 1.1802,
      "nll_loss": 1.1857421398162842,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07429198920726776,
      "rewards/margins": 0.06023254245519638,
      "rewards/rejected": -0.13454589247703552,
      "step": 780
    },
    {
      "epoch": 0.10194863853400439,
      "grad_norm": 1.531836559028637,
      "learning_rate": 2.84627226785928e-06,
      "log_odds_chosen": 0.911425769329071,
      "log_odds_ratio": -0.47539061307907104,
      "logits/chosen": -1.0353515148162842,
      "logits/rejected": -0.939257800579071,
      "logps/chosen": -0.722949206829071,
      "logps/rejected": -1.3044922351837158,
      "loss": 1.158,
      "nll_loss": 1.1261718273162842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07227782905101776,
      "rewards/margins": 0.0582275390625,
      "rewards/rejected": -0.1304931640625,
      "step": 790
    },
    {
      "epoch": 0.10323912762937153,
      "grad_norm": 1.6744841844286158,
      "learning_rate": 2.82842712474619e-06,
      "log_odds_chosen": 1.2077147960662842,
      "log_odds_ratio": -0.4222656190395355,
      "logits/chosen": -1.0773437023162842,
      "logits/rejected": -0.9906250238418579,
      "logps/chosen": -0.721484363079071,
      "logps/rejected": -1.562890648841858,
      "loss": 1.1284,
      "nll_loss": 1.058203101158142,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0721435546875,
      "rewards/margins": 0.0842437744140625,
      "rewards/rejected": -0.15644530951976776,
      "step": 800
    },
    {
      "epoch": 0.10452961672473868,
      "grad_norm": 1.7616114353890147,
      "learning_rate": 2.810913475705226e-06,
      "log_odds_chosen": 1.028710961341858,
      "log_odds_ratio": -0.46235352754592896,
      "logits/chosen": -1.04296875,
      "logits/rejected": -0.955859363079071,
      "logps/chosen": -0.7494140863418579,
      "logps/rejected": -1.4855468273162842,
      "loss": 1.1538,
      "nll_loss": 1.0769531726837158,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07490234076976776,
      "rewards/margins": 0.073699951171875,
      "rewards/rejected": -0.148681640625,
      "step": 810
    },
    {
      "epoch": 0.10582010582010581,
      "grad_norm": 1.545585643820695,
      "learning_rate": 2.7937211830783128e-06,
      "log_odds_chosen": 1.189843773841858,
      "log_odds_ratio": -0.41508787870407104,
      "logits/chosen": -1.001367211341858,
      "logits/rejected": -0.9009765386581421,
      "logps/chosen": -0.685351550579071,
      "logps/rejected": -1.5017578601837158,
      "loss": 1.1299,
      "nll_loss": 1.0158202648162842,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06853027641773224,
      "rewards/margins": 0.08150329440832138,
      "rewards/rejected": -0.15009765326976776,
      "step": 820
    },
    {
      "epoch": 0.10711059491547296,
      "grad_norm": 1.5550866908049985,
      "learning_rate": 2.776840538002493e-06,
      "log_odds_chosen": 1.126123070716858,
      "log_odds_ratio": -0.44648438692092896,
      "logits/chosen": -1.0517578125,
      "logits/rejected": -0.980664074420929,
      "logps/chosen": -0.75,
      "logps/rejected": -1.559179663658142,
      "loss": 1.1321,
      "nll_loss": 1.0859375,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07501220703125,
      "rewards/margins": 0.08095703274011612,
      "rewards/rejected": -0.156005859375,
      "step": 830
    },
    {
      "epoch": 0.10840108401084012,
      "grad_norm": 1.5210116552557222,
      "learning_rate": 2.7602622373694163e-06,
      "log_odds_chosen": 1.1129882335662842,
      "log_odds_ratio": -0.43413084745407104,
      "logits/chosen": -1.0857422351837158,
      "logits/rejected": -1.0021483898162842,
      "logps/chosen": -0.7275390625,
      "logps/rejected": -1.5146484375,
      "loss": 1.1458,
      "nll_loss": 1.1027343273162842,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07272949069738388,
      "rewards/margins": 0.07881470024585724,
      "rewards/rejected": -0.151611328125,
      "step": 840
    },
    {
      "epoch": 0.10969157310620725,
      "grad_norm": 1.768088107797017,
      "learning_rate": 2.743977362280141e-06,
      "log_odds_chosen": 0.885424792766571,
      "log_odds_ratio": -0.504687488079071,
      "logits/chosen": -1.1222655773162842,
      "logits/rejected": -1.032617211341858,
      "logps/chosen": -0.775195300579071,
      "logps/rejected": -1.421875,
      "loss": 1.1348,
      "nll_loss": 1.0701172351837158,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07746581733226776,
      "rewards/margins": 0.064666748046875,
      "rewards/rejected": -0.14211425185203552,
      "step": 850
    },
    {
      "epoch": 0.1109820622015744,
      "grad_norm": 1.5985370415036835,
      "learning_rate": 2.7279773578818937e-06,
      "log_odds_chosen": 1.095312476158142,
      "log_odds_ratio": -0.4722656309604645,
      "logits/chosen": -1.085546851158142,
      "logits/rejected": -0.983203113079071,
      "logps/chosen": -0.7933593988418579,
      "logps/rejected": -1.5789062976837158,
      "loss": 1.1329,
      "nll_loss": 1.099023461341858,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07938232272863388,
      "rewards/margins": 0.07849731296300888,
      "rewards/rejected": -0.15778808295726776,
      "step": 860
    },
    {
      "epoch": 0.11227255129694154,
      "grad_norm": 1.7249370408807103,
      "learning_rate": 2.7122540144832417e-06,
      "log_odds_chosen": 1.007421851158142,
      "log_odds_ratio": -0.5185546875,
      "logits/chosen": -1.063867211341858,
      "logits/rejected": -0.97265625,
      "logps/chosen": -0.7749999761581421,
      "logps/rejected": -1.4921875,
      "loss": 1.1126,
      "nll_loss": 1.0994141101837158,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07746581733226776,
      "rewards/margins": 0.07174377143383026,
      "rewards/rejected": -0.1492919921875,
      "step": 870
    },
    {
      "epoch": 0.11356304039230869,
      "grad_norm": 1.555632738471784,
      "learning_rate": 2.696799449852968e-06,
      "log_odds_chosen": 0.95458984375,
      "log_odds_ratio": -0.4662109315395355,
      "logits/chosen": -1.0226562023162842,
      "logits/rejected": -0.9574218988418579,
      "logps/chosen": -0.721972644329071,
      "logps/rejected": -1.3582031726837158,
      "loss": 1.1364,
      "nll_loss": 1.0837891101837158,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07220458984375,
      "rewards/margins": 0.06359253078699112,
      "rewards/rejected": -0.13576659560203552,
      "step": 880
    },
    {
      "epoch": 0.11485352948767583,
      "grad_norm": 1.6198246488260135,
      "learning_rate": 2.6816060926159636e-06,
      "log_odds_chosen": 0.923535168170929,
      "log_odds_ratio": -0.48417967557907104,
      "logits/chosen": -1.0226562023162842,
      "logits/rejected": -0.92578125,
      "logps/chosen": -0.754687488079071,
      "logps/rejected": -1.392968773841858,
      "loss": 1.1351,
      "nll_loss": 1.17578125,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07548828423023224,
      "rewards/margins": 0.063812255859375,
      "rewards/rejected": -0.13933105766773224,
      "step": 890
    },
    {
      "epoch": 0.11614401858304298,
      "grad_norm": 1.661192150882734,
      "learning_rate": 2.6666666666666664e-06,
      "log_odds_chosen": 0.8763672113418579,
      "log_odds_ratio": -0.531542956829071,
      "logits/chosen": -1.020117163658142,
      "logits/rejected": -0.9410156011581421,
      "logps/chosen": -0.763867199420929,
      "logps/rejected": -1.3660156726837158,
      "loss": 1.1142,
      "nll_loss": 1.0427734851837158,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07642821967601776,
      "rewards/margins": 0.06026611477136612,
      "rewards/rejected": -0.13671875,
      "step": 900
    },
    {
      "epoch": 0.11743450767841011,
      "grad_norm": 1.5883789972327746,
      "learning_rate": 2.6519741765271837e-06,
      "log_odds_chosen": 0.9326171875,
      "log_odds_ratio": -0.489990234375,
      "logits/chosen": -1.0373046398162842,
      "logits/rejected": -0.9457031488418579,
      "logps/chosen": -0.7191406488418579,
      "logps/rejected": -1.34765625,
      "loss": 1.1114,
      "nll_loss": 1.090429663658142,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07185058295726776,
      "rewards/margins": 0.06281737983226776,
      "rewards/rejected": -0.13474121689796448,
      "step": 910
    },
    {
      "epoch": 0.11872499677377726,
      "grad_norm": 1.731644337613153,
      "learning_rate": 2.637521893583148e-06,
      "log_odds_chosen": 0.9296875,
      "log_odds_ratio": -0.50927734375,
      "logits/chosen": -1.0564453601837158,
      "logits/rejected": -0.991406261920929,
      "logps/chosen": -0.6996093988418579,
      "logps/rejected": -1.3390624523162842,
      "loss": 1.1127,
      "nll_loss": 1.0177733898162842,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0699462890625,
      "rewards/margins": 0.0639495849609375,
      "rewards/rejected": -0.13395996391773224,
      "step": 920
    },
    {
      "epoch": 0.1200154858691444,
      "grad_norm": 1.6481604602312732,
      "learning_rate": 2.6233033431358115e-06,
      "log_odds_chosen": 1.1953125,
      "log_odds_ratio": -0.43950194120407104,
      "logits/chosen": -1.058203101158142,
      "logits/rejected": -0.94140625,
      "logps/chosen": -0.7158203125,
      "logps/rejected": -1.537109375,
      "loss": 1.1128,
      "nll_loss": 0.9898437261581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07154540717601776,
      "rewards/margins": 0.08209228515625,
      "rewards/rejected": -0.15375976264476776,
      "step": 930
    },
    {
      "epoch": 0.12130597496451155,
      "grad_norm": 1.5901608390173674,
      "learning_rate": 2.6093122922137685e-06,
      "log_odds_chosen": 1.112829566001892,
      "log_odds_ratio": -0.43095701932907104,
      "logits/chosen": -1.0457031726837158,
      "logits/rejected": -0.9232422113418579,
      "logps/chosen": -0.6998046636581421,
      "logps/rejected": -1.4552733898162842,
      "loss": 1.1251,
      "nll_loss": 1.0,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06989745795726776,
      "rewards/margins": 0.07556457817554474,
      "rewards/rejected": -0.14548340439796448,
      "step": 940
    },
    {
      "epoch": 0.12259646405987869,
      "grad_norm": 1.767880469701478,
      "learning_rate": 2.5955427380922006e-06,
      "log_odds_chosen": 1.1389648914337158,
      "log_odds_ratio": -0.420166015625,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.9046875238418579,
      "logps/chosen": -0.6761718988418579,
      "logps/rejected": -1.44921875,
      "loss": 1.1072,
      "nll_loss": 1.019921898841858,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06768798828125,
      "rewards/margins": 0.07720641791820526,
      "rewards/rejected": -0.14492186903953552,
      "step": 950
    },
    {
      "epoch": 0.12388695315524584,
      "grad_norm": 1.5338120757900684,
      "learning_rate": 2.5819888974716113e-06,
      "log_odds_chosen": 1.094335913658142,
      "log_odds_ratio": -0.4586425721645355,
      "logits/chosen": -1.016015648841858,
      "logits/rejected": -0.907031238079071,
      "logps/chosen": -0.7789062261581421,
      "logps/rejected": -1.5568358898162842,
      "loss": 1.1138,
      "nll_loss": 1.0832030773162842,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07795409858226776,
      "rewards/margins": 0.07776489108800888,
      "rewards/rejected": -0.15568847954273224,
      "step": 960
    },
    {
      "epoch": 0.12517744225061297,
      "grad_norm": 1.5037369828396514,
      "learning_rate": 2.5686451962717425e-06,
      "log_odds_chosen": 1.14697265625,
      "log_odds_ratio": -0.43242186307907104,
      "logits/chosen": -1.0408203601837158,
      "logits/rejected": -0.9341796636581421,
      "logps/chosen": -0.7105468511581421,
      "logps/rejected": -1.521484375,
      "loss": 1.1447,
      "nll_loss": 1.071679711341858,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07105712592601776,
      "rewards/margins": 0.08092651516199112,
      "rewards/rejected": -0.15205077826976776,
      "step": 970
    },
    {
      "epoch": 0.12646793134598014,
      "grad_norm": 1.6024807805584391,
      "learning_rate": 2.5555062599997596e-06,
      "log_odds_chosen": 1.3054687976837158,
      "log_odds_ratio": -0.3855957090854645,
      "logits/chosen": -1.0400390625,
      "logits/rejected": -0.9146484136581421,
      "logps/chosen": -0.73388671875,
      "logps/rejected": -1.658593773841858,
      "loss": 1.0907,
      "nll_loss": 1.084570288658142,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07337646186351776,
      "rewards/margins": 0.09254150092601776,
      "rewards/rejected": -0.16591796278953552,
      "step": 980
    },
    {
      "epoch": 0.12775842044134728,
      "grad_norm": 1.8114580329142012,
      "learning_rate": 2.5425669046549126e-06,
      "log_odds_chosen": 0.9522460699081421,
      "log_odds_ratio": -0.531445324420929,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.937304675579071,
      "logps/chosen": -0.7718750238418579,
      "logps/rejected": -1.4572265148162842,
      "loss": 1.1036,
      "nll_loss": 1.068945288658142,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.0771484375,
      "rewards/margins": 0.06858520209789276,
      "rewards/rejected": -0.14580078423023224,
      "step": 990
    },
    {
      "epoch": 0.1290489095367144,
      "grad_norm": 1.597110549231449,
      "learning_rate": 2.5298221281347034e-06,
      "log_odds_chosen": 1.1908447742462158,
      "log_odds_ratio": -0.4569335877895355,
      "logits/chosen": -1.0105469226837158,
      "logits/rejected": -0.8958984613418579,
      "logps/chosen": -0.72607421875,
      "logps/rejected": -1.547265648841858,
      "loss": 1.0939,
      "nll_loss": 1.0656249523162842,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0726318359375,
      "rewards/margins": 0.08215637505054474,
      "rewards/rejected": -0.15471191704273224,
      "step": 1000
    },
    {
      "epoch": 0.13033939863208155,
      "grad_norm": 1.8135077350243465,
      "learning_rate": 2.5172671021102103e-06,
      "log_odds_chosen": 0.970947265625,
      "log_odds_ratio": -0.46796876192092896,
      "logits/chosen": -1.0642578601837158,
      "logits/rejected": -0.9619140625,
      "logps/chosen": -0.7242187261581421,
      "logps/rejected": -1.3816406726837158,
      "loss": 1.1012,
      "nll_loss": 1.0568358898162842,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07248535007238388,
      "rewards/margins": 0.06572265923023224,
      "rewards/rejected": -0.13808593153953552,
      "step": 1010
    },
    {
      "epoch": 0.1316298877274487,
      "grad_norm": 1.622748978393355,
      "learning_rate": 2.504897164340598e-06,
      "log_odds_chosen": 1.115136742591858,
      "log_odds_ratio": -0.46484375,
      "logits/chosen": -1.0300781726837158,
      "logits/rejected": -0.9244140386581421,
      "logps/chosen": -0.7562500238418579,
      "logps/rejected": -1.5205078125,
      "loss": 1.1071,
      "nll_loss": 1.0701172351837158,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07562255859375,
      "rewards/margins": 0.07652588188648224,
      "rewards/rejected": -0.152099609375,
      "step": 1020
    },
    {
      "epoch": 0.13292037682281585,
      "grad_norm": 1.7832072132435153,
      "learning_rate": 2.492707811399023e-06,
      "log_odds_chosen": 1.091528296470642,
      "log_odds_ratio": -0.5047363042831421,
      "logits/chosen": -1.018945336341858,
      "logits/rejected": -0.937695324420929,
      "logps/chosen": -0.7857421636581421,
      "logps/rejected": -1.5890624523162842,
      "loss": 1.1328,
      "nll_loss": 1.139257788658142,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07850341498851776,
      "rewards/margins": 0.08042602241039276,
      "rewards/rejected": -0.15888671576976776,
      "step": 1030
    },
    {
      "epoch": 0.134210865918183,
      "grad_norm": 1.7205279311344388,
      "learning_rate": 2.480694691784169e-06,
      "log_odds_chosen": 1.224829077720642,
      "log_odds_ratio": -0.42915040254592896,
      "logits/chosen": -0.9974609613418579,
      "logits/rejected": -0.9195312261581421,
      "logps/chosen": -0.7349609136581421,
      "logps/rejected": -1.5822265148162842,
      "loss": 1.0878,
      "nll_loss": 1.046875,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07353515923023224,
      "rewards/margins": 0.08487243950366974,
      "rewards/rejected": -0.15839843451976776,
      "step": 1040
    },
    {
      "epoch": 0.13550135501355012,
      "grad_norm": 1.7635253076010384,
      "learning_rate": 2.4688535993934706e-06,
      "log_odds_chosen": 0.7667602300643921,
      "log_odds_ratio": -0.5552734136581421,
      "logits/chosen": -1.020898461341858,
      "logits/rejected": -0.9417968988418579,
      "logps/chosen": -0.7757812738418579,
      "logps/rejected": -1.297460913658142,
      "loss": 1.1424,
      "nll_loss": 1.14453125,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07761230319738388,
      "rewards/margins": 0.05221252515912056,
      "rewards/rejected": -0.12983398139476776,
      "step": 1050
    },
    {
      "epoch": 0.1367918441089173,
      "grad_norm": 1.6202241462062774,
      "learning_rate": 2.457180467335805e-06,
      "log_odds_chosen": 1.074951171875,
      "log_odds_ratio": -0.49589842557907104,
      "logits/chosen": -1.0556640625,
      "logits/rejected": -0.9732421636581421,
      "logps/chosen": -0.772265613079071,
      "logps/rejected": -1.575781226158142,
      "loss": 1.1023,
      "nll_loss": 1.068359375,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07719726860523224,
      "rewards/margins": 0.08048705756664276,
      "rewards/rejected": -0.15754394233226776,
      "step": 1060
    },
    {
      "epoch": 0.13808233320428442,
      "grad_norm": 1.5675096517669997,
      "learning_rate": 2.4456713620629725e-06,
      "log_odds_chosen": 1.06982421875,
      "log_odds_ratio": -0.44111329317092896,
      "logits/chosen": -1.0451171398162842,
      "logits/rejected": -0.955273449420929,
      "logps/chosen": -0.7392578125,
      "logps/rejected": -1.4484374523162842,
      "loss": 1.1237,
      "nll_loss": 1.0886719226837158,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07396240532398224,
      "rewards/margins": 0.07095947116613388,
      "rewards/rejected": -0.14494629204273224,
      "step": 1070
    },
    {
      "epoch": 0.13937282229965156,
      "grad_norm": 1.561658398090954,
      "learning_rate": 2.4343224778007378e-06,
      "log_odds_chosen": 1.064355492591858,
      "log_odds_ratio": -0.4766601622104645,
      "logits/chosen": -1.0966796875,
      "logits/rejected": -1.022070288658142,
      "logps/chosen": -0.7601562738418579,
      "logps/rejected": -1.5324218273162842,
      "loss": 1.1034,
      "nll_loss": 1.067773461341858,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07603760063648224,
      "rewards/margins": 0.07728271186351776,
      "rewards/rejected": -0.15324707329273224,
      "step": 1080
    },
    {
      "epoch": 0.14066331139501873,
      "grad_norm": 1.634800621120358,
      "learning_rate": 2.4231301312615306e-06,
      "log_odds_chosen": 1.0540039539337158,
      "log_odds_ratio": -0.4715942442417145,
      "logits/chosen": -1.091406226158142,
      "logits/rejected": -0.966601550579071,
      "logps/chosen": -0.761523425579071,
      "logps/rejected": -1.4992187023162842,
      "loss": 1.0938,
      "nll_loss": 1.0408203601837158,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.076171875,
      "rewards/margins": 0.07371826469898224,
      "rewards/rejected": -0.14995117485523224,
      "step": 1090
    },
    {
      "epoch": 0.14195380049038586,
      "grad_norm": 1.578021190122169,
      "learning_rate": 2.412090756622109e-06,
      "log_odds_chosen": 0.999072253704071,
      "log_odds_ratio": -0.4278320372104645,
      "logits/chosen": -1.0849609375,
      "logits/rejected": -1.0126953125,
      "logps/chosen": -0.7236328125,
      "logps/rejected": -1.354882836341858,
      "loss": 1.1109,
      "nll_loss": 1.025781273841858,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07236327975988388,
      "rewards/margins": 0.06315002590417862,
      "rewards/rejected": -0.13544921576976776,
      "step": 1100
    },
    {
      "epoch": 0.143244289585753,
      "grad_norm": 1.6302990854173744,
      "learning_rate": 2.401200900750657e-06,
      "log_odds_chosen": 1.1398437023162842,
      "log_odds_ratio": -0.4596191346645355,
      "logits/chosen": -1.0654296875,
      "logits/rejected": -0.945117175579071,
      "logps/chosen": -0.751953125,
      "logps/rejected": -1.5740234851837158,
      "loss": 1.1006,
      "nll_loss": 1.0300781726837158,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07514648139476776,
      "rewards/margins": 0.0821990966796875,
      "rewards/rejected": -0.157470703125,
      "step": 1110
    },
    {
      "epoch": 0.14453477868112014,
      "grad_norm": 1.8149470491956978,
      "learning_rate": 2.390457218668787e-06,
      "log_odds_chosen": 1.140679955482483,
      "log_odds_ratio": -0.4345703125,
      "logits/chosen": -1.037695288658142,
      "logits/rejected": -0.9345703125,
      "logps/chosen": -0.7427734136581421,
      "logps/rejected": -1.5255858898162842,
      "loss": 1.0955,
      "nll_loss": 1.064453125,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07424316555261612,
      "rewards/margins": 0.07821045070886612,
      "rewards/rejected": -0.152587890625,
      "step": 1120
    },
    {
      "epoch": 0.1458252677764873,
      "grad_norm": 1.7510726371956102,
      "learning_rate": 2.379856469234918e-06,
      "log_odds_chosen": 1.1335937976837158,
      "log_odds_ratio": -0.4415039122104645,
      "logits/chosen": -1.075585961341858,
      "logits/rejected": -0.984570324420929,
      "logps/chosen": -0.6875,
      "logps/rejected": -1.446874976158142,
      "loss": 1.0648,
      "nll_loss": 1.057226538658142,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06878662109375,
      "rewards/margins": 0.07589111477136612,
      "rewards/rejected": -0.14470215141773224,
      "step": 1130
    },
    {
      "epoch": 0.14711575687185444,
      "grad_norm": 1.6918038305265324,
      "learning_rate": 2.369395511036369e-06,
      "log_odds_chosen": 0.850048840045929,
      "log_odds_ratio": -0.5113769769668579,
      "logits/chosen": -1.078710913658142,
      "logits/rejected": -0.969531238079071,
      "logps/chosen": -0.724609375,
      "logps/rejected": -1.310546875,
      "loss": 1.0958,
      "nll_loss": 1.061914086341858,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07241211086511612,
      "rewards/margins": 0.05860137939453125,
      "rewards/rejected": -0.13100585341453552,
      "step": 1140
    },
    {
      "epoch": 0.14840624596722157,
      "grad_norm": 1.6109741652060867,
      "learning_rate": 2.359071298478354e-06,
      "log_odds_chosen": 1.119042992591858,
      "log_odds_ratio": -0.44243162870407104,
      "logits/chosen": -0.980664074420929,
      "logits/rejected": -0.897265613079071,
      "logps/chosen": -0.708203136920929,
      "logps/rejected": -1.48828125,
      "loss": 1.0906,
      "nll_loss": 1.0554687976837158,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07081298530101776,
      "rewards/margins": 0.07807616889476776,
      "rewards/rejected": -0.14882811903953552,
      "step": 1150
    },
    {
      "epoch": 0.1496967350625887,
      "grad_norm": 1.59390160720588,
      "learning_rate": 2.3488808780588137e-06,
      "log_odds_chosen": 1.0622069835662842,
      "log_odds_ratio": -0.47514647245407104,
      "logits/chosen": -1.07421875,
      "logits/rejected": -0.9740234613418579,
      "logps/chosen": -0.774218738079071,
      "logps/rejected": -1.529296875,
      "loss": 1.0955,
      "nll_loss": 1.018164038658142,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07745361328125,
      "rewards/margins": 0.07558593899011612,
      "rewards/rejected": -0.15302734076976776,
      "step": 1160
    },
    {
      "epoch": 0.15098722415795587,
      "grad_norm": 1.9225141615763093,
      "learning_rate": 2.3388213848187446e-06,
      "log_odds_chosen": 1.0421874523162842,
      "log_odds_ratio": -0.4791503846645355,
      "logits/chosen": -1.056054711341858,
      "logits/rejected": -0.937304675579071,
      "logps/chosen": -0.721386730670929,
      "logps/rejected": -1.4259765148162842,
      "loss": 1.082,
      "nll_loss": 1.0402343273162842,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07215575873851776,
      "rewards/margins": 0.07044067233800888,
      "rewards/rejected": -0.14274902641773224,
      "step": 1170
    },
    {
      "epoch": 0.152277713253323,
      "grad_norm": 1.5715053291265315,
      "learning_rate": 2.328890038958328e-06,
      "log_odds_chosen": 1.18994140625,
      "log_odds_ratio": -0.42524415254592896,
      "logits/chosen": -1.087890625,
      "logits/rejected": -0.939648449420929,
      "logps/chosen": -0.729687511920929,
      "logps/rejected": -1.542578101158142,
      "loss": 1.099,
      "nll_loss": 1.0109374523162842,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07303466647863388,
      "rewards/margins": 0.08123779296875,
      "rewards/rejected": -0.15427246689796448,
      "step": 1180
    },
    {
      "epoch": 0.15356820234869015,
      "grad_norm": 1.689601754111662,
      "learning_rate": 2.3190841426097937e-06,
      "log_odds_chosen": 0.7685546875,
      "log_odds_ratio": -0.583203136920929,
      "logits/chosen": -1.0623047351837158,
      "logits/rejected": -0.9808593988418579,
      "logps/chosen": -0.783203125,
      "logps/rejected": -1.334375023841858,
      "loss": 1.0844,
      "nll_loss": 1.0603516101837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07823486626148224,
      "rewards/margins": 0.05517425388097763,
      "rewards/rejected": -0.13349609076976776,
      "step": 1190
    },
    {
      "epoch": 0.1548586914440573,
      "grad_norm": 1.7802478300761817,
      "learning_rate": 2.309401076758503e-06,
      "log_odds_chosen": 1.037072777748108,
      "log_odds_ratio": -0.4749999940395355,
      "logits/chosen": -0.9994140863418579,
      "logits/rejected": -0.8998047113418579,
      "logps/chosen": -0.783984363079071,
      "logps/rejected": -1.509374976158142,
      "loss": 1.0871,
      "nll_loss": 1.147070288658142,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07845459133386612,
      "rewards/margins": 0.07257156074047089,
      "rewards/rejected": -0.15104980766773224,
      "step": 1200
    },
    {
      "epoch": 0.15614918053942445,
      "grad_norm": 1.7082649024909833,
      "learning_rate": 2.299838298304276e-06,
      "log_odds_chosen": 1.10516357421875,
      "log_odds_ratio": -0.4549316465854645,
      "logits/chosen": -1.078515648841858,
      "logits/rejected": -0.9644531011581421,
      "logps/chosen": -0.768750011920929,
      "logps/rejected": -1.546289086341858,
      "loss": 1.0795,
      "nll_loss": 1.033789038658142,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.076904296875,
      "rewards/margins": 0.07789917290210724,
      "rewards/rejected": -0.15458984673023224,
      "step": 1210
    },
    {
      "epoch": 0.15743966963479158,
      "grad_norm": 1.8234367863977954,
      "learning_rate": 2.2903933372554728e-06,
      "log_odds_chosen": 1.1671874523162842,
      "log_odds_ratio": -0.45551759004592896,
      "logits/chosen": -1.0263671875,
      "logits/rejected": -0.9125000238418579,
      "logps/chosen": -0.7124999761581421,
      "logps/rejected": -1.5574219226837158,
      "loss": 1.0894,
      "nll_loss": 1.0068359375,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0712890625,
      "rewards/margins": 0.08432617038488388,
      "rewards/rejected": -0.15566405653953552,
      "step": 1220
    },
    {
      "epoch": 0.15873015873015872,
      "grad_norm": 1.588205997418655,
      "learning_rate": 2.281063794048804e-06,
      "log_odds_chosen": 1.072363257408142,
      "log_odds_ratio": -0.4695800840854645,
      "logits/chosen": -1.042578101158142,
      "logits/rejected": -0.9546874761581421,
      "logps/chosen": -0.696484386920929,
      "logps/rejected": -1.459570288658142,
      "loss": 1.0689,
      "nll_loss": 1.021093726158142,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06971435248851776,
      "rewards/margins": 0.07634277641773224,
      "rewards/rejected": -0.14599609375,
      "step": 1230
    },
    {
      "epoch": 0.16002064782552589,
      "grad_norm": 1.5837192481896127,
      "learning_rate": 2.271847336988259e-06,
      "log_odds_chosen": 1.1006348133087158,
      "log_odds_ratio": -0.48627930879592896,
      "logits/chosen": -1.0109374523162842,
      "logits/rejected": -0.9037109613418579,
      "logps/chosen": -0.763671875,
      "logps/rejected": -1.5378906726837158,
      "loss": 1.0855,
      "nll_loss": 1.05859375,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07648925483226776,
      "rewards/margins": 0.07727356255054474,
      "rewards/rejected": -0.15378418564796448,
      "step": 1240
    },
    {
      "epoch": 0.16131113692089302,
      "grad_norm": 1.6814446118097723,
      "learning_rate": 2.262741699796952e-06,
      "log_odds_chosen": 1.051171898841858,
      "log_odds_ratio": -0.4861816465854645,
      "logits/chosen": -1.0544922351837158,
      "logits/rejected": -0.9498046636581421,
      "logps/chosen": -0.756054699420929,
      "logps/rejected": -1.508203148841858,
      "loss": 1.0961,
      "nll_loss": 1.0244140625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07561035454273224,
      "rewards/margins": 0.07518615573644638,
      "rewards/rejected": -0.15085449814796448,
      "step": 1250
    },
    {
      "epoch": 0.16260162601626016,
      "grad_norm": 1.6279052465322343,
      "learning_rate": 2.253744679276044e-06,
      "log_odds_chosen": 1.2472655773162842,
      "log_odds_ratio": -0.4207519590854645,
      "logits/chosen": -1.014257788658142,
      "logits/rejected": -0.8968750238418579,
      "logps/chosen": -0.702929675579071,
      "logps/rejected": -1.5734374523162842,
      "loss": 1.0903,
      "nll_loss": 1.067773461341858,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07027588039636612,
      "rewards/margins": 0.08710937201976776,
      "rewards/rejected": -0.15739746391773224,
      "step": 1260
    },
    {
      "epoch": 0.1638921151116273,
      "grad_norm": 1.822900387791992,
      "learning_rate": 2.244854133065255e-06,
      "log_odds_chosen": 1.402734398841858,
      "log_odds_ratio": -0.3648925721645355,
      "logits/chosen": -1.0330078601837158,
      "logits/rejected": -0.9164062738418579,
      "logps/chosen": -0.700976550579071,
      "logps/rejected": -1.703125,
      "loss": 1.0729,
      "nll_loss": 1.0925781726837158,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.07008056342601776,
      "rewards/margins": 0.10039062798023224,
      "rewards/rejected": -0.17043456435203552,
      "step": 1270
    },
    {
      "epoch": 0.16518260420699446,
      "grad_norm": 1.571882177906907,
      "learning_rate": 2.2360679774997895e-06,
      "log_odds_chosen": 1.14208984375,
      "log_odds_ratio": -0.4735351502895355,
      "logits/chosen": -1.090234398841858,
      "logits/rejected": -0.994921863079071,
      "logps/chosen": -0.736621081829071,
      "logps/rejected": -1.549218773841858,
      "loss": 1.0586,
      "nll_loss": 0.984375,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07368163764476776,
      "rewards/margins": 0.08121947944164276,
      "rewards/rejected": -0.15488281846046448,
      "step": 1280
    },
    {
      "epoch": 0.1664730933023616,
      "grad_norm": 1.6608712242308838,
      "learning_rate": 2.2273841855588183e-06,
      "log_odds_chosen": 1.014892578125,
      "log_odds_ratio": -0.48486328125,
      "logits/chosen": -1.088281273841858,
      "logits/rejected": -0.9859374761581421,
      "logps/chosen": -0.757617175579071,
      "logps/rejected": -1.4773437976837158,
      "loss": 1.0922,
      "nll_loss": 1.047265648841858,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07578124850988388,
      "rewards/margins": 0.07194824516773224,
      "rewards/rejected": -0.14780274033546448,
      "step": 1290
    },
    {
      "epoch": 0.16776358239772873,
      "grad_norm": 1.7771548745200367,
      "learning_rate": 2.2188007849009167e-06,
      "log_odds_chosen": 1.14990234375,
      "log_odds_ratio": -0.4537597596645355,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.9605468511581421,
      "logps/chosen": -0.747265636920929,
      "logps/rejected": -1.5792968273162842,
      "loss": 1.0813,
      "nll_loss": 1.0613281726837158,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07476806640625,
      "rewards/margins": 0.08305053412914276,
      "rewards/rejected": -0.15800781548023224,
      "step": 1300
    },
    {
      "epoch": 0.16905407149309587,
      "grad_norm": 1.6194885122633327,
      "learning_rate": 2.2103158559821502e-06,
      "log_odds_chosen": 1.316308617591858,
      "log_odds_ratio": -0.39140623807907104,
      "logits/chosen": -1.073632836341858,
      "logits/rejected": -0.949999988079071,
      "logps/chosen": -0.687695324420929,
      "logps/rejected": -1.5867187976837158,
      "loss": 1.0597,
      "nll_loss": 0.996874988079071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06878662109375,
      "rewards/margins": 0.08988036960363388,
      "rewards/rejected": -0.15859374403953552,
      "step": 1310
    },
    {
      "epoch": 0.17034456058846303,
      "grad_norm": 1.6597539638400745,
      "learning_rate": 2.2019275302527213e-06,
      "log_odds_chosen": 1.071191430091858,
      "log_odds_ratio": -0.45683592557907104,
      "logits/chosen": -1.0517578125,
      "logits/rejected": -0.949999988079071,
      "logps/chosen": -0.7115234136581421,
      "logps/rejected": -1.45703125,
      "loss": 1.0511,
      "nll_loss": 1.031640648841858,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07117919623851776,
      "rewards/margins": 0.074432373046875,
      "rewards/rejected": -0.14565429091453552,
      "step": 1320
    },
    {
      "epoch": 0.17163504968383017,
      "grad_norm": 1.625531017423128,
      "learning_rate": 2.193633988428327e-06,
      "log_odds_chosen": 1.3430664539337158,
      "log_odds_ratio": -0.4144531190395355,
      "logits/chosen": -1.017968773841858,
      "logits/rejected": -0.931445300579071,
      "logps/chosen": -0.7051757574081421,
      "logps/rejected": -1.666406273841858,
      "loss": 1.0865,
      "nll_loss": 1.115625023841858,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07053222507238388,
      "rewards/margins": 0.09592895209789276,
      "rewards/rejected": -0.1666259765625,
      "step": 1330
    },
    {
      "epoch": 0.1729255387791973,
      "grad_norm": 1.8433906695783988,
      "learning_rate": 2.185433458832612e-06,
      "log_odds_chosen": 1.173486351966858,
      "log_odds_ratio": -0.41791993379592896,
      "logits/chosen": -1.1046874523162842,
      "logits/rejected": -1.0011718273162842,
      "logps/chosen": -0.728320300579071,
      "logps/rejected": -1.5625,
      "loss": 1.0901,
      "nll_loss": 1.032812476158142,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07280273735523224,
      "rewards/margins": 0.083526611328125,
      "rewards/rejected": -0.15634766221046448,
      "step": 1340
    },
    {
      "epoch": 0.17421602787456447,
      "grad_norm": 1.6857588285270955,
      "learning_rate": 2.177324215807269e-06,
      "log_odds_chosen": 1.0805785655975342,
      "log_odds_ratio": -0.45380860567092896,
      "logits/chosen": -1.125,
      "logits/rejected": -1.021484375,
      "logps/chosen": -0.673535168170929,
      "logps/rejected": -1.406640648841858,
      "loss": 1.061,
      "nll_loss": 0.984179675579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06740722805261612,
      "rewards/margins": 0.07321777194738388,
      "rewards/rejected": -0.14069823920726776,
      "step": 1350
    },
    {
      "epoch": 0.1755065169699316,
      "grad_norm": 1.6247597108830008,
      "learning_rate": 2.1693045781865616e-06,
      "log_odds_chosen": 1.202734351158142,
      "log_odds_ratio": -0.4351562559604645,
      "logits/chosen": -1.111718773841858,
      "logits/rejected": -0.990234375,
      "logps/chosen": -0.7562500238418579,
      "logps/rejected": -1.6259765625,
      "loss": 1.066,
      "nll_loss": 1.041406273841858,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07565917819738388,
      "rewards/margins": 0.08698730170726776,
      "rewards/rejected": -0.16259765625,
      "step": 1360
    },
    {
      "epoch": 0.17679700606529875,
      "grad_norm": 1.6040775977140727,
      "learning_rate": 2.1613729078331965e-06,
      "log_odds_chosen": 1.3590819835662842,
      "log_odds_ratio": -0.4395507872104645,
      "logits/chosen": -1.129492163658142,
      "logits/rejected": -1.024999976158142,
      "logps/chosen": -0.7740234136581421,
      "logps/rejected": -1.732421875,
      "loss": 1.026,
      "nll_loss": 0.991406261920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07736816257238388,
      "rewards/margins": 0.0958099365234375,
      "rewards/rejected": -0.17319336533546448,
      "step": 1370
    },
    {
      "epoch": 0.17808749516066588,
      "grad_norm": 1.6383541537297692,
      "learning_rate": 2.1535276082326617e-06,
      "log_odds_chosen": 1.363867163658142,
      "log_odds_ratio": -0.41645509004592896,
      "logits/chosen": -1.0470702648162842,
      "logits/rejected": -0.9638671875,
      "logps/chosen": -0.742968738079071,
      "logps/rejected": -1.7433593273162842,
      "loss": 1.0935,
      "nll_loss": 1.0486328601837158,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0743408203125,
      "rewards/margins": 0.10007324069738388,
      "rewards/rejected": -0.17436523735523224,
      "step": 1380
    },
    {
      "epoch": 0.17937798425603305,
      "grad_norm": 2.183509521851013,
      "learning_rate": 2.14576712314328e-06,
      "log_odds_chosen": 1.1744263172149658,
      "log_odds_ratio": -0.431396484375,
      "logits/chosen": -1.042382836341858,
      "logits/rejected": -0.9320312738418579,
      "logps/chosen": -0.723828136920929,
      "logps/rejected": -1.521484375,
      "loss": 1.0835,
      "nll_loss": 1.0654296875,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07242431491613388,
      "rewards/margins": 0.07967529445886612,
      "rewards/rejected": -0.15214844048023224,
      "step": 1390
    },
    {
      "epoch": 0.18066847335140018,
      "grad_norm": 1.8701538031075127,
      "learning_rate": 2.138089935299395e-06,
      "log_odds_chosen": 1.1837890148162842,
      "log_odds_ratio": -0.45751953125,
      "logits/chosen": -1.0861327648162842,
      "logits/rejected": -0.982421875,
      "logps/chosen": -0.6871093511581421,
      "logps/rejected": -1.547460913658142,
      "loss": 1.067,
      "nll_loss": 0.9755859375,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06864013522863388,
      "rewards/margins": 0.08605346828699112,
      "rewards/rejected": -0.15485839545726776,
      "step": 1400
    },
    {
      "epoch": 0.18195896244676732,
      "grad_norm": 1.7013313021529102,
      "learning_rate": 2.1304945651652297e-06,
      "log_odds_chosen": 1.3183104991912842,
      "log_odds_ratio": -0.394775390625,
      "logits/chosen": -1.1027343273162842,
      "logits/rejected": -0.9691406488418579,
      "logps/chosen": -0.6820312738418579,
      "logps/rejected": -1.606054663658142,
      "loss": 1.037,
      "nll_loss": 0.94921875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06818847358226776,
      "rewards/margins": 0.09233398735523224,
      "rewards/rejected": -0.16042479872703552,
      "step": 1410
    },
    {
      "epoch": 0.18324945154213446,
      "grad_norm": 2.2167624789978477,
      "learning_rate": 2.122979569737101e-06,
      "log_odds_chosen": 1.172143578529358,
      "log_odds_ratio": -0.4599609375,
      "logits/chosen": -1.0851562023162842,
      "logits/rejected": -0.984179675579071,
      "logps/chosen": -0.7701171636581421,
      "logps/rejected": -1.6082031726837158,
      "loss": 1.0877,
      "nll_loss": 1.0173828601837158,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07694091647863388,
      "rewards/margins": 0.08378143608570099,
      "rewards/rejected": -0.16074219346046448,
      "step": 1420
    },
    {
      "epoch": 0.18453994063750162,
      "grad_norm": 1.613254195536557,
      "learning_rate": 2.11554354139178e-06,
      "log_odds_chosen": 1.155664086341858,
      "log_odds_ratio": -0.421630859375,
      "logits/chosen": -1.060546875,
      "logits/rejected": -0.9593750238418579,
      "logps/chosen": -0.748242199420929,
      "logps/rejected": -1.5505859851837158,
      "loss": 1.0431,
      "nll_loss": 1.098046898841858,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07480468600988388,
      "rewards/margins": 0.08023681491613388,
      "rewards/rejected": -0.15498046576976776,
      "step": 1430
    },
    {
      "epoch": 0.18583042973286876,
      "grad_norm": 1.7258647812467083,
      "learning_rate": 2.1081851067789196e-06,
      "log_odds_chosen": 0.9495849609375,
      "log_odds_ratio": -0.5160156488418579,
      "logits/chosen": -1.069726586341858,
      "logits/rejected": -0.9898437261581421,
      "logps/chosen": -0.718554675579071,
      "logps/rejected": -1.388085961341858,
      "loss": 1.0395,
      "nll_loss": 1.005859375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07183837890625,
      "rewards/margins": 0.066925048828125,
      "rewards/rejected": -0.13874511420726776,
      "step": 1440
    },
    {
      "epoch": 0.1871209188282359,
      "grad_norm": 1.9234373337632442,
      "learning_rate": 2.1009029257555606e-06,
      "log_odds_chosen": 1.3720703125,
      "log_odds_ratio": -0.3963378965854645,
      "logits/chosen": -1.0751953125,
      "logits/rejected": -0.9740234613418579,
      "logps/chosen": -0.739062488079071,
      "logps/rejected": -1.713281273841858,
      "loss": 1.0658,
      "nll_loss": 0.988476574420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07396240532398224,
      "rewards/margins": 0.09743042290210724,
      "rewards/rejected": -0.17128905653953552,
      "step": 1450
    },
    {
      "epoch": 0.18841140792360306,
      "grad_norm": 1.6352679101256875,
      "learning_rate": 2.0936956903608545e-06,
      "log_odds_chosen": 1.16650390625,
      "log_odds_ratio": -0.45087891817092896,
      "logits/chosen": -1.0632812976837158,
      "logits/rejected": -0.9853515625,
      "logps/chosen": -0.744335949420929,
      "logps/rejected": -1.584570288658142,
      "loss": 1.0447,
      "nll_loss": 0.967578113079071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.074462890625,
      "rewards/margins": 0.08408508449792862,
      "rewards/rejected": -0.15852050483226776,
      "step": 1460
    },
    {
      "epoch": 0.1897018970189702,
      "grad_norm": 1.8258394457494709,
      "learning_rate": 2.0865621238292046e-06,
      "log_odds_chosen": 1.376855492591858,
      "log_odds_ratio": -0.4065918028354645,
      "logits/chosen": -1.0812499523162842,
      "logits/rejected": -0.9638671875,
      "logps/chosen": -0.720410168170929,
      "logps/rejected": -1.6882812976837158,
      "loss": 1.0396,
      "nll_loss": 0.9722656011581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07203368842601776,
      "rewards/margins": 0.096832275390625,
      "rewards/rejected": -0.1689453125,
      "step": 1470
    },
    {
      "epoch": 0.19099238611433733,
      "grad_norm": 1.6442729868164676,
      "learning_rate": 2.079500979640145e-06,
      "log_odds_chosen": 1.0532715320587158,
      "log_odds_ratio": -0.49604493379592896,
      "logits/chosen": -1.1013672351837158,
      "logits/rejected": -0.992968738079071,
      "logps/chosen": -0.7466796636581421,
      "logps/rejected": -1.4736328125,
      "loss": 1.0641,
      "nll_loss": 0.990429699420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07465820014476776,
      "rewards/margins": 0.07262115180492401,
      "rewards/rejected": -0.14738769829273224,
      "step": 1480
    },
    {
      "epoch": 0.19228287520970447,
      "grad_norm": 1.8606689334576478,
      "learning_rate": 2.072511040603359e-06,
      "log_odds_chosen": 1.1824219226837158,
      "log_odds_ratio": -0.4297851622104645,
      "logits/chosen": -1.0994141101837158,
      "logits/rejected": -0.9878906011581421,
      "logps/chosen": -0.7466796636581421,
      "logps/rejected": -1.576171875,
      "loss": 1.0298,
      "nll_loss": 0.97119140625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07462158054113388,
      "rewards/margins": 0.08292236179113388,
      "rewards/rejected": -0.15761718153953552,
      "step": 1490
    },
    {
      "epoch": 0.19357336430507163,
      "grad_norm": 1.9858143925699727,
      "learning_rate": 2.065591117977289e-06,
      "log_odds_chosen": 1.0282714366912842,
      "log_odds_ratio": -0.520263671875,
      "logits/chosen": -1.114648461341858,
      "logits/rejected": -1.0126953125,
      "logps/chosen": -0.724414050579071,
      "logps/rejected": -1.4833984375,
      "loss": 1.0169,
      "nll_loss": 0.984179675579071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07242431491613388,
      "rewards/margins": 0.07600555568933487,
      "rewards/rejected": -0.14838866889476776,
      "step": 1500
    },
    {
      "epoch": 0.19486385340043877,
      "grad_norm": 1.918319757325415,
      "learning_rate": 2.058740050619915e-06,
      "log_odds_chosen": 1.299218773841858,
      "log_odds_ratio": -0.41865235567092896,
      "logits/chosen": -1.1181640625,
      "logits/rejected": -1.0265624523162842,
      "logps/chosen": -0.704296886920929,
      "logps/rejected": -1.592187523841858,
      "loss": 1.0561,
      "nll_loss": 1.0,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07045898586511612,
      "rewards/margins": 0.08869018405675888,
      "rewards/rejected": -0.15922851860523224,
      "step": 1510
    },
    {
      "epoch": 0.1961543424958059,
      "grad_norm": 1.788846599945369,
      "learning_rate": 2.0519567041703083e-06,
      "log_odds_chosen": 0.985852062702179,
      "log_odds_ratio": -0.47221678495407104,
      "logits/chosen": -1.1027343273162842,
      "logits/rejected": -0.994921863079071,
      "logps/chosen": -0.734375,
      "logps/rejected": -1.415429711341858,
      "loss": 1.0618,
      "nll_loss": 1.0134766101837158,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07347412407398224,
      "rewards/margins": 0.06799087673425674,
      "rewards/rejected": -0.14155273139476776,
      "step": 1520
    },
    {
      "epoch": 0.19744483159117304,
      "grad_norm": 1.8482591091328722,
      "learning_rate": 2.0452399702596544e-06,
      "log_odds_chosen": 1.2839844226837158,
      "log_odds_ratio": -0.3934082090854645,
      "logits/chosen": -1.094140648841858,
      "logits/rejected": -0.9789062738418579,
      "logps/chosen": -0.7123047113418579,
      "logps/rejected": -1.6111328601837158,
      "loss": 1.0525,
      "nll_loss": 1.0349609851837158,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07119140774011612,
      "rewards/margins": 0.08989258110523224,
      "rewards/rejected": -0.1611328125,
      "step": 1530
    },
    {
      "epoch": 0.1987353206865402,
      "grad_norm": 1.8974600577195853,
      "learning_rate": 2.0385887657505017e-06,
      "log_odds_chosen": 1.4958984851837158,
      "log_odds_ratio": -0.378173828125,
      "logits/chosen": -1.112890601158142,
      "logits/rejected": -0.964648425579071,
      "logps/chosen": -0.6908203363418579,
      "logps/rejected": -1.773046851158142,
      "loss": 1.0427,
      "nll_loss": 1.014257788658142,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06910400092601776,
      "rewards/margins": 0.10810546576976776,
      "rewards/rejected": -0.17714843153953552,
      "step": 1540
    },
    {
      "epoch": 0.20002580978190734,
      "grad_norm": 1.7499697202413866,
      "learning_rate": 2.032002032003048e-06,
      "log_odds_chosen": 1.201171875,
      "log_odds_ratio": -0.43681639432907104,
      "logits/chosen": -1.085351586341858,
      "logits/rejected": -0.9794921875,
      "logps/chosen": -0.7007812261581421,
      "logps/rejected": -1.5369141101837158,
      "loss": 1.0307,
      "nll_loss": 0.9720703363418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07005615532398224,
      "rewards/margins": 0.08378295600414276,
      "rewards/rejected": -0.1539306640625,
      "step": 1550
    },
    {
      "epoch": 0.20131629887727448,
      "grad_norm": 1.7929081879999744,
      "learning_rate": 2.025478734167333e-06,
      "log_odds_chosen": 1.1455566883087158,
      "log_odds_ratio": -0.4432128965854645,
      "logits/chosen": -1.067968726158142,
      "logits/rejected": -0.989062488079071,
      "logps/chosen": -0.68603515625,
      "logps/rejected": -1.4658203125,
      "loss": 1.011,
      "nll_loss": 0.976367175579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06862793117761612,
      "rewards/margins": 0.07796020805835724,
      "rewards/rejected": -0.14665527641773224,
      "step": 1560
    },
    {
      "epoch": 0.20260678797264164,
      "grad_norm": 1.959589815186418,
      "learning_rate": 2.0190178605002747e-06,
      "log_odds_chosen": 1.232421875,
      "log_odds_ratio": -0.4590820372104645,
      "logits/chosen": -1.0753905773162842,
      "logits/rejected": -0.96484375,
      "logps/chosen": -0.693066418170929,
      "logps/rejected": -1.5769531726837158,
      "loss": 1.0405,
      "nll_loss": 0.9515625238418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06929931789636612,
      "rewards/margins": 0.08840332180261612,
      "rewards/rejected": -0.15769043564796448,
      "step": 1570
    },
    {
      "epoch": 0.20389727706800878,
      "grad_norm": 1.9034651448438396,
      "learning_rate": 2.0126184217065104e-06,
      "log_odds_chosen": 1.1453125476837158,
      "log_odds_ratio": -0.4483398497104645,
      "logits/chosen": -1.084570288658142,
      "logits/rejected": -0.9798828363418579,
      "logps/chosen": -0.724609375,
      "logps/rejected": -1.4669921398162842,
      "loss": 1.0462,
      "nll_loss": 1.0007812976837158,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07242431491613388,
      "rewards/margins": 0.07419433444738388,
      "rewards/rejected": -0.14677734673023224,
      "step": 1580
    },
    {
      "epoch": 0.20518776616337592,
      "grad_norm": 1.790260904860215,
      "learning_rate": 2.0062794503020765e-06,
      "log_odds_chosen": 1.1943359375,
      "log_odds_ratio": -0.43671876192092896,
      "logits/chosen": -1.101953148841858,
      "logits/rejected": -1.005859375,
      "logps/chosen": -0.7027343511581421,
      "logps/rejected": -1.524999976158142,
      "loss": 1.0407,
      "nll_loss": 0.898242175579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07030029594898224,
      "rewards/margins": 0.08206786960363388,
      "rewards/rejected": -0.15236815810203552,
      "step": 1590
    },
    {
      "epoch": 0.20647825525874305,
      "grad_norm": 1.6915203190033656,
      "learning_rate": 2e-06,
      "log_odds_chosen": 1.326208472251892,
      "log_odds_ratio": -0.42768555879592896,
      "logits/chosen": -1.034570336341858,
      "logits/rejected": -0.9624999761581421,
      "logps/chosen": -0.685546875,
      "logps/rejected": -1.6427733898162842,
      "loss": 1.0309,
      "nll_loss": 1.0119140148162842,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06854248046875,
      "rewards/margins": 0.095850370824337,
      "rewards/rejected": -0.16451415419578552,
      "step": 1600
    },
    {
      "epoch": 0.20776874435411022,
      "grad_norm": 1.9363627386524394,
      "learning_rate": 1.993779145116907e-06,
      "log_odds_chosen": 1.4005858898162842,
      "log_odds_ratio": -0.40327149629592896,
      "logits/chosen": -1.091210961341858,
      "logits/rejected": -0.9839843511581421,
      "logps/chosen": -0.687695324420929,
      "logps/rejected": -1.7218749523162842,
      "loss": 1.0314,
      "nll_loss": 0.949414074420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06882324069738388,
      "rewards/margins": 0.10336913913488388,
      "rewards/rejected": -0.17214354872703552,
      "step": 1610
    },
    {
      "epoch": 0.20905923344947736,
      "grad_norm": 1.8151339948576386,
      "learning_rate": 1.987615979999813e-06,
      "log_odds_chosen": 1.250817894935608,
      "log_odds_ratio": -0.4544921815395355,
      "logits/chosen": -1.0314452648162842,
      "logits/rejected": -0.9076172113418579,
      "logps/chosen": -0.779980480670929,
      "logps/rejected": -1.6814453601837158,
      "loss": 1.0293,
      "nll_loss": 1.046289086341858,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07801513373851776,
      "rewards/margins": 0.09031829982995987,
      "rewards/rejected": -0.16831055283546448,
      "step": 1620
    },
    {
      "epoch": 0.2103497225448445,
      "grad_norm": 1.7090267203033147,
      "learning_rate": 1.9815096184722797e-06,
      "log_odds_chosen": 1.2394530773162842,
      "log_odds_ratio": -0.40947264432907104,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.939453125,
      "logps/chosen": -0.6488281488418579,
      "logps/rejected": -1.490234375,
      "loss": 1.0406,
      "nll_loss": 0.9673827886581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06490478664636612,
      "rewards/margins": 0.08427734673023224,
      "rewards/rejected": -0.14923095703125,
      "step": 1630
    },
    {
      "epoch": 0.21164021164021163,
      "grad_norm": 1.900075085153943,
      "learning_rate": 1.9754591932991793e-06,
      "log_odds_chosen": 1.357031226158142,
      "log_odds_ratio": -0.4049316346645355,
      "logits/chosen": -1.067968726158142,
      "logits/rejected": -0.96484375,
      "logps/chosen": -0.6982421875,
      "logps/rejected": -1.671875,
      "loss": 1.0247,
      "nll_loss": 0.9664062261581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06976318359375,
      "rewards/margins": 0.09752197563648224,
      "rewards/rejected": -0.16740722954273224,
      "step": 1640
    },
    {
      "epoch": 0.2129307007355788,
      "grad_norm": 1.8767779541122893,
      "learning_rate": 1.9694638556693235e-06,
      "log_odds_chosen": 1.2439453601837158,
      "log_odds_ratio": -0.4251953065395355,
      "logits/chosen": -1.0681641101837158,
      "logits/rejected": -0.9457031488418579,
      "logps/chosen": -0.6929687261581421,
      "logps/rejected": -1.539453148841858,
      "loss": 1.0378,
      "nll_loss": 1.0060546398162842,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06932373344898224,
      "rewards/margins": 0.08466186374425888,
      "rewards/rejected": -0.15385742485523224,
      "step": 1650
    },
    {
      "epoch": 0.21422118983094593,
      "grad_norm": 1.7661927131597999,
      "learning_rate": 1.963522774695264e-06,
      "log_odds_chosen": 1.19140625,
      "log_odds_ratio": -0.469970703125,
      "logits/chosen": -1.0740234851837158,
      "logits/rejected": -0.9765625,
      "logps/chosen": -0.7157226800918579,
      "logps/rejected": -1.568750023841858,
      "loss": 1.0244,
      "nll_loss": 0.9873046875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07158203423023224,
      "rewards/margins": 0.08530273288488388,
      "rewards/rejected": -0.15688475966453552,
      "step": 1660
    },
    {
      "epoch": 0.21551167892631307,
      "grad_norm": 1.9847941188964193,
      "learning_rate": 1.9576351369295853e-06,
      "log_odds_chosen": 1.279687523841858,
      "log_odds_ratio": -0.49755859375,
      "logits/chosen": -1.0867187976837158,
      "logits/rejected": -0.927929699420929,
      "logps/chosen": -0.728515625,
      "logps/rejected": -1.6765625476837158,
      "loss": 1.0027,
      "nll_loss": 0.9974609613418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07283935695886612,
      "rewards/margins": 0.094696044921875,
      "rewards/rejected": -0.16757813096046448,
      "step": 1670
    },
    {
      "epoch": 0.21680216802168023,
      "grad_norm": 2.353315294198319,
      "learning_rate": 1.951800145897066e-06,
      "log_odds_chosen": 1.3696777820587158,
      "log_odds_ratio": -0.41606444120407104,
      "logits/chosen": -1.0705077648162842,
      "logits/rejected": -0.9642578363418579,
      "logps/chosen": -0.6908203363418579,
      "logps/rejected": -1.677734375,
      "loss": 1.0216,
      "nll_loss": 0.931445300579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06894531100988388,
      "rewards/margins": 0.0986785888671875,
      "rewards/rejected": -0.167724609375,
      "step": 1680
    },
    {
      "epoch": 0.21809265711704737,
      "grad_norm": 1.9236629940310346,
      "learning_rate": 1.9460170216420797e-06,
      "log_odds_chosen": 1.263085961341858,
      "log_odds_ratio": -0.39531248807907104,
      "logits/chosen": -1.080664038658142,
      "logits/rejected": -0.978710949420929,
      "logps/chosen": -0.695605456829071,
      "logps/rejected": -1.56640625,
      "loss": 1.0278,
      "nll_loss": 1.011132836341858,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06964111328125,
      "rewards/margins": 0.08699951320886612,
      "rewards/rejected": -0.1566162109375,
      "step": 1690
    },
    {
      "epoch": 0.2193831462124145,
      "grad_norm": 1.847855353325965,
      "learning_rate": 1.9402850002906637e-06,
      "log_odds_chosen": 1.4462890625,
      "log_odds_ratio": -0.40336912870407104,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.932421863079071,
      "logps/chosen": -0.7484375238418579,
      "logps/rejected": -1.8400390148162842,
      "loss": 1.0411,
      "nll_loss": 1.005468726158142,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07484130561351776,
      "rewards/margins": 0.10910644382238388,
      "rewards/rejected": -0.18405762314796448,
      "step": 1700
    },
    {
      "epoch": 0.22067363530778164,
      "grad_norm": 1.7438480589718366,
      "learning_rate": 1.9346033336266974e-06,
      "log_odds_chosen": 1.2914550304412842,
      "log_odds_ratio": -0.4862304627895355,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.916015625,
      "logps/chosen": -0.7925781011581421,
      "logps/rejected": -1.7423827648162842,
      "loss": 1.0411,
      "nll_loss": 1.0080077648162842,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07921142876148224,
      "rewards/margins": 0.094940185546875,
      "rewards/rejected": -0.17426757514476776,
      "step": 1710
    },
    {
      "epoch": 0.2219641244031488,
      "grad_norm": 1.6490194040959008,
      "learning_rate": 1.9289712886816486e-06,
      "log_odds_chosen": 1.066796898841858,
      "log_odds_ratio": -0.44892579317092896,
      "logits/chosen": -1.0802733898162842,
      "logits/rejected": -0.9527343511581421,
      "logps/chosen": -0.7432616949081421,
      "logps/rejected": -1.46875,
      "loss": 1.0157,
      "nll_loss": 0.9794921875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07435302436351776,
      "rewards/margins": 0.07256011664867401,
      "rewards/rejected": -0.14694824814796448,
      "step": 1720
    },
    {
      "epoch": 0.22325461349851594,
      "grad_norm": 1.7524221110672547,
      "learning_rate": 1.92338814733738e-06,
      "log_odds_chosen": 1.458398461341858,
      "log_odds_ratio": -0.39960938692092896,
      "logits/chosen": -1.0861327648162842,
      "logits/rejected": -0.965039074420929,
      "logps/chosen": -0.723828136920929,
      "logps/rejected": -1.7644531726837158,
      "loss": 1.0181,
      "nll_loss": 0.9398437738418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07243652641773224,
      "rewards/margins": 0.10396728664636612,
      "rewards/rejected": -0.17636719346046448,
      "step": 1730
    },
    {
      "epoch": 0.22454510259388308,
      "grad_norm": 1.771147413657448,
      "learning_rate": 1.9178532059415367e-06,
      "log_odds_chosen": 1.3468749523162842,
      "log_odds_ratio": -0.422119140625,
      "logits/chosen": -1.025976538658142,
      "logits/rejected": -0.9126952886581421,
      "logps/chosen": -0.712109386920929,
      "logps/rejected": -1.666015625,
      "loss": 1.0125,
      "nll_loss": 0.998828113079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07121582329273224,
      "rewards/margins": 0.09525146335363388,
      "rewards/rejected": -0.16660156846046448,
      "step": 1740
    },
    {
      "epoch": 0.22583559168925021,
      "grad_norm": 2.0107128806607433,
      "learning_rate": 1.9123657749350298e-06,
      "log_odds_chosen": 1.4989013671875,
      "log_odds_ratio": -0.4176025390625,
      "logits/chosen": -1.073828101158142,
      "logits/rejected": -0.938671886920929,
      "logps/chosen": -0.7496093511581421,
      "logps/rejected": -1.855078101158142,
      "loss": 1.0079,
      "nll_loss": 0.961718738079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07497558742761612,
      "rewards/margins": 0.11058960109949112,
      "rewards/rejected": -0.18574218451976776,
      "step": 1750
    },
    {
      "epoch": 0.22712608078461738,
      "grad_norm": 1.8339794528013524,
      "learning_rate": 1.9069251784911844e-06,
      "log_odds_chosen": 1.3408203125,
      "log_odds_ratio": -0.3848632872104645,
      "logits/chosen": -1.0929687023162842,
      "logits/rejected": -0.972460925579071,
      "logps/chosen": -0.658984363079071,
      "logps/rejected": -1.5832030773162842,
      "loss": 1.0264,
      "nll_loss": 0.9384765625,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06590576469898224,
      "rewards/margins": 0.09238891303539276,
      "rewards/rejected": -0.15825195610523224,
      "step": 1760
    },
    {
      "epoch": 0.22841656987998452,
      "grad_norm": 2.0519499712912888,
      "learning_rate": 1.9015307541661132e-06,
      "log_odds_chosen": 1.380859375,
      "log_odds_ratio": -0.40583497285842896,
      "logits/chosen": -1.0525391101837158,
      "logits/rejected": -0.9476562738418579,
      "logps/chosen": -0.7564452886581421,
      "logps/rejected": -1.764062523841858,
      "loss": 1.0061,
      "nll_loss": 0.95703125,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07565917819738388,
      "rewards/margins": 0.10075531154870987,
      "rewards/rejected": -0.17641600966453552,
      "step": 1770
    },
    {
      "epoch": 0.22970705897535165,
      "grad_norm": 1.905928387669216,
      "learning_rate": 1.8961818525599089e-06,
      "log_odds_chosen": 1.5802733898162842,
      "log_odds_ratio": -0.3426757752895355,
      "logits/chosen": -1.1222655773162842,
      "logits/rejected": -0.98046875,
      "logps/chosen": -0.6689453125,
      "logps/rejected": -1.775390625,
      "loss": 1.0186,
      "nll_loss": 0.915820300579071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06689453125,
      "rewards/margins": 0.11054076999425888,
      "rewards/rejected": -0.17727050185203552,
      "step": 1780
    },
    {
      "epoch": 0.2309975480707188,
      "grad_norm": 1.9958109275082005,
      "learning_rate": 1.890877836988262e-06,
      "log_odds_chosen": 1.0451171398162842,
      "log_odds_ratio": -0.4544921815395355,
      "logits/chosen": -1.0802733898162842,
      "logits/rejected": -0.963085949420929,
      "logps/chosen": -0.6958984136581421,
      "logps/rejected": -1.38671875,
      "loss": 1.0346,
      "nll_loss": 0.9964843988418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06960449367761612,
      "rewards/margins": 0.069061279296875,
      "rewards/rejected": -0.13872070610523224,
      "step": 1790
    },
    {
      "epoch": 0.23228803716608595,
      "grad_norm": 1.960774124923341,
      "learning_rate": 1.8856180831641269e-06,
      "log_odds_chosen": 1.4796874523162842,
      "log_odds_ratio": -0.3751464784145355,
      "logits/chosen": -1.12890625,
      "logits/rejected": -0.9931640625,
      "logps/chosen": -0.6802734136581421,
      "logps/rejected": -1.7179687023162842,
      "loss": 1.0287,
      "nll_loss": 0.9507812261581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06807861477136612,
      "rewards/margins": 0.103759765625,
      "rewards/rejected": -0.17182616889476776,
      "step": 1800
    },
    {
      "epoch": 0.2335785262614531,
      "grad_norm": 1.7577970521484003,
      "learning_rate": 1.8804019788890737e-06,
      "log_odds_chosen": 1.1182372570037842,
      "log_odds_ratio": -0.4847656190395355,
      "logits/chosen": -1.095117211341858,
      "logits/rejected": -0.9925781488418579,
      "logps/chosen": -0.7279297113418579,
      "logps/rejected": -1.524023413658142,
      "loss": 1.0068,
      "nll_loss": 1.0107421875,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07275390625,
      "rewards/margins": 0.07970428466796875,
      "rewards/rejected": -0.1524658203125,
      "step": 1810
    },
    {
      "epoch": 0.23486901535682023,
      "grad_norm": 1.828566841756213,
      "learning_rate": 1.8752289237539816e-06,
      "log_odds_chosen": 1.5224609375,
      "log_odds_ratio": -0.40239256620407104,
      "logits/chosen": -1.123437523841858,
      "logits/rejected": -0.9951171875,
      "logps/chosen": -0.709179699420929,
      "logps/rejected": -1.8214843273162842,
      "loss": 1.0015,
      "nll_loss": 0.956250011920929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07089843600988388,
      "rewards/margins": 0.11115722358226776,
      "rewards/rejected": -0.18205566704273224,
      "step": 1820
    },
    {
      "epoch": 0.2361595044521874,
      "grad_norm": 1.9578878317925366,
      "learning_rate": 1.8700983288487376e-06,
      "log_odds_chosen": 1.33349609375,
      "log_odds_ratio": -0.4251464903354645,
      "logits/chosen": -1.0564453601837158,
      "logits/rejected": -0.9671875238418579,
      "logps/chosen": -0.682812511920929,
      "logps/rejected": -1.639062523841858,
      "loss": 1.0165,
      "nll_loss": 0.9521484375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06828613579273224,
      "rewards/margins": 0.09553222358226776,
      "rewards/rejected": -0.16396483778953552,
      "step": 1830
    },
    {
      "epoch": 0.23744999354755453,
      "grad_norm": 2.043795104358643,
      "learning_rate": 1.8650096164806275e-06,
      "log_odds_chosen": 1.1906249523162842,
      "log_odds_ratio": -0.449951171875,
      "logits/chosen": -1.062890648841858,
      "logits/rejected": -0.971875011920929,
      "logps/chosen": -0.74609375,
      "logps/rejected": -1.5986328125,
      "loss": 1.0141,
      "nll_loss": 1.0066406726837158,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07459716498851776,
      "rewards/margins": 0.08531494438648224,
      "rewards/rejected": -0.15996094048023224,
      "step": 1840
    },
    {
      "epoch": 0.23874048264292166,
      "grad_norm": 1.8929614559178243,
      "learning_rate": 1.8599622199011084e-06,
      "log_odds_chosen": 1.498437523841858,
      "log_odds_ratio": -0.384521484375,
      "logits/chosen": -1.103906273841858,
      "logits/rejected": -0.978320300579071,
      "logps/chosen": -0.6685546636581421,
      "logps/rejected": -1.7234375476837158,
      "loss": 1.0226,
      "nll_loss": 0.8880859613418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06687011569738388,
      "rewards/margins": 0.10538329929113388,
      "rewards/rejected": -0.17226561903953552,
      "step": 1850
    },
    {
      "epoch": 0.2400309717382888,
      "grad_norm": 1.7375694576786638,
      "learning_rate": 1.854955583040673e-06,
      "log_odds_chosen": 1.338842749595642,
      "log_odds_ratio": -0.4287109375,
      "logits/chosen": -1.061132788658142,
      "logits/rejected": -0.978515625,
      "logps/chosen": -0.73095703125,
      "logps/rejected": -1.6632812023162842,
      "loss": 1.0189,
      "nll_loss": 0.9654296636581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07304687798023224,
      "rewards/margins": 0.09332122653722763,
      "rewards/rejected": -0.16640624403953552,
      "step": 1860
    },
    {
      "epoch": 0.24132146083365597,
      "grad_norm": 1.8571759239084946,
      "learning_rate": 1.849989160251521e-06,
      "log_odds_chosen": 1.219580054283142,
      "log_odds_ratio": -0.46796876192092896,
      "logits/chosen": -1.117773413658142,
      "logits/rejected": -0.9957031011581421,
      "logps/chosen": -0.69873046875,
      "logps/rejected": -1.5652344226837158,
      "loss": 1.0095,
      "nll_loss": 0.9814453125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06989745795726776,
      "rewards/margins": 0.08649902045726776,
      "rewards/rejected": -0.15656737983226776,
      "step": 1870
    },
    {
      "epoch": 0.2426119499290231,
      "grad_norm": 1.8627691716602723,
      "learning_rate": 1.8450624160577701e-06,
      "log_odds_chosen": 1.166772484779358,
      "log_odds_ratio": -0.449951171875,
      "logits/chosen": -1.102148413658142,
      "logits/rejected": -0.978320300579071,
      "logps/chosen": -0.7515624761581421,
      "logps/rejected": -1.603515625,
      "loss": 1.0014,
      "nll_loss": 0.9554687738418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07513427734375,
      "rewards/margins": 0.08519592136144638,
      "rewards/rejected": -0.1602783203125,
      "step": 1880
    },
    {
      "epoch": 0.24390243902439024,
      "grad_norm": 1.9801332914589682,
      "learning_rate": 1.8401748249129445e-06,
      "log_odds_chosen": 0.9989258050918579,
      "log_odds_ratio": -0.5022948980331421,
      "logits/chosen": -1.146875023841858,
      "logits/rejected": -1.041406273841858,
      "logps/chosen": -0.7943359613418579,
      "logps/rejected": -1.5021483898162842,
      "loss": 1.0018,
      "nll_loss": 0.961718738079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07943115383386612,
      "rewards/margins": 0.07076416164636612,
      "rewards/rejected": -0.15034179389476776,
      "step": 1890
    },
    {
      "epoch": 0.24519292811975738,
      "grad_norm": 2.0434413971587087,
      "learning_rate": 1.8353258709644938e-06,
      "log_odds_chosen": 1.327978491783142,
      "log_odds_ratio": -0.4107421934604645,
      "logits/chosen": -1.1076171398162842,
      "logits/rejected": -0.9898437261581421,
      "logps/chosen": -0.6957031488418579,
      "logps/rejected": -1.6242187023162842,
      "loss": 0.9972,
      "nll_loss": 0.9365234375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06962890923023224,
      "rewards/margins": 0.0927886962890625,
      "rewards/rejected": -0.16242675483226776,
      "step": 1900
    },
    {
      "epoch": 0.24648341721512454,
      "grad_norm": 2.267804489776882,
      "learning_rate": 1.830515047825102e-06,
      "log_odds_chosen": 1.5583984851837158,
      "log_odds_ratio": -0.392578125,
      "logits/chosen": -1.150390625,
      "logits/rejected": -1.021093726158142,
      "logps/chosen": -0.7173827886581421,
      "logps/rejected": -1.8386719226837158,
      "loss": 0.99,
      "nll_loss": 0.9306640625,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07178954780101776,
      "rewards/margins": 0.11186523735523224,
      "rewards/rejected": -0.1837158203125,
      "step": 1910
    },
    {
      "epoch": 0.24777390631049168,
      "grad_norm": 1.981433139134881,
      "learning_rate": 1.8257418583505536e-06,
      "log_odds_chosen": 1.4382812976837158,
      "log_odds_ratio": -0.3760742247104645,
      "logits/chosen": -1.067968726158142,
      "logits/rejected": -0.9517577886581421,
      "logps/chosen": -0.7066406011581421,
      "logps/rejected": -1.7453124523162842,
      "loss": 0.9942,
      "nll_loss": 1.0226562023162842,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.07064209133386612,
      "rewards/margins": 0.10389403998851776,
      "rewards/rejected": -0.17451171576976776,
      "step": 1920
    },
    {
      "epoch": 0.2490643954058588,
      "grad_norm": 2.3292566183517853,
      "learning_rate": 1.8210058144239416e-06,
      "log_odds_chosen": 1.183007836341858,
      "log_odds_ratio": -0.4544433653354645,
      "logits/chosen": -1.0919921398162842,
      "logits/rejected": -0.9921875,
      "logps/chosen": -0.7113281488418579,
      "logps/rejected": -1.5294921398162842,
      "loss": 0.9805,
      "nll_loss": 0.9683593511581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07108154147863388,
      "rewards/margins": 0.08187256008386612,
      "rewards/rejected": -0.15297850966453552,
      "step": 1930
    },
    {
      "epoch": 0.25035488450122595,
      "grad_norm": 1.889968354069505,
      "learning_rate": 1.816306436745999e-06,
      "log_odds_chosen": 1.2128417491912842,
      "log_odds_ratio": -0.45512694120407104,
      "logits/chosen": -1.0525391101837158,
      "logits/rejected": -0.9605468511581421,
      "logps/chosen": -0.7583984136581421,
      "logps/rejected": -1.622656226158142,
      "loss": 1.002,
      "nll_loss": 1.0128905773162842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07581786811351776,
      "rewards/margins": 0.08653564751148224,
      "rewards/rejected": -0.16240234673023224,
      "step": 1940
    },
    {
      "epoch": 0.2516453735965931,
      "grad_norm": 1.8766147054177726,
      "learning_rate": 1.8116432546313529e-06,
      "log_odds_chosen": 1.2451171875,
      "log_odds_ratio": -0.43110352754592896,
      "logits/chosen": -1.1277344226837158,
      "logits/rejected": -1.0236327648162842,
      "logps/chosen": -0.7176758050918579,
      "logps/rejected": -1.5876953601837158,
      "loss": 0.9953,
      "nll_loss": 0.9300781488418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07174072414636612,
      "rewards/margins": 0.08705444633960724,
      "rewards/rejected": -0.15891113877296448,
      "step": 1950
    },
    {
      "epoch": 0.2529358626919603,
      "grad_norm": 1.7255454976140303,
      "learning_rate": 1.8070158058105026e-06,
      "log_odds_chosen": 1.3191406726837158,
      "log_odds_ratio": -0.43256837129592896,
      "logits/chosen": -1.085351586341858,
      "logits/rejected": -0.963085949420929,
      "logps/chosen": -0.7242187261581421,
      "logps/rejected": -1.7033202648162842,
      "loss": 1.0032,
      "nll_loss": 0.9765625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0723876953125,
      "rewards/margins": 0.09794922173023224,
      "rewards/rejected": -0.17048339545726776,
      "step": 1960
    },
    {
      "epoch": 0.2542263517873274,
      "grad_norm": 1.9668158476690776,
      "learning_rate": 1.8024236362373315e-06,
      "log_odds_chosen": 1.1480712890625,
      "log_odds_ratio": -0.42744141817092896,
      "logits/chosen": -1.112890601158142,
      "logits/rejected": -1.0275390148162842,
      "logps/chosen": -0.7017577886581421,
      "logps/rejected": -1.49609375,
      "loss": 0.9915,
      "nll_loss": 0.9515625238418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07016601413488388,
      "rewards/margins": 0.07947234809398651,
      "rewards/rejected": -0.149658203125,
      "step": 1970
    },
    {
      "epoch": 0.25551684088269455,
      "grad_norm": 2.231921970695306,
      "learning_rate": 1.7978662999019787e-06,
      "log_odds_chosen": 1.562402367591858,
      "log_odds_ratio": -0.39802247285842896,
      "logits/chosen": -1.1164062023162842,
      "logits/rejected": -0.9925781488418579,
      "logps/chosen": -0.7005859613418579,
      "logps/rejected": -1.827734351158142,
      "loss": 0.9954,
      "nll_loss": 0.946093738079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07014159858226776,
      "rewards/margins": 0.11249389499425888,
      "rewards/rejected": -0.18259277939796448,
      "step": 1980
    },
    {
      "epoch": 0.25680732997806166,
      "grad_norm": 1.8469423778860274,
      "learning_rate": 1.793343358648881e-06,
      "log_odds_chosen": 1.4264647960662842,
      "log_odds_ratio": -0.39082032442092896,
      "logits/chosen": -1.1160156726837158,
      "logits/rejected": -1.0029296875,
      "logps/chosen": -0.7261718511581421,
      "logps/rejected": -1.7599608898162842,
      "loss": 0.9936,
      "nll_loss": 0.9566406011581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07260742038488388,
      "rewards/margins": 0.1033935546875,
      "rewards/rejected": -0.17604979872703552,
      "step": 1990
    },
    {
      "epoch": 0.2580978190734288,
      "grad_norm": 2.038143405170428,
      "learning_rate": 1.7888543819998317e-06,
      "log_odds_chosen": 1.3347656726837158,
      "log_odds_ratio": -0.43254393339157104,
      "logits/chosen": -1.073828101158142,
      "logits/rejected": -0.9839843511581421,
      "logps/chosen": -0.7691406011581421,
      "logps/rejected": -1.7566406726837158,
      "loss": 1.0004,
      "nll_loss": 0.9242187738418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07691650092601776,
      "rewards/margins": 0.09867553412914276,
      "rewards/rejected": -0.17563477158546448,
      "step": 2000
    },
    {
      "epoch": 0.259388308168796,
      "grad_norm": 1.71629593434994,
      "learning_rate": 1.7843989469818819e-06,
      "log_odds_chosen": 1.1843750476837158,
      "log_odds_ratio": -0.47480469942092896,
      "logits/chosen": -1.107812523841858,
      "logits/rejected": -1.0089843273162842,
      "logps/chosen": -0.799609363079071,
      "logps/rejected": -1.648828148841858,
      "loss": 0.9824,
      "nll_loss": 0.985546886920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07999267429113388,
      "rewards/margins": 0.08493652194738388,
      "rewards/rejected": -0.1649169921875,
      "step": 2010
    },
    {
      "epoch": 0.2606787972641631,
      "grad_norm": 1.9980005739190831,
      "learning_rate": 1.779976637959939e-06,
      "log_odds_chosen": 1.375585913658142,
      "log_odds_ratio": -0.37236326932907104,
      "logits/chosen": -1.109765648841858,
      "logits/rejected": -1.0107421875,
      "logps/chosen": -0.6605468988418579,
      "logps/rejected": -1.6023437976837158,
      "loss": 0.995,
      "nll_loss": 0.9058593511581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06601562350988388,
      "rewards/margins": 0.09421386569738388,
      "rewards/rejected": -0.16025391221046448,
      "step": 2020
    },
    {
      "epoch": 0.26196928635953026,
      "grad_norm": 2.317264434242284,
      "learning_rate": 1.7755870464739012e-06,
      "log_odds_chosen": 1.48681640625,
      "log_odds_ratio": -0.4222168028354645,
      "logits/chosen": -1.1083984375,
      "logits/rejected": -0.988085925579071,
      "logps/chosen": -0.6568359136581421,
      "logps/rejected": -1.722265601158142,
      "loss": 0.973,
      "nll_loss": 0.887499988079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06564941257238388,
      "rewards/margins": 0.10652466118335724,
      "rewards/rejected": -0.17231445014476776,
      "step": 2030
    },
    {
      "epoch": 0.2632597754548974,
      "grad_norm": 1.9929324475113108,
      "learning_rate": 1.7712297710801907e-06,
      "log_odds_chosen": 1.116308569908142,
      "log_odds_ratio": -0.47612303495407104,
      "logits/chosen": -1.0869140625,
      "logits/rejected": -0.9847656488418579,
      "logps/chosen": -0.7124999761581421,
      "logps/rejected": -1.4582030773162842,
      "loss": 0.9997,
      "nll_loss": 0.9945312738418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07132568210363388,
      "rewards/margins": 0.074432373046875,
      "rewards/rejected": -0.14572754502296448,
      "step": 2040
    },
    {
      "epoch": 0.26455026455026454,
      "grad_norm": 1.8890787836954452,
      "learning_rate": 1.7669044171975444e-06,
      "log_odds_chosen": 1.2815430164337158,
      "log_odds_ratio": -0.3895507752895355,
      "logits/chosen": -1.1779296398162842,
      "logits/rejected": -1.0498046875,
      "logps/chosen": -0.646289050579071,
      "logps/rejected": -1.5078125,
      "loss": 0.9938,
      "nll_loss": 0.9126952886581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06467285007238388,
      "rewards/margins": 0.08612670749425888,
      "rewards/rejected": -0.15078124403953552,
      "step": 2050
    },
    {
      "epoch": 0.2658407536456317,
      "grad_norm": 1.9073685667302505,
      "learning_rate": 1.7626105969569268e-06,
      "log_odds_chosen": 1.208349585533142,
      "log_odds_ratio": -0.45903319120407104,
      "logits/chosen": -1.0607421398162842,
      "logits/rejected": -0.959765613079071,
      "logps/chosen": -0.728515625,
      "logps/rejected": -1.6189453601837158,
      "loss": 0.9824,
      "nll_loss": 1.029882788658142,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07283935695886612,
      "rewards/margins": 0.08916626125574112,
      "rewards/rejected": -0.16193847358226776,
      "step": 2060
    },
    {
      "epoch": 0.26713124274099886,
      "grad_norm": 1.90905841538505,
      "learning_rate": 1.758347929055432e-06,
      "log_odds_chosen": 1.634765625,
      "log_odds_ratio": -0.38200682401657104,
      "logits/chosen": -1.0802733898162842,
      "logits/rejected": -0.9693359136581421,
      "logps/chosen": -0.775390625,
      "logps/rejected": -2.010546922683716,
      "loss": 1.0004,
      "nll_loss": 0.998828113079071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07758788764476776,
      "rewards/margins": 0.12361755222082138,
      "rewards/rejected": -0.201171875,
      "step": 2070
    },
    {
      "epoch": 0.268421731836366,
      "grad_norm": 1.957762989865166,
      "learning_rate": 1.7541160386140582e-06,
      "log_odds_chosen": 1.34814453125,
      "log_odds_ratio": -0.4408203065395355,
      "logits/chosen": -1.093164086341858,
      "logits/rejected": -0.990039050579071,
      "logps/chosen": -0.7066406011581421,
      "logps/rejected": -1.6982421875,
      "loss": 0.9877,
      "nll_loss": 0.9341796636581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07065429538488388,
      "rewards/margins": 0.099090576171875,
      "rewards/rejected": -0.16982421278953552,
      "step": 2080
    },
    {
      "epoch": 0.26971222093173314,
      "grad_norm": 2.1402202001542743,
      "learning_rate": 1.7499145570392284e-06,
      "log_odds_chosen": 1.1083984375,
      "log_odds_ratio": -0.46757811307907104,
      "logits/chosen": -1.122656226158142,
      "logits/rejected": -1.0089843273162842,
      "logps/chosen": -0.766406238079071,
      "logps/rejected": -1.5546875,
      "loss": 0.9992,
      "nll_loss": 1.022070288658142,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07662353664636612,
      "rewards/margins": 0.078826904296875,
      "rewards/rejected": -0.15541991591453552,
      "step": 2090
    },
    {
      "epoch": 0.27100271002710025,
      "grad_norm": 2.002304987533016,
      "learning_rate": 1.745743121887939e-06,
      "log_odds_chosen": 1.13037109375,
      "log_odds_ratio": -0.46308594942092896,
      "logits/chosen": -1.0751953125,
      "logits/rejected": -0.9867187738418579,
      "logps/chosen": -0.729687511920929,
      "logps/rejected": -1.493554711341858,
      "loss": 0.998,
      "nll_loss": 0.956835925579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07301025092601776,
      "rewards/margins": 0.07625122368335724,
      "rewards/rejected": -0.14934082329273224,
      "step": 2100
    },
    {
      "epoch": 0.2722931991224674,
      "grad_norm": 1.9860846185121883,
      "learning_rate": 1.7416013767364324e-06,
      "log_odds_chosen": 1.156396508216858,
      "log_odds_ratio": -0.41508787870407104,
      "logits/chosen": -1.096289038658142,
      "logits/rejected": -0.9701172113418579,
      "logps/chosen": -0.6795898675918579,
      "logps/rejected": -1.467382788658142,
      "loss": 0.9833,
      "nll_loss": 0.9320312738418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06791992485523224,
      "rewards/margins": 0.07879028469324112,
      "rewards/rejected": -0.146728515625,
      "step": 2110
    },
    {
      "epoch": 0.2735836882178346,
      "grad_norm": 1.941539192385113,
      "learning_rate": 1.7374889710522776e-06,
      "log_odds_chosen": 1.4441406726837158,
      "log_odds_ratio": -0.4044433534145355,
      "logits/chosen": -1.066992163658142,
      "logits/rejected": -0.959179699420929,
      "logps/chosen": -0.6910156011581421,
      "logps/rejected": -1.68359375,
      "loss": 0.9656,
      "nll_loss": 0.9564453363418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06910400092601776,
      "rewards/margins": 0.09940185397863388,
      "rewards/rejected": -0.16855469346046448,
      "step": 2120
    },
    {
      "epoch": 0.2748741773132017,
      "grad_norm": 2.041060423732975,
      "learning_rate": 1.7334055600697579e-06,
      "log_odds_chosen": 1.3286864757537842,
      "log_odds_ratio": -0.42167967557907104,
      "logits/chosen": -1.083593726158142,
      "logits/rejected": -0.9632812738418579,
      "logps/chosen": -0.724316418170929,
      "logps/rejected": -1.669335961341858,
      "loss": 0.9808,
      "nll_loss": 0.98046875,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07236327975988388,
      "rewards/margins": 0.09449615329504013,
      "rewards/rejected": -0.16689452528953552,
      "step": 2130
    },
    {
      "epoch": 0.27616466640856885,
      "grad_norm": 1.9075370731392578,
      "learning_rate": 1.7293508046684678e-06,
      "log_odds_chosen": 1.3720703125,
      "log_odds_ratio": -0.4010253846645355,
      "logits/chosen": -1.0968749523162842,
      "logits/rejected": -1.002539038658142,
      "logps/chosen": -0.638378918170929,
      "logps/rejected": -1.544531226158142,
      "loss": 0.9895,
      "nll_loss": 0.892773449420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06385497748851776,
      "rewards/margins": 0.090576171875,
      "rewards/rejected": -0.15432128310203552,
      "step": 2140
    },
    {
      "epoch": 0.277455155503936,
      "grad_norm": 2.009234180149321,
      "learning_rate": 1.7253243712550145e-06,
      "log_odds_chosen": 1.4076049327850342,
      "log_odds_ratio": -0.3729248046875,
      "logits/chosen": -1.134374976158142,
      "logits/rejected": -1.0076172351837158,
      "logps/chosen": -0.7066406011581421,
      "logps/rejected": -1.675390601158142,
      "loss": 0.9658,
      "nll_loss": 0.8666015863418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.0706787109375,
      "rewards/margins": 0.09674682468175888,
      "rewards/rejected": -0.16745606064796448,
      "step": 2150
    },
    {
      "epoch": 0.2787456445993031,
      "grad_norm": 2.076459027835872,
      "learning_rate": 1.7213259316477406e-06,
      "log_odds_chosen": 1.4539062976837158,
      "log_odds_ratio": -0.4154296815395355,
      "logits/chosen": -1.109375,
      "logits/rejected": -0.991015613079071,
      "logps/chosen": -0.7037109136581421,
      "logps/rejected": -1.78125,
      "loss": 0.9961,
      "nll_loss": 0.9722656011581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07033691555261612,
      "rewards/margins": 0.10784912109375,
      "rewards/rejected": -0.17812499403953552,
      "step": 2160
    },
    {
      "epoch": 0.2800361336946703,
      "grad_norm": 2.1798826672938527,
      "learning_rate": 1.7173551629643674e-06,
      "log_odds_chosen": 1.5755615234375,
      "log_odds_ratio": -0.3691650331020355,
      "logits/chosen": -1.084570288658142,
      "logits/rejected": -0.9810546636581421,
      "logps/chosen": -0.673535168170929,
      "logps/rejected": -1.8083984851837158,
      "loss": 0.9786,
      "nll_loss": 0.921093761920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06733398139476776,
      "rewards/margins": 0.11351928859949112,
      "rewards/rejected": -0.18081054091453552,
      "step": 2170
    },
    {
      "epoch": 0.28132662279003745,
      "grad_norm": 1.795091926253059,
      "learning_rate": 1.713411747512477e-06,
      "log_odds_chosen": 1.31591796875,
      "log_odds_ratio": -0.40410155057907104,
      "logits/chosen": -1.0662109851837158,
      "logits/rejected": -0.9593750238418579,
      "logps/chosen": -0.712597668170929,
      "logps/rejected": -1.649023413658142,
      "loss": 0.9843,
      "nll_loss": 0.9599609375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07132568210363388,
      "rewards/margins": 0.09360351413488388,
      "rewards/rejected": -0.16486816108226776,
      "step": 2180
    },
    {
      "epoch": 0.28261711188540456,
      "grad_norm": 2.023036255381263,
      "learning_rate": 1.709495372682753e-06,
      "log_odds_chosen": 1.247705101966858,
      "log_odds_ratio": -0.45673829317092896,
      "logits/chosen": -1.0818359851837158,
      "logits/rejected": -0.9613281488418579,
      "logps/chosen": -0.740039050579071,
      "logps/rejected": -1.6472656726837158,
      "loss": 0.9746,
      "nll_loss": 0.958984375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07406006008386612,
      "rewards/margins": 0.09071044623851776,
      "rewards/rejected": -0.16472168266773224,
      "step": 2190
    },
    {
      "epoch": 0.2839076009807717,
      "grad_norm": 2.078865384482121,
      "learning_rate": 1.7056057308448832e-06,
      "log_odds_chosen": 1.3982422351837158,
      "log_odds_ratio": -0.4281249940395355,
      "logits/chosen": -1.0880858898162842,
      "logits/rejected": -0.961718738079071,
      "logps/chosen": -0.728808581829071,
      "logps/rejected": -1.7468750476837158,
      "loss": 0.9725,
      "nll_loss": 0.963085949420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07286377251148224,
      "rewards/margins": 0.10177002102136612,
      "rewards/rejected": -0.17460937798023224,
      "step": 2200
    },
    {
      "epoch": 0.28519809007613883,
      "grad_norm": 2.3646280577802488,
      "learning_rate": 1.701742519246068e-06,
      "log_odds_chosen": 1.3142578601837158,
      "log_odds_ratio": -0.4349121153354645,
      "logits/chosen": -1.0906250476837158,
      "logits/rejected": -0.9798828363418579,
      "logps/chosen": -0.720507800579071,
      "logps/rejected": -1.6691405773162842,
      "loss": 0.9412,
      "nll_loss": 0.9482421875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.072021484375,
      "rewards/margins": 0.09483642876148224,
      "rewards/rejected": -0.16677245497703552,
      "step": 2210
    },
    {
      "epoch": 0.286488579171506,
      "grad_norm": 2.084125375167007,
      "learning_rate": 1.6979054399120355e-06,
      "log_odds_chosen": 0.9654785394668579,
      "log_odds_ratio": -0.48759764432907104,
      "logits/chosen": -1.035742163658142,
      "logits/rejected": -0.9609375,
      "logps/chosen": -0.7074218988418579,
      "logps/rejected": -1.400390625,
      "loss": 0.9759,
      "nll_loss": 0.9326171875,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07078857719898224,
      "rewards/margins": 0.06927490234375,
      "rewards/rejected": -0.14005127549171448,
      "step": 2220
    },
    {
      "epoch": 0.28777906826687316,
      "grad_norm": 1.8961502767519962,
      "learning_rate": 1.6940941995505069e-06,
      "log_odds_chosen": 1.5011718273162842,
      "log_odds_ratio": -0.40185546875,
      "logits/chosen": -1.0439453125,
      "logits/rejected": -0.9271484613418579,
      "logps/chosen": -0.7464843988418579,
      "logps/rejected": -1.8503906726837158,
      "loss": 1.0058,
      "nll_loss": 1.0031249523162842,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07462158054113388,
      "rewards/margins": 0.110107421875,
      "rewards/rejected": -0.184814453125,
      "step": 2230
    },
    {
      "epoch": 0.28906955736224027,
      "grad_norm": 1.9331323663213622,
      "learning_rate": 1.6903085094570331e-06,
      "log_odds_chosen": 1.3385741710662842,
      "log_odds_ratio": -0.39409178495407104,
      "logits/chosen": -1.103124976158142,
      "logits/rejected": -1.001562476158142,
      "logps/chosen": -0.69140625,
      "logps/rejected": -1.603906273841858,
      "loss": 0.9742,
      "nll_loss": 0.8929687738418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06910400092601776,
      "rewards/margins": 0.091278076171875,
      "rewards/rejected": -0.16042479872703552,
      "step": 2240
    },
    {
      "epoch": 0.29036004645760743,
      "grad_norm": 2.2015718361659884,
      "learning_rate": 1.6865480854231356e-06,
      "log_odds_chosen": 1.2791016101837158,
      "log_odds_ratio": -0.427734375,
      "logits/chosen": -1.0958983898162842,
      "logits/rejected": -0.9876953363418579,
      "logps/chosen": -0.685546875,
      "logps/rejected": -1.5460937023162842,
      "loss": 0.979,
      "nll_loss": 0.9208984375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06864013522863388,
      "rewards/margins": 0.08588256686925888,
      "rewards/rejected": -0.15451660752296448,
      "step": 2250
    },
    {
      "epoch": 0.2916505355529746,
      "grad_norm": 1.8472525293505426,
      "learning_rate": 1.682812647646685e-06,
      "log_odds_chosen": 1.1560547351837158,
      "log_odds_ratio": -0.4437499940395355,
      "logits/chosen": -1.047460913658142,
      "logits/rejected": -0.930859386920929,
      "logps/chosen": -0.7646484375,
      "logps/rejected": -1.593359351158142,
      "loss": 0.9735,
      "nll_loss": 0.984179675579071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.0765380859375,
      "rewards/margins": 0.08292236179113388,
      "rewards/rejected": -0.15944823622703552,
      "step": 2260
    },
    {
      "epoch": 0.2929410246483417,
      "grad_norm": 2.030898313924733,
      "learning_rate": 1.6791019206444541e-06,
      "log_odds_chosen": 1.4142577648162842,
      "log_odds_ratio": -0.4139160215854645,
      "logits/chosen": -1.0822265148162842,
      "logits/rejected": -0.9888671636581421,
      "logps/chosen": -0.69580078125,
      "logps/rejected": -1.6730468273162842,
      "loss": 0.9723,
      "nll_loss": 0.964062511920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06966552883386612,
      "rewards/margins": 0.09755249321460724,
      "rewards/rejected": -0.16721192002296448,
      "step": 2270
    },
    {
      "epoch": 0.2942315137437089,
      "grad_norm": 2.0596262602857136,
      "learning_rate": 1.675415633166782e-06,
      "log_odds_chosen": 1.4601562023162842,
      "log_odds_ratio": -0.42387694120407104,
      "logits/chosen": -1.103515625,
      "logits/rejected": -1.004492163658142,
      "logps/chosen": -0.732421875,
      "logps/rejected": -1.834570288658142,
      "loss": 0.9692,
      "nll_loss": 0.944140613079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07325439155101776,
      "rewards/margins": 0.1102294921875,
      "rewards/rejected": -0.18364258110523224,
      "step": 2280
    },
    {
      "epoch": 0.29552200283907604,
      "grad_norm": 2.1963128963170924,
      "learning_rate": 1.6717535181142914e-06,
      "log_odds_chosen": 1.4229004383087158,
      "log_odds_ratio": -0.43583983182907104,
      "logits/chosen": -1.136132836341858,
      "logits/rejected": -1.0197265148162842,
      "logps/chosen": -0.7294921875,
      "logps/rejected": -1.779296875,
      "loss": 0.9696,
      "nll_loss": 0.9251953363418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07294921576976776,
      "rewards/margins": 0.10507659614086151,
      "rewards/rejected": -0.17805175483226776,
      "step": 2290
    },
    {
      "epoch": 0.29681249193444315,
      "grad_norm": 1.9770195618619957,
      "learning_rate": 1.668115312456598e-06,
      "log_odds_chosen": 1.514257788658142,
      "log_odds_ratio": -0.4085937440395355,
      "logits/chosen": -1.081445336341858,
      "logits/rejected": -0.976367175579071,
      "logps/chosen": -0.7177734375,
      "logps/rejected": -1.791406273841858,
      "loss": 0.9757,
      "nll_loss": 0.9537109136581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07177734375,
      "rewards/margins": 0.10747680813074112,
      "rewards/rejected": -0.17917481064796448,
      "step": 2300
    },
    {
      "epoch": 0.2981029810298103,
      "grad_norm": 1.977538510281707,
      "learning_rate": 1.6645007571529578e-06,
      "log_odds_chosen": 1.5125000476837158,
      "log_odds_ratio": -0.37763673067092896,
      "logits/chosen": -1.107031226158142,
      "logits/rejected": -1.0048828125,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -1.837890625,
      "loss": 0.9658,
      "nll_loss": 0.921679675579071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07163085788488388,
      "rewards/margins": 0.11224365234375,
      "rewards/rejected": -0.183837890625,
      "step": 2310
    },
    {
      "epoch": 0.2993934701251774,
      "grad_norm": 1.8924266379133956,
      "learning_rate": 1.6609095970747992e-06,
      "log_odds_chosen": 1.3981444835662842,
      "log_odds_ratio": -0.43706053495407104,
      "logits/chosen": -1.08984375,
      "logits/rejected": -0.9974609613418579,
      "logps/chosen": -0.72021484375,
      "logps/rejected": -1.742578148841858,
      "loss": 0.9503,
      "nll_loss": 0.955859363079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07196044921875,
      "rewards/margins": 0.10220947116613388,
      "rewards/rejected": -0.1741943359375,
      "step": 2320
    },
    {
      "epoch": 0.3006839592205446,
      "grad_norm": 1.958436971448942,
      "learning_rate": 1.6573415809300833e-06,
      "log_odds_chosen": 1.3729979991912842,
      "log_odds_ratio": -0.3827148377895355,
      "logits/chosen": -1.089257836341858,
      "logits/rejected": -0.975390613079071,
      "logps/chosen": -0.636425793170929,
      "logps/rejected": -1.565820336341858,
      "loss": 0.9492,
      "nll_loss": 0.8921874761581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06365966796875,
      "rewards/margins": 0.09296264499425888,
      "rewards/rejected": -0.15666504204273224,
      "step": 2330
    },
    {
      "epoch": 0.30197444831591175,
      "grad_norm": 2.125568710977181,
      "learning_rate": 1.6537964611894462e-06,
      "log_odds_chosen": 1.3732421398162842,
      "log_odds_ratio": -0.45263671875,
      "logits/chosen": -1.036718726158142,
      "logits/rejected": -0.934765636920929,
      "logps/chosen": -0.727832019329071,
      "logps/rejected": -1.677734375,
      "loss": 0.9712,
      "nll_loss": 0.948046863079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07275390625,
      "rewards/margins": 0.09508056938648224,
      "rewards/rejected": -0.16779784858226776,
      "step": 2340
    },
    {
      "epoch": 0.30326493741127886,
      "grad_norm": 2.008929483801467,
      "learning_rate": 1.6502739940140692e-06,
      "log_odds_chosen": 1.60546875,
      "log_odds_ratio": -0.38325196504592896,
      "logits/chosen": -1.0720703601837158,
      "logits/rejected": -0.9634765386581421,
      "logps/chosen": -0.6961914300918579,
      "logps/rejected": -1.87109375,
      "loss": 0.9474,
      "nll_loss": 0.936718761920929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06966552883386612,
      "rewards/margins": 0.11756591498851776,
      "rewards/rejected": -0.18728026747703552,
      "step": 2350
    },
    {
      "epoch": 0.304555426506646,
      "grad_norm": 1.9511623937681515,
      "learning_rate": 1.6467739391852364e-06,
      "log_odds_chosen": 1.6203124523162842,
      "log_odds_ratio": -0.33740234375,
      "logits/chosen": -1.0910155773162842,
      "logits/rejected": -0.9515625238418579,
      "logps/chosen": -0.6787109375,
      "logps/rejected": -1.7833983898162842,
      "loss": 0.9498,
      "nll_loss": 0.9017578363418579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06787109375,
      "rewards/margins": 0.11066436767578125,
      "rewards/rejected": -0.17844238877296448,
      "step": 2360
    },
    {
      "epoch": 0.3058459156020132,
      "grad_norm": 1.923801220210875,
      "learning_rate": 1.6432960600355221e-06,
      "log_odds_chosen": 1.0947754383087158,
      "log_odds_ratio": -0.5220702886581421,
      "logits/chosen": -1.0861327648162842,
      "logits/rejected": -0.970703125,
      "logps/chosen": -0.74609375,
      "logps/rejected": -1.5314452648162842,
      "loss": 0.9367,
      "nll_loss": 0.9248046875,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07457275688648224,
      "rewards/margins": 0.07846679538488388,
      "rewards/rejected": -0.15302734076976776,
      "step": 2370
    },
    {
      "epoch": 0.3071364046973803,
      "grad_norm": 2.1016509843085585,
      "learning_rate": 1.6398401233815756e-06,
      "log_odds_chosen": 1.4208495616912842,
      "log_odds_ratio": -0.3995117247104645,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.9154297113418579,
      "logps/chosen": -0.664746105670929,
      "logps/rejected": -1.6884765625,
      "loss": 0.9759,
      "nll_loss": 0.93359375,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06647948920726776,
      "rewards/margins": 0.102447509765625,
      "rewards/rejected": -0.16889648139476776,
      "step": 2380
    },
    {
      "epoch": 0.30842689379274746,
      "grad_norm": 1.9301575415109438,
      "learning_rate": 1.6364058994584524e-06,
      "log_odds_chosen": 1.3161132335662842,
      "log_odds_ratio": -0.41938477754592896,
      "logits/chosen": -1.06640625,
      "logits/rejected": -0.9398437738418579,
      "logps/chosen": -0.7529296875,
      "logps/rejected": -1.701562523841858,
      "loss": 0.9669,
      "nll_loss": 0.9623047113418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07530517876148224,
      "rewards/margins": 0.0947723388671875,
      "rewards/rejected": -0.17009277641773224,
      "step": 2390
    },
    {
      "epoch": 0.3097173828881146,
      "grad_norm": 13.056396738927718,
      "learning_rate": 1.6329931618554522e-06,
      "log_odds_chosen": 1.580810546875,
      "log_odds_ratio": -0.36767578125,
      "logits/chosen": -1.100000023841858,
      "logits/rejected": -0.9912109375,
      "logps/chosen": -0.6796875,
      "logps/rejected": -1.8244140148162842,
      "loss": 0.9382,
      "nll_loss": 0.9085937738418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06802978366613388,
      "rewards/margins": 0.11450805515050888,
      "rewards/rejected": -0.18239745497703552,
      "step": 2400
    },
    {
      "epoch": 0.31100787198348173,
      "grad_norm": 1.9559474158813306,
      "learning_rate": 1.6296016874534209e-06,
      "log_odds_chosen": 1.3899414539337158,
      "log_odds_ratio": -0.44853514432907104,
      "logits/chosen": -1.04296875,
      "logits/rejected": -0.937304675579071,
      "logps/chosen": -0.717578113079071,
      "logps/rejected": -1.7421875,
      "loss": 0.9804,
      "nll_loss": 0.9375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07175292819738388,
      "rewards/margins": 0.10238037258386612,
      "rewards/rejected": -0.17431640625,
      "step": 2410
    },
    {
      "epoch": 0.3122983610788489,
      "grad_norm": 2.0843655293309236,
      "learning_rate": 1.6262312563634835e-06,
      "log_odds_chosen": 1.432275414466858,
      "log_odds_ratio": -0.38447266817092896,
      "logits/chosen": -1.0558593273162842,
      "logits/rejected": -0.962890625,
      "logps/chosen": -0.6456054449081421,
      "logps/rejected": -1.6066405773162842,
      "loss": 0.9409,
      "nll_loss": 0.8480468988418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06456299126148224,
      "rewards/margins": 0.09611205756664276,
      "rewards/rejected": -0.16054686903953552,
      "step": 2420
    },
    {
      "epoch": 0.313588850174216,
      "grad_norm": 1.9819790974241083,
      "learning_rate": 1.6228816518671587e-06,
      "log_odds_chosen": 1.2680175304412842,
      "log_odds_ratio": -0.42973631620407104,
      "logits/chosen": -1.0158202648162842,
      "logits/rejected": -0.9273437261581421,
      "logps/chosen": -0.7051757574081421,
      "logps/rejected": -1.5701172351837158,
      "loss": 0.9844,
      "nll_loss": 0.9564453363418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07050780951976776,
      "rewards/margins": 0.08645935356616974,
      "rewards/rejected": -0.15688475966453552,
      "step": 2430
    },
    {
      "epoch": 0.31487933926958317,
      "grad_norm": 2.0828854961588905,
      "learning_rate": 1.619552660357832e-06,
      "log_odds_chosen": 1.448828101158142,
      "log_odds_ratio": -0.3851074278354645,
      "logits/chosen": -1.0400390625,
      "logits/rejected": -0.9501953125,
      "logps/chosen": -0.710156261920929,
      "logps/rejected": -1.747656226158142,
      "loss": 0.9506,
      "nll_loss": 0.9249023199081421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07102050632238388,
      "rewards/margins": 0.10364989936351776,
      "rewards/rejected": -0.17475585639476776,
      "step": 2440
    },
    {
      "epoch": 0.31616982836495033,
      "grad_norm": 2.3897246020316696,
      "learning_rate": 1.616244071283537e-06,
      "log_odds_chosen": 1.4738280773162842,
      "log_odds_ratio": -0.403076171875,
      "logits/chosen": -1.1466796398162842,
      "logits/rejected": -1.009374976158142,
      "logps/chosen": -0.7158203125,
      "logps/rejected": -1.829492211341858,
      "loss": 0.9458,
      "nll_loss": 0.8531249761581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07154540717601776,
      "rewards/margins": 0.11129150539636612,
      "rewards/rejected": -0.18281249701976776,
      "step": 2450
    },
    {
      "epoch": 0.31746031746031744,
      "grad_norm": 2.120436053991648,
      "learning_rate": 1.6129556770910235e-06,
      "log_odds_chosen": 1.2205078601837158,
      "log_odds_ratio": -0.449462890625,
      "logits/chosen": -1.037109375,
      "logits/rejected": -0.9427734613418579,
      "logps/chosen": -0.6923828125,
      "logps/rejected": -1.5505859851837158,
      "loss": 0.9628,
      "nll_loss": 0.952929675579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06923828274011612,
      "rewards/margins": 0.08575439453125,
      "rewards/rejected": -0.15500488877296448,
      "step": 2460
    },
    {
      "epoch": 0.3187508065556846,
      "grad_norm": 2.1947110111241424,
      "learning_rate": 1.6096872731710673e-06,
      "log_odds_chosen": 1.4088866710662842,
      "log_odds_ratio": -0.404052734375,
      "logits/chosen": -1.0263671875,
      "logits/rejected": -0.9046875238418579,
      "logps/chosen": -0.6685546636581421,
      "logps/rejected": -1.6818358898162842,
      "loss": 0.9552,
      "nll_loss": 0.8578125238418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06684570014476776,
      "rewards/margins": 0.10125732421875,
      "rewards/rejected": -0.16818848252296448,
      "step": 2470
    },
    {
      "epoch": 0.32004129565105177,
      "grad_norm": 2.2200342193498868,
      "learning_rate": 1.6064386578049978e-06,
      "log_odds_chosen": 1.5285155773162842,
      "log_odds_ratio": -0.3631347715854645,
      "logits/chosen": -1.00390625,
      "logits/rejected": -0.892382800579071,
      "logps/chosen": -0.645703136920929,
      "logps/rejected": -1.705468773841858,
      "loss": 0.9409,
      "nll_loss": 0.9273437261581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06455077975988388,
      "rewards/margins": 0.10594482719898224,
      "rewards/rejected": -0.17060546576976776,
      "step": 2480
    },
    {
      "epoch": 0.3213317847464189,
      "grad_norm": 2.2304061145062986,
      "learning_rate": 1.6032096321124046e-06,
      "log_odds_chosen": 1.397851586341858,
      "log_odds_ratio": -0.445556640625,
      "logits/chosen": -1.0832030773162842,
      "logits/rejected": -0.995312511920929,
      "logps/chosen": -0.688281238079071,
      "logps/rejected": -1.658203125,
      "loss": 0.9473,
      "nll_loss": 0.901562511920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06884765625,
      "rewards/margins": 0.09713134914636612,
      "rewards/rejected": -0.1658935546875,
      "step": 2490
    },
    {
      "epoch": 0.32262227384178604,
      "grad_norm": 2.377947714091295,
      "learning_rate": 1.6e-06,
      "log_odds_chosen": 1.6448242664337158,
      "log_odds_ratio": -0.3722167909145355,
      "logits/chosen": -1.078515648841858,
      "logits/rejected": -0.9566406011581421,
      "logps/chosen": -0.6698242425918579,
      "logps/rejected": -1.8380858898162842,
      "loss": 0.9403,
      "nll_loss": 0.944531261920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06699218600988388,
      "rewards/margins": 0.1168212890625,
      "rewards/rejected": -0.18391112983226776,
      "step": 2500
    },
    {
      "epoch": 0.32391276293715315,
      "grad_norm": 2.300392004036583,
      "learning_rate": 1.5968095681115984e-06,
      "log_odds_chosen": 1.3513672351837158,
      "log_odds_ratio": -0.4256347715854645,
      "logits/chosen": -1.0927734375,
      "logits/rejected": -0.974414050579071,
      "logps/chosen": -0.7544921636581421,
      "logps/rejected": -1.7296874523162842,
      "loss": 0.94,
      "nll_loss": 0.9166015386581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07550048828125,
      "rewards/margins": 0.09747314453125,
      "rewards/rejected": -0.1729736328125,
      "step": 2510
    },
    {
      "epoch": 0.3252032520325203,
      "grad_norm": 2.0674390296944356,
      "learning_rate": 1.5936381457791914e-06,
      "log_odds_chosen": 1.3544921875,
      "log_odds_ratio": -0.38422852754592896,
      "logits/chosen": -1.034570336341858,
      "logits/rejected": -0.9345703125,
      "logps/chosen": -0.6820312738418579,
      "logps/rejected": -1.6173827648162842,
      "loss": 0.9507,
      "nll_loss": 0.930859386920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06826172024011612,
      "rewards/margins": 0.093475341796875,
      "rewards/rejected": -0.16164550185203552,
      "step": 2520
    },
    {
      "epoch": 0.3264937411278875,
      "grad_norm": 1.9172736536656358,
      "learning_rate": 1.590485544975088e-06,
      "log_odds_chosen": 1.3621094226837158,
      "log_odds_ratio": -0.388916015625,
      "logits/chosen": -1.0183594226837158,
      "logits/rejected": -0.913281261920929,
      "logps/chosen": -0.7374023199081421,
      "logps/rejected": -1.6902344226837158,
      "loss": 0.9323,
      "nll_loss": 0.9068359136581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07370605319738388,
      "rewards/margins": 0.095245361328125,
      "rewards/rejected": -0.1689453125,
      "step": 2530
    },
    {
      "epoch": 0.3277842302232546,
      "grad_norm": 3.2126149753454074,
      "learning_rate": 1.5873515802650901e-06,
      "log_odds_chosen": 1.1038818359375,
      "log_odds_ratio": -0.455322265625,
      "logits/chosen": -1.0128905773162842,
      "logits/rejected": -0.925585925579071,
      "logps/chosen": -0.6839843988418579,
      "logps/rejected": -1.4599609375,
      "loss": 0.9788,
      "nll_loss": 0.9320312738418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.068359375,
      "rewards/margins": 0.07758941501379013,
      "rewards/rejected": -0.14609375596046448,
      "step": 2540
    },
    {
      "epoch": 0.32907471931862176,
      "grad_norm": 2.0434255775968015,
      "learning_rate": 1.584236068762679e-06,
      "log_odds_chosen": 1.182373046875,
      "log_odds_ratio": -0.46088868379592896,
      "logits/chosen": -1.0400390625,
      "logits/rejected": -0.9359375238418579,
      "logps/chosen": -0.727734386920929,
      "logps/rejected": -1.5705077648162842,
      "loss": 0.9331,
      "nll_loss": 0.8892577886581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07279052585363388,
      "rewards/margins": 0.08410187065601349,
      "rewards/rejected": -0.15703125298023224,
      "step": 2550
    },
    {
      "epoch": 0.3303652084139889,
      "grad_norm": 2.029179533722563,
      "learning_rate": 1.5811388300841894e-06,
      "log_odds_chosen": 1.392675757408142,
      "log_odds_ratio": -0.39892578125,
      "logits/chosen": -1.053125023841858,
      "logits/rejected": -0.9058593511581421,
      "logps/chosen": -0.668749988079071,
      "logps/rejected": -1.6613280773162842,
      "loss": 0.9747,
      "nll_loss": 0.8890625238418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06683349609375,
      "rewards/margins": 0.09939269721508026,
      "rewards/rejected": -0.16616210341453552,
      "step": 2560
    },
    {
      "epoch": 0.33165569750935603,
      "grad_norm": 2.2390783632310893,
      "learning_rate": 1.5780596863049431e-06,
      "log_odds_chosen": 1.551171898841858,
      "log_odds_ratio": -0.37255859375,
      "logits/chosen": -1.0373046398162842,
      "logits/rejected": -0.936328113079071,
      "logps/chosen": -0.676953136920929,
      "logps/rejected": -1.771875023841858,
      "loss": 0.9335,
      "nll_loss": 0.8804687261581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06771240383386612,
      "rewards/margins": 0.10931396484375,
      "rewards/rejected": -0.177001953125,
      "step": 2570
    },
    {
      "epoch": 0.3329461866047232,
      "grad_norm": 1.9841917589839912,
      "learning_rate": 1.5749984619163156e-06,
      "log_odds_chosen": 1.435546875,
      "log_odds_ratio": -0.37641602754592896,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.9242187738418579,
      "logps/chosen": -0.7259765863418579,
      "logps/rejected": -1.738671898841858,
      "loss": 0.9478,
      "nll_loss": 0.870898425579071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07257080078125,
      "rewards/margins": 0.10120849311351776,
      "rewards/rejected": -0.173828125,
      "step": 2580
    },
    {
      "epoch": 0.33423667570009036,
      "grad_norm": 1.950501641004389,
      "learning_rate": 1.5719549837837187e-06,
      "log_odds_chosen": 1.3630859851837158,
      "log_odds_ratio": -0.3977294862270355,
      "logits/chosen": -1.0480468273162842,
      "logits/rejected": -0.9173828363418579,
      "logps/chosen": -0.6478515863418579,
      "logps/rejected": -1.587890625,
      "loss": 0.96,
      "nll_loss": 0.866992175579071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06483153998851776,
      "rewards/margins": 0.09396972507238388,
      "rewards/rejected": -0.15886230766773224,
      "step": 2590
    },
    {
      "epoch": 0.33552716479545747,
      "grad_norm": 2.187898097698019,
      "learning_rate": 1.5689290811054722e-06,
      "log_odds_chosen": 1.5107421875,
      "log_odds_ratio": -0.3720703125,
      "logits/chosen": -1.0431640148162842,
      "logits/rejected": -0.943359375,
      "logps/chosen": -0.6474609375,
      "logps/rejected": -1.732031226158142,
      "loss": 0.9185,
      "nll_loss": 0.856640636920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06472168117761612,
      "rewards/margins": 0.10847167670726776,
      "rewards/rejected": -0.17324218153953552,
      "step": 2600
    },
    {
      "epoch": 0.33681765389082463,
      "grad_norm": 2.003002173030459,
      "learning_rate": 1.5659205853725426e-06,
      "log_odds_chosen": 1.217187523841858,
      "log_odds_ratio": -0.4754394590854645,
      "logits/chosen": -1.049414038658142,
      "logits/rejected": -0.9453125,
      "logps/chosen": -0.708984375,
      "logps/rejected": -1.563085913658142,
      "loss": 0.9603,
      "nll_loss": 0.975781261920929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07084961235523224,
      "rewards/margins": 0.08536987006664276,
      "rewards/rejected": -0.15625,
      "step": 2610
    },
    {
      "epoch": 0.33810814298619174,
      "grad_norm": 2.2218005545165775,
      "learning_rate": 1.562929330329127e-06,
      "log_odds_chosen": 1.262109398841858,
      "log_odds_ratio": -0.4210449159145355,
      "logits/chosen": -1.108007788658142,
      "logits/rejected": -0.9794921875,
      "logps/chosen": -0.6962890625,
      "logps/rejected": -1.572265625,
      "loss": 0.9357,
      "nll_loss": 0.820117175579071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.069580078125,
      "rewards/margins": 0.0875244140625,
      "rewards/rejected": -0.15717773139476776,
      "step": 2620
    },
    {
      "epoch": 0.3393986320815589,
      "grad_norm": 2.070099973752578,
      "learning_rate": 1.5599551519340636e-06,
      "log_odds_chosen": 1.4599609375,
      "log_odds_ratio": -0.35771483182907104,
      "logits/chosen": -1.072265625,
      "logits/rejected": -0.92578125,
      "logps/chosen": -0.687304675579071,
      "logps/rejected": -1.685937523841858,
      "loss": 0.9409,
      "nll_loss": 0.9339843988418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06877441704273224,
      "rewards/margins": 0.09991455078125,
      "rewards/rejected": -0.1685791015625,
      "step": 2630
    },
    {
      "epoch": 0.34068912117692607,
      "grad_norm": 2.024766874149091,
      "learning_rate": 1.556997888323046e-06,
      "log_odds_chosen": 1.4877440929412842,
      "log_odds_ratio": -0.4181152284145355,
      "logits/chosen": -1.0701172351837158,
      "logits/rejected": -0.947460949420929,
      "logps/chosen": -0.738476574420929,
      "logps/rejected": -1.84765625,
      "loss": 0.9304,
      "nll_loss": 0.8177734613418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07384033501148224,
      "rewards/margins": 0.111114501953125,
      "rewards/rejected": -0.18498535454273224,
      "step": 2640
    },
    {
      "epoch": 0.3419796102722932,
      "grad_norm": 2.783882454055213,
      "learning_rate": 1.5540573797716226e-06,
      "log_odds_chosen": 1.6238281726837158,
      "log_odds_ratio": -0.3966064453125,
      "logits/chosen": -1.080078125,
      "logits/rejected": -0.9380859136581421,
      "logps/chosen": -0.737988293170929,
      "logps/rejected": -1.916406273841858,
      "loss": 0.9195,
      "nll_loss": 0.9037109613418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07377929985523224,
      "rewards/margins": 0.1177978515625,
      "rewards/rejected": -0.19150391221046448,
      "step": 2650
    },
    {
      "epoch": 0.34327009936766034,
      "grad_norm": 1.999546690187206,
      "learning_rate": 1.5511334686589623e-06,
      "log_odds_chosen": 1.4326171875,
      "log_odds_ratio": -0.3836914002895355,
      "logits/chosen": -1.054101586341858,
      "logits/rejected": -0.935351550579071,
      "logps/chosen": -0.6806640625,
      "logps/rejected": -1.673437476158142,
      "loss": 0.9479,
      "nll_loss": 0.9583984613418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06805419921875,
      "rewards/margins": 0.09908447414636612,
      "rewards/rejected": -0.16713866591453552,
      "step": 2660
    },
    {
      "epoch": 0.3445605884630275,
      "grad_norm": 2.0591283552834727,
      "learning_rate": 1.548225999432367e-06,
      "log_odds_chosen": 1.408105492591858,
      "log_odds_ratio": -0.4193359315395355,
      "logits/chosen": -1.0373046398162842,
      "logits/rejected": -0.9056640863418579,
      "logps/chosen": -0.6708008050918579,
      "logps/rejected": -1.6599609851837158,
      "loss": 0.9655,
      "nll_loss": 0.909960925579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06706543266773224,
      "rewards/margins": 0.09891967475414276,
      "rewards/rejected": -0.166015625,
      "step": 2670
    },
    {
      "epoch": 0.3458510775583946,
      "grad_norm": 2.239065379690969,
      "learning_rate": 1.5453348185725114e-06,
      "log_odds_chosen": 1.3125,
      "log_odds_ratio": -0.385498046875,
      "logits/chosen": -1.0791015625,
      "logits/rejected": -0.96875,
      "logps/chosen": -0.6611328125,
      "logps/rejected": -1.537109375,
      "loss": 0.9343,
      "nll_loss": 0.8662109375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06612548977136612,
      "rewards/margins": 0.08754882961511612,
      "rewards/rejected": -0.15375976264476776,
      "step": 2680
    },
    {
      "epoch": 0.3471415666537618,
      "grad_norm": 1.9161639376434576,
      "learning_rate": 1.542459774559398e-06,
      "log_odds_chosen": 1.176171898841858,
      "log_odds_ratio": -0.4443359375,
      "logits/chosen": -1.0910155773162842,
      "logits/rejected": -0.988085925579071,
      "logps/chosen": -0.688281238079071,
      "logps/rejected": -1.495703101158142,
      "loss": 0.9256,
      "nll_loss": 0.891406238079071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06883545219898224,
      "rewards/margins": 0.08069457858800888,
      "rewards/rejected": -0.14960937201976776,
      "step": 2690
    },
    {
      "epoch": 0.34843205574912894,
      "grad_norm": 2.3292320975888448,
      "learning_rate": 1.539600717839002e-06,
      "log_odds_chosen": 1.40185546875,
      "log_odds_ratio": -0.3963378965854645,
      "logits/chosen": -1.066796898841858,
      "logits/rejected": -0.9478515386581421,
      "logps/chosen": -0.686328113079071,
      "logps/rejected": -1.697656273841858,
      "loss": 0.9309,
      "nll_loss": 0.877734363079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06865234673023224,
      "rewards/margins": 0.101104736328125,
      "rewards/rejected": -0.169921875,
      "step": 2700
    },
    {
      "epoch": 0.34972254484449605,
      "grad_norm": 2.231762197305753,
      "learning_rate": 1.536757500790597e-06,
      "log_odds_chosen": 1.2559082508087158,
      "log_odds_ratio": -0.4477783143520355,
      "logits/chosen": -1.0828125476837158,
      "logits/rejected": -0.9781249761581421,
      "logps/chosen": -0.726367175579071,
      "logps/rejected": -1.611718773841858,
      "loss": 0.9412,
      "nll_loss": 0.9033203125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07264403998851776,
      "rewards/margins": 0.08851929008960724,
      "rewards/rejected": -0.16110840439796448,
      "step": 2710
    },
    {
      "epoch": 0.3510130339398632,
      "grad_norm": 2.2165375443443462,
      "learning_rate": 1.5339299776947407e-06,
      "log_odds_chosen": 1.2438476085662842,
      "log_odds_ratio": -0.477783203125,
      "logits/chosen": -1.052734375,
      "logits/rejected": -0.9560546875,
      "logps/chosen": -0.7300781011581421,
      "logps/rejected": -1.6355469226837158,
      "loss": 0.9155,
      "nll_loss": 0.853320300579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07297363132238388,
      "rewards/margins": 0.090484619140625,
      "rewards/rejected": -0.16335448622703552,
      "step": 2720
    },
    {
      "epoch": 0.3523035230352303,
      "grad_norm": 2.2029813829200457,
      "learning_rate": 1.5311180047019054e-06,
      "log_odds_chosen": 1.460595726966858,
      "log_odds_ratio": -0.38725584745407104,
      "logits/chosen": -1.102148413658142,
      "logits/rejected": -0.961132824420929,
      "logps/chosen": -0.6841796636581421,
      "logps/rejected": -1.685937523841858,
      "loss": 0.9145,
      "nll_loss": 0.8921874761581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06839599460363388,
      "rewards/margins": 0.10020141303539276,
      "rewards/rejected": -0.16850586235523224,
      "step": 2730
    },
    {
      "epoch": 0.3535940121305975,
      "grad_norm": 2.19836589458878,
      "learning_rate": 1.5283214398017402e-06,
      "log_odds_chosen": 1.453222632408142,
      "log_odds_ratio": -0.40019530057907104,
      "logits/chosen": -1.071679711341858,
      "logits/rejected": -0.9791015386581421,
      "logps/chosen": -0.7071288824081421,
      "logps/rejected": -1.751562476158142,
      "loss": 0.9321,
      "nll_loss": 0.8837890625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07065429538488388,
      "rewards/margins": 0.10456085205078125,
      "rewards/rejected": -0.17519530653953552,
      "step": 2740
    },
    {
      "epoch": 0.35488450122596465,
      "grad_norm": 2.140994701777331,
      "learning_rate": 1.5255401427929477e-06,
      "log_odds_chosen": 1.413476586341858,
      "log_odds_ratio": -0.3888183534145355,
      "logits/chosen": -1.1052734851837158,
      "logits/rejected": -0.9898437261581421,
      "logps/chosen": -0.725390613079071,
      "logps/rejected": -1.748046875,
      "loss": 0.9673,
      "nll_loss": 0.942187488079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07248535007238388,
      "rewards/margins": 0.10224609076976776,
      "rewards/rejected": -0.1746826171875,
      "step": 2750
    },
    {
      "epoch": 0.35617499032133176,
      "grad_norm": 2.173269858720238,
      "learning_rate": 1.5227739752537617e-06,
      "log_odds_chosen": 1.7960937023162842,
      "log_odds_ratio": -0.3263183534145355,
      "logits/chosen": -1.1150391101837158,
      "logits/rejected": -0.955273449420929,
      "logps/chosen": -0.6473633050918579,
      "logps/rejected": -1.9201171398162842,
      "loss": 0.9336,
      "nll_loss": 0.8763672113418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06472168117761612,
      "rewards/margins": 0.12742920219898224,
      "rewards/rejected": -0.19211426377296448,
      "step": 2760
    },
    {
      "epoch": 0.35746547941669893,
      "grad_norm": 2.223267993956148,
      "learning_rate": 1.5200228005130127e-06,
      "log_odds_chosen": 1.2712891101837158,
      "log_odds_ratio": -0.46147459745407104,
      "logits/chosen": -1.0929687023162842,
      "logits/rejected": -1.003320336341858,
      "logps/chosen": -0.7113281488418579,
      "logps/rejected": -1.599218726158142,
      "loss": 0.9178,
      "nll_loss": 0.8822265863418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07117919623851776,
      "rewards/margins": 0.08876953274011612,
      "rewards/rejected": -0.15988770127296448,
      "step": 2770
    },
    {
      "epoch": 0.3587559685120661,
      "grad_norm": 2.158383242149607,
      "learning_rate": 1.5172864836217631e-06,
      "log_odds_chosen": 1.5007812976837158,
      "log_odds_ratio": -0.3785644471645355,
      "logits/chosen": -1.1183593273162842,
      "logits/rejected": -0.992968738079071,
      "logps/chosen": -0.719531238079071,
      "logps/rejected": -1.7765624523162842,
      "loss": 0.9238,
      "nll_loss": 0.875195324420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07200928032398224,
      "rewards/margins": 0.105865478515625,
      "rewards/rejected": -0.17778320610523224,
      "step": 2780
    },
    {
      "epoch": 0.3600464576074332,
      "grad_norm": 2.218316246766675,
      "learning_rate": 1.514564891325506e-06,
      "log_odds_chosen": 1.4109375476837158,
      "log_odds_ratio": -0.4332031309604645,
      "logits/chosen": -1.1298828125,
      "logits/rejected": -1.0283203125,
      "logps/chosen": -0.708300769329071,
      "logps/rejected": -1.744531273841858,
      "loss": 0.947,
      "nll_loss": 0.875781238079071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07088623195886612,
      "rewards/margins": 0.1036834716796875,
      "rewards/rejected": -0.17448730766773224,
      "step": 2790
    },
    {
      "epoch": 0.36133694670280037,
      "grad_norm": 2.4698003691974657,
      "learning_rate": 1.5118578920369086e-06,
      "log_odds_chosen": 1.02850341796875,
      "log_odds_ratio": -0.508349597454071,
      "logits/chosen": -1.0812499523162842,
      "logits/rejected": -0.978710949420929,
      "logps/chosen": -0.756054699420929,
      "logps/rejected": -1.5011718273162842,
      "loss": 0.9293,
      "nll_loss": 0.880078136920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07559814304113388,
      "rewards/margins": 0.07447509467601776,
      "rewards/rejected": -0.15004882216453552,
      "step": 2800
    },
    {
      "epoch": 0.36262743579816753,
      "grad_norm": 2.2028695165912495,
      "learning_rate": 1.5091653558090898e-06,
      "log_odds_chosen": 1.438867211341858,
      "log_odds_ratio": -0.4041503965854645,
      "logits/chosen": -1.076757788658142,
      "logits/rejected": -0.9462890625,
      "logps/chosen": -0.70166015625,
      "logps/rejected": -1.7253906726837158,
      "loss": 0.9179,
      "nll_loss": 0.8564453125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07020263373851776,
      "rewards/margins": 0.10239257663488388,
      "rewards/rejected": -0.17250975966453552,
      "step": 2810
    },
    {
      "epoch": 0.36391792489353464,
      "grad_norm": 2.339315440052923,
      "learning_rate": 1.506487154309419e-06,
      "log_odds_chosen": 1.4617187976837158,
      "log_odds_ratio": -0.4110351502895355,
      "logits/chosen": -1.0818359851837158,
      "logits/rejected": -0.968554675579071,
      "logps/chosen": -0.6812499761581421,
      "logps/rejected": -1.737695336341858,
      "loss": 0.9144,
      "nll_loss": 0.848828136920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06818847358226776,
      "rewards/margins": 0.10574951022863388,
      "rewards/rejected": -0.17387695610523224,
      "step": 2820
    },
    {
      "epoch": 0.3652084139889018,
      "grad_norm": 2.352984640649129,
      "learning_rate": 1.5038231607938247e-06,
      "log_odds_chosen": 1.524511694908142,
      "log_odds_ratio": -0.3854003846645355,
      "logits/chosen": -1.103124976158142,
      "logits/rejected": -0.9671875238418579,
      "logps/chosen": -0.7027343511581421,
      "logps/rejected": -1.790429711341858,
      "loss": 0.8964,
      "nll_loss": 0.834765613079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07021484524011612,
      "rewards/margins": 0.10875244438648224,
      "rewards/rejected": -0.17897948622703552,
      "step": 2830
    },
    {
      "epoch": 0.3664989030842689,
      "grad_norm": 2.180353864980968,
      "learning_rate": 1.501173250081603e-06,
      "log_odds_chosen": 1.3330078125,
      "log_odds_ratio": -0.4073242247104645,
      "logits/chosen": -1.0554687976837158,
      "logits/rejected": -0.9320312738418579,
      "logps/chosen": -0.702441394329071,
      "logps/rejected": -1.62109375,
      "loss": 0.9172,
      "nll_loss": 0.9115234613418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07030029594898224,
      "rewards/margins": 0.09189452975988388,
      "rewards/rejected": -0.162109375,
      "step": 2840
    },
    {
      "epoch": 0.3677893921796361,
      "grad_norm": 2.2163317375584506,
      "learning_rate": 1.4985372985307103e-06,
      "log_odds_chosen": 1.589257836341858,
      "log_odds_ratio": -0.3683105409145355,
      "logits/chosen": -1.0693359375,
      "logits/rejected": -0.979296863079071,
      "logps/chosen": -0.712109386920929,
      "logps/rejected": -1.8429687023162842,
      "loss": 0.9116,
      "nll_loss": 0.8675781488418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07122802734375,
      "rewards/margins": 0.11309814453125,
      "rewards/rejected": -0.18430176377296448,
      "step": 2850
    },
    {
      "epoch": 0.36907988127500324,
      "grad_norm": 1.9894316525822717,
      "learning_rate": 1.4959151840135313e-06,
      "log_odds_chosen": 1.305273413658142,
      "log_odds_ratio": -0.42919921875,
      "logits/chosen": -1.0343749523162842,
      "logits/rejected": -0.949999988079071,
      "logps/chosen": -0.633593738079071,
      "logps/rejected": -1.518164038658142,
      "loss": 0.9239,
      "nll_loss": 0.828320324420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06341552734375,
      "rewards/margins": 0.08831787109375,
      "rewards/rejected": -0.15168456733226776,
      "step": 2860
    },
    {
      "epoch": 0.37037037037037035,
      "grad_norm": 2.296578878256503,
      "learning_rate": 1.4933067858931148e-06,
      "log_odds_chosen": 1.4514648914337158,
      "log_odds_ratio": -0.4212890565395355,
      "logits/chosen": -1.0470702648162842,
      "logits/rejected": -0.953320324420929,
      "logps/chosen": -0.712890625,
      "logps/rejected": -1.7374999523162842,
      "loss": 0.9045,
      "nll_loss": 0.920703113079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07125244289636612,
      "rewards/margins": 0.10255737602710724,
      "rewards/rejected": -0.17380371689796448,
      "step": 2870
    },
    {
      "epoch": 0.3716608594657375,
      "grad_norm": 2.327849144509304,
      "learning_rate": 1.4907119849998597e-06,
      "log_odds_chosen": 1.528222680091858,
      "log_odds_ratio": -0.38422852754592896,
      "logits/chosen": -1.088281273841858,
      "logits/rejected": -0.9697265625,
      "logps/chosen": -0.6781250238418579,
      "logps/rejected": -1.7781250476837158,
      "loss": 0.9219,
      "nll_loss": 0.8912109136581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06783447414636612,
      "rewards/margins": 0.109893798828125,
      "rewards/rejected": -0.1778564453125,
      "step": 2880
    },
    {
      "epoch": 0.3729513485611047,
      "grad_norm": 1.991872639751304,
      "learning_rate": 1.488130663608649e-06,
      "log_odds_chosen": 1.5209472179412842,
      "log_odds_ratio": -0.38090819120407104,
      "logits/chosen": -1.0603516101837158,
      "logits/rejected": -0.9234374761581421,
      "logps/chosen": -0.7066406011581421,
      "logps/rejected": -1.8210937976837158,
      "loss": 0.9101,
      "nll_loss": 0.8832031488418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07069091498851776,
      "rewards/margins": 0.11154785007238388,
      "rewards/rejected": -0.18208007514476776,
      "step": 2890
    },
    {
      "epoch": 0.3742418376564718,
      "grad_norm": 2.6380174611677636,
      "learning_rate": 1.4855627054164149e-06,
      "log_odds_chosen": 1.492578148841858,
      "log_odds_ratio": -0.40229493379592896,
      "logits/chosen": -1.096093773841858,
      "logits/rejected": -0.9697265625,
      "logps/chosen": -0.6751953363418579,
      "logps/rejected": -1.7410156726837158,
      "loss": 0.9174,
      "nll_loss": 0.8500000238418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0675048828125,
      "rewards/margins": 0.10684814304113388,
      "rewards/rejected": -0.17424316704273224,
      "step": 2900
    },
    {
      "epoch": 0.37553232675183895,
      "grad_norm": 2.6729643311940245,
      "learning_rate": 1.4830079955201294e-06,
      "log_odds_chosen": 1.4480469226837158,
      "log_odds_ratio": -0.4193359315395355,
      "logits/chosen": -1.064843773841858,
      "logits/rejected": -0.943164050579071,
      "logps/chosen": -0.708789050579071,
      "logps/rejected": -1.7589843273162842,
      "loss": 0.9029,
      "nll_loss": 0.8248046636581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07086181640625,
      "rewards/margins": 0.10493163764476776,
      "rewards/rejected": -0.1759033203125,
      "step": 2910
    },
    {
      "epoch": 0.3768228158472061,
      "grad_norm": 2.3302031609808855,
      "learning_rate": 1.4804664203952103e-06,
      "log_odds_chosen": 1.4973633289337158,
      "log_odds_ratio": -0.4120117127895355,
      "logits/chosen": -1.074804663658142,
      "logits/rejected": -0.9634765386581421,
      "logps/chosen": -0.670214831829071,
      "logps/rejected": -1.697656273841858,
      "loss": 0.9238,
      "nll_loss": 0.9159179925918579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06700439751148224,
      "rewards/margins": 0.102813720703125,
      "rewards/rejected": -0.16977539658546448,
      "step": 2920
    },
    {
      "epoch": 0.3781133049425732,
      "grad_norm": 2.2728313996672282,
      "learning_rate": 1.4779378678743327e-06,
      "log_odds_chosen": 1.2537109851837158,
      "log_odds_ratio": -0.41645509004592896,
      "logits/chosen": -1.079687476158142,
      "logits/rejected": -0.9326171875,
      "logps/chosen": -0.694531261920929,
      "logps/rejected": -1.5750000476837158,
      "loss": 0.9214,
      "nll_loss": 0.8330078125,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06949462741613388,
      "rewards/margins": 0.08800049126148224,
      "rewards/rejected": -0.15749511122703552,
      "step": 2930
    },
    {
      "epoch": 0.3794037940379404,
      "grad_norm": 2.2694505848878106,
      "learning_rate": 1.4754222271266348e-06,
      "log_odds_chosen": 1.3136718273162842,
      "log_odds_ratio": -0.4032226502895355,
      "logits/chosen": -1.0236327648162842,
      "logits/rejected": -0.9410156011581421,
      "logps/chosen": -0.714648425579071,
      "logps/rejected": -1.6164062023162842,
      "loss": 0.8792,
      "nll_loss": 0.8792968988418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07144775241613388,
      "rewards/margins": 0.09028320014476776,
      "rewards/rejected": -0.16169433295726776,
      "step": 2940
    },
    {
      "epoch": 0.3806942831333075,
      "grad_norm": 2.188051247903212,
      "learning_rate": 1.4729193886373175e-06,
      "log_odds_chosen": 1.187768578529358,
      "log_odds_ratio": -0.46196287870407104,
      "logits/chosen": -1.040624976158142,
      "logits/rejected": -0.9341796636581421,
      "logps/chosen": -0.70703125,
      "logps/rejected": -1.5490233898162842,
      "loss": 0.9091,
      "nll_loss": 0.8529297113418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0706787109375,
      "rewards/margins": 0.08422698825597763,
      "rewards/rejected": -0.15493163466453552,
      "step": 2950
    },
    {
      "epoch": 0.38198477222867466,
      "grad_norm": 2.5782415724903682,
      "learning_rate": 1.4704292441876156e-06,
      "log_odds_chosen": 1.486425757408142,
      "log_odds_ratio": -0.4056640565395355,
      "logits/chosen": -1.0564453601837158,
      "logits/rejected": -0.9732421636581421,
      "logps/chosen": -0.686328113079071,
      "logps/rejected": -1.7527344226837158,
      "loss": 0.9282,
      "nll_loss": 0.7997070550918579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06862793117761612,
      "rewards/margins": 0.10683593899011612,
      "rewards/rejected": -0.17546387016773224,
      "step": 2960
    },
    {
      "epoch": 0.3832752613240418,
      "grad_norm": 2.4548753552772142,
      "learning_rate": 1.4679516868351474e-06,
      "log_odds_chosen": 1.44580078125,
      "log_odds_ratio": -0.4093261659145355,
      "logits/chosen": -1.057226538658142,
      "logits/rejected": -0.9457031488418579,
      "logps/chosen": -0.6908203363418579,
      "logps/rejected": -1.7275390625,
      "loss": 0.8948,
      "nll_loss": 0.873828113079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.069091796875,
      "rewards/margins": 0.103790283203125,
      "rewards/rejected": -0.1727294921875,
      "step": 2970
    },
    {
      "epoch": 0.38456575041940894,
      "grad_norm": 2.132330130618494,
      "learning_rate": 1.4654866108946234e-06,
      "log_odds_chosen": 1.387109398841858,
      "log_odds_ratio": -0.40478515625,
      "logits/chosen": -1.054296851158142,
      "logits/rejected": -0.9410156011581421,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -1.7195312976837158,
      "loss": 0.9176,
      "nll_loss": 0.8890625238418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07077636569738388,
      "rewards/margins": 0.10122070461511612,
      "rewards/rejected": -0.171875,
      "step": 2980
    },
    {
      "epoch": 0.3858562395147761,
      "grad_norm": 2.049242938288744,
      "learning_rate": 1.4630339119189101e-06,
      "log_odds_chosen": 1.592382788658142,
      "log_odds_ratio": -0.358642578125,
      "logits/chosen": -0.995312511920929,
      "logits/rejected": -0.8994140625,
      "logps/chosen": -0.6439453363418579,
      "logps/rejected": -1.7392578125,
      "loss": 0.9298,
      "nll_loss": 0.921093761920929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06439208984375,
      "rewards/margins": 0.10946045070886612,
      "rewards/rejected": -0.17387695610523224,
      "step": 2990
    },
    {
      "epoch": 0.38714672861014326,
      "grad_norm": 2.2445356272196126,
      "learning_rate": 1.4605934866804429e-06,
      "log_odds_chosen": 1.403100609779358,
      "log_odds_ratio": -0.384765625,
      "logits/chosen": -1.061914086341858,
      "logits/rejected": -0.943554699420929,
      "logps/chosen": -0.685351550579071,
      "logps/rejected": -1.670312523841858,
      "loss": 0.9274,
      "nll_loss": 0.8433593511581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06856689602136612,
      "rewards/margins": 0.09843750298023224,
      "rewards/rejected": -0.1671142578125,
      "step": 3000
    },
    {
      "epoch": 0.3884372177055104,
      "grad_norm": 2.265413061346713,
      "learning_rate": 1.4581652331529784e-06,
      "log_odds_chosen": 1.551855444908142,
      "log_odds_ratio": -0.4266113340854645,
      "logits/chosen": -1.061914086341858,
      "logits/rejected": -0.946093738079071,
      "logps/chosen": -0.7411133050918579,
      "logps/rejected": -1.8738281726837158,
      "loss": 0.9227,
      "nll_loss": 0.9169921875,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07406006008386612,
      "rewards/margins": 0.11335144191980362,
      "rewards/rejected": -0.18740233778953552,
      "step": 3010
    },
    {
      "epoch": 0.38972770680087754,
      "grad_norm": 2.1253393004993217,
      "learning_rate": 1.4557490504936778e-06,
      "log_odds_chosen": 1.2938964366912842,
      "log_odds_ratio": -0.435546875,
      "logits/chosen": -1.1115233898162842,
      "logits/rejected": -1.0173828601837158,
      "logps/chosen": -0.720996081829071,
      "logps/rejected": -1.6318359375,
      "loss": 0.9057,
      "nll_loss": 0.832812488079071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07211913913488388,
      "rewards/margins": 0.09123687446117401,
      "rewards/rejected": -0.16323241591453552,
      "step": 3020
    },
    {
      "epoch": 0.3910181958962447,
      "grad_norm": 2.1631027609743874,
      "learning_rate": 1.453344839025519e-06,
      "log_odds_chosen": 1.589257836341858,
      "log_odds_ratio": -0.341796875,
      "logits/chosen": -1.096093773841858,
      "logits/rejected": -0.9759765863418579,
      "logps/chosen": -0.67578125,
      "logps/rejected": -1.796875,
      "loss": 0.9068,
      "nll_loss": 0.8267577886581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06760253757238388,
      "rewards/margins": 0.11229248344898224,
      "rewards/rejected": -0.17983397841453552,
      "step": 3030
    },
    {
      "epoch": 0.3923086849916118,
      "grad_norm": 2.322600743876153,
      "learning_rate": 1.4509525002200234e-06,
      "log_odds_chosen": 1.4470703601837158,
      "log_odds_ratio": -0.4080566465854645,
      "logits/chosen": -1.024023413658142,
      "logits/rejected": -0.9046875238418579,
      "logps/chosen": -0.7035156488418579,
      "logps/rejected": -1.7166016101837158,
      "loss": 0.887,
      "nll_loss": 0.962695300579071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07033691555261612,
      "rewards/margins": 0.101348876953125,
      "rewards/rejected": -0.171875,
      "step": 3040
    },
    {
      "epoch": 0.393599174086979,
      "grad_norm": 2.6702551813489506,
      "learning_rate": 1.4485719366802965e-06,
      "log_odds_chosen": 1.3859374523162842,
      "log_odds_ratio": -0.41950684785842896,
      "logits/chosen": -1.054296851158142,
      "logits/rejected": -0.973437488079071,
      "logps/chosen": -0.722460925579071,
      "logps/rejected": -1.704687476158142,
      "loss": 0.9069,
      "nll_loss": 0.8421875238418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07219238579273224,
      "rewards/margins": 0.09824218600988388,
      "rewards/rejected": -0.17058105766773224,
      "step": 3050
    },
    {
      "epoch": 0.3948896631823461,
      "grad_norm": 2.3131468943945337,
      "learning_rate": 1.4462030521243742e-06,
      "log_odds_chosen": 1.5419921875,
      "log_odds_ratio": -0.36333006620407104,
      "logits/chosen": -1.0341796875,
      "logits/rejected": -0.92578125,
      "logps/chosen": -0.6170898675918579,
      "logps/rejected": -1.6554687023162842,
      "loss": 0.8956,
      "nll_loss": 0.87548828125,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06171875074505806,
      "rewards/margins": 0.1038818359375,
      "rewards/rejected": -0.16557617485523224,
      "step": 3060
    },
    {
      "epoch": 0.39618015227771325,
      "grad_norm": 2.568959404562904,
      "learning_rate": 1.443845751368867e-06,
      "log_odds_chosen": 1.4158203601837158,
      "log_odds_ratio": -0.3808349668979645,
      "logits/chosen": -1.098046898841858,
      "logits/rejected": -0.956250011920929,
      "logps/chosen": -0.664257824420929,
      "logps/rejected": -1.655859351158142,
      "loss": 0.883,
      "nll_loss": 0.8623046875,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06649170070886612,
      "rewards/margins": 0.09916992485523224,
      "rewards/rejected": -0.16557617485523224,
      "step": 3070
    },
    {
      "epoch": 0.3974706413730804,
      "grad_norm": 3.3023441803874216,
      "learning_rate": 1.4414999403128943e-06,
      "log_odds_chosen": 1.585351586341858,
      "log_odds_ratio": -0.3646484315395355,
      "logits/chosen": -1.1056640148162842,
      "logits/rejected": -0.971484363079071,
      "logps/chosen": -0.687304675579071,
      "logps/rejected": -1.788671851158142,
      "loss": 0.9168,
      "nll_loss": 0.8160156011581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06870117038488388,
      "rewards/margins": 0.11007080227136612,
      "rewards/rejected": -0.17878417670726776,
      "step": 3080
    },
    {
      "epoch": 0.3987611304684475,
      "grad_norm": 2.499591219384109,
      "learning_rate": 1.439165525922309e-06,
      "log_odds_chosen": 1.5525391101837158,
      "log_odds_ratio": -0.37543946504592896,
      "logits/chosen": -1.099023461341858,
      "logits/rejected": -0.971875011920929,
      "logps/chosen": -0.653515636920929,
      "logps/rejected": -1.729101538658142,
      "loss": 0.8885,
      "nll_loss": 0.8271484375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06531982123851776,
      "rewards/margins": 0.10755614936351776,
      "rewards/rejected": -0.17292480170726776,
      "step": 3090
    },
    {
      "epoch": 0.4000516195638147,
      "grad_norm": 2.4298851182223036,
      "learning_rate": 1.4368424162141992e-06,
      "log_odds_chosen": 1.460351586341858,
      "log_odds_ratio": -0.40380859375,
      "logits/chosen": -1.037109375,
      "logits/rejected": -0.9212890863418579,
      "logps/chosen": -0.757617175579071,
      "logps/rejected": -1.8359375,
      "loss": 0.9029,
      "nll_loss": 0.884765625,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07572021335363388,
      "rewards/margins": 0.107879638671875,
      "rewards/rejected": -0.18369141221046448,
      "step": 3100
    },
    {
      "epoch": 0.40134210865918185,
      "grad_norm": 2.219547420574873,
      "learning_rate": 1.434530520241665e-06,
      "log_odds_chosen": 1.664453148841858,
      "log_odds_ratio": -0.36376953125,
      "logits/chosen": -1.0947265625,
      "logits/rejected": -0.9697265625,
      "logps/chosen": -0.6826171875,
      "logps/rejected": -1.868749976158142,
      "loss": 0.9039,
      "nll_loss": 0.800000011920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06821288913488388,
      "rewards/margins": 0.11879882961511612,
      "rewards/rejected": -0.18701171875,
      "step": 3110
    },
    {
      "epoch": 0.40263259775454896,
      "grad_norm": 2.262039010065835,
      "learning_rate": 1.4322297480788657e-06,
      "log_odds_chosen": 1.6453125476837158,
      "log_odds_ratio": -0.3831787109375,
      "logits/chosen": -1.0769531726837158,
      "logits/rejected": -0.979296863079071,
      "logps/chosen": -0.7017577886581421,
      "logps/rejected": -1.8605468273162842,
      "loss": 0.9025,
      "nll_loss": 0.8369140625,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07016601413488388,
      "rewards/margins": 0.11580810695886612,
      "rewards/rejected": -0.1859130859375,
      "step": 3120
    },
    {
      "epoch": 0.4039230868499161,
      "grad_norm": 2.1847990364208516,
      "learning_rate": 1.4299400108063247e-06,
      "log_odds_chosen": 1.39697265625,
      "log_odds_ratio": -0.404052734375,
      "logits/chosen": -1.1121094226837158,
      "logits/rejected": -0.990039050579071,
      "logps/chosen": -0.6865234375,
      "logps/rejected": -1.662500023841858,
      "loss": 0.8588,
      "nll_loss": 0.7705078125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06866455078125,
      "rewards/margins": 0.09757079929113388,
      "rewards/rejected": -0.16618652641773224,
      "step": 3130
    },
    {
      "epoch": 0.4052135759452833,
      "grad_norm": 2.24383435017787,
      "learning_rate": 1.4276612204964992e-06,
      "log_odds_chosen": 1.686132788658142,
      "log_odds_ratio": -0.3419433534145355,
      "logits/chosen": -1.0517578125,
      "logits/rejected": -0.9349609613418579,
      "logps/chosen": -0.69189453125,
      "logps/rejected": -1.904296875,
      "loss": 0.8929,
      "nll_loss": 0.8794921636581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06926269829273224,
      "rewards/margins": 0.12119750678539276,
      "rewards/rejected": -0.19035644829273224,
      "step": 3140
    },
    {
      "epoch": 0.4065040650406504,
      "grad_norm": 2.117643275971798,
      "learning_rate": 1.4253932901995967e-06,
      "log_odds_chosen": 1.5070312023162842,
      "log_odds_ratio": -0.41044920682907104,
      "logits/chosen": -1.0978515148162842,
      "logits/rejected": -0.973828136920929,
      "logps/chosen": -0.6806640625,
      "logps/rejected": -1.7097656726837158,
      "loss": 0.8965,
      "nll_loss": 0.7994140386581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06813964992761612,
      "rewards/margins": 0.10302734375,
      "rewards/rejected": -0.17104491591453552,
      "step": 3150
    },
    {
      "epoch": 0.40779455413601756,
      "grad_norm": 2.435057584129839,
      "learning_rate": 1.42313613392964e-06,
      "log_odds_chosen": 1.7966797351837158,
      "log_odds_ratio": -0.3165527284145355,
      "logits/chosen": -1.082617163658142,
      "logits/rejected": -0.937304675579071,
      "logps/chosen": -0.65576171875,
      "logps/rejected": -1.948632836341858,
      "loss": 0.8784,
      "nll_loss": 0.805468738079071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06566162407398224,
      "rewards/margins": 0.129302978515625,
      "rewards/rejected": -0.19487304985523224,
      "step": 3160
    },
    {
      "epoch": 0.40908504323138467,
      "grad_norm": 2.3871356778384536,
      "learning_rate": 1.4208896666507756e-06,
      "log_odds_chosen": 1.5526854991912842,
      "log_odds_ratio": -0.3926025331020355,
      "logits/chosen": -1.0714843273162842,
      "logits/rejected": -0.9759765863418579,
      "logps/chosen": -0.698535144329071,
      "logps/rejected": -1.773828148841858,
      "loss": 0.9064,
      "nll_loss": 0.876757800579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06981201469898224,
      "rewards/margins": 0.10760650783777237,
      "rewards/rejected": -0.17734375596046448,
      "step": 3170
    },
    {
      "epoch": 0.41037553232675184,
      "grad_norm": 2.536948909826965,
      "learning_rate": 1.4186538042638173e-06,
      "log_odds_chosen": 1.5473144054412842,
      "log_odds_ratio": -0.396240234375,
      "logits/chosen": -1.0167968273162842,
      "logits/rejected": -0.914257824420929,
      "logps/chosen": -0.681835949420929,
      "logps/rejected": -1.8035156726837158,
      "loss": 0.9006,
      "nll_loss": 0.874218761920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06815185397863388,
      "rewards/margins": 0.11218567192554474,
      "rewards/rejected": -0.18027344346046448,
      "step": 3180
    },
    {
      "epoch": 0.411666021422119,
      "grad_norm": 2.7546053231094927,
      "learning_rate": 1.416428463593022e-06,
      "log_odds_chosen": 1.442724585533142,
      "log_odds_ratio": -0.4417968690395355,
      "logits/chosen": -1.052343726158142,
      "logits/rejected": -0.9369140863418579,
      "logps/chosen": -0.717968761920929,
      "logps/rejected": -1.7751953601837158,
      "loss": 0.8795,
      "nll_loss": 0.835742175579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07187499850988388,
      "rewards/margins": 0.10564575344324112,
      "rewards/rejected": -0.17756347358226776,
      "step": 3190
    },
    {
      "epoch": 0.4129565105174861,
      "grad_norm": 2.2126552889760345,
      "learning_rate": 1.414213562373095e-06,
      "log_odds_chosen": 1.520898461341858,
      "log_odds_ratio": -0.4220214784145355,
      "logits/chosen": -1.0734374523162842,
      "logits/rejected": -0.9544922113418579,
      "logps/chosen": -0.708300769329071,
      "logps/rejected": -1.8166015148162842,
      "loss": 0.8899,
      "nll_loss": 0.8814452886581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07084961235523224,
      "rewards/margins": 0.11072387546300888,
      "rewards/rejected": -0.18154296278953552,
      "step": 3200
    },
    {
      "epoch": 0.4142469996128533,
      "grad_norm": 2.285547284513354,
      "learning_rate": 1.4120090192364154e-06,
      "log_odds_chosen": 1.324121117591858,
      "log_odds_ratio": -0.44169920682907104,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.9300781488418579,
      "logps/chosen": -0.7339843511581421,
      "logps/rejected": -1.683984398841858,
      "loss": 0.9234,
      "nll_loss": 0.9291015863418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07338867336511612,
      "rewards/margins": 0.09508667141199112,
      "rewards/rejected": -0.16845703125,
      "step": 3210
    },
    {
      "epoch": 0.41553748870822044,
      "grad_norm": 2.5108858624384376,
      "learning_rate": 1.4098147537004828e-06,
      "log_odds_chosen": 1.6686522960662842,
      "log_odds_ratio": -0.3433593809604645,
      "logits/chosen": -1.036718726158142,
      "logits/rejected": -0.925976574420929,
      "logps/chosen": -0.6470702886581421,
      "logps/rejected": -1.80078125,
      "loss": 0.8851,
      "nll_loss": 0.851855456829071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06472168117761612,
      "rewards/margins": 0.11524657905101776,
      "rewards/rejected": -0.1800537109375,
      "step": 3220
    },
    {
      "epoch": 0.41682797780358755,
      "grad_norm": 2.500557064429634,
      "learning_rate": 1.4076306861555735e-06,
      "log_odds_chosen": 1.3761718273162842,
      "log_odds_ratio": -0.4400878846645355,
      "logits/chosen": -1.101171851158142,
      "logits/rejected": -1.0154297351837158,
      "logps/chosen": -0.6958984136581421,
      "logps/rejected": -1.6652343273162842,
      "loss": 0.8735,
      "nll_loss": 0.781054675579071,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06961669772863388,
      "rewards/margins": 0.09698486328125,
      "rewards/rejected": -0.16665038466453552,
      "step": 3230
    },
    {
      "epoch": 0.4181184668989547,
      "grad_norm": 2.199301404153063,
      "learning_rate": 1.405456737852613e-06,
      "log_odds_chosen": 1.7003905773162842,
      "log_odds_ratio": -0.3285888731479645,
      "logits/chosen": -1.0607421398162842,
      "logits/rejected": -0.939257800579071,
      "logps/chosen": -0.634765625,
      "logps/rejected": -1.8019530773162842,
      "loss": 0.8682,
      "nll_loss": 0.799023449420929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06348876655101776,
      "rewards/margins": 0.11667480319738388,
      "rewards/rejected": -0.18017578125,
      "step": 3240
    },
    {
      "epoch": 0.4194089559943219,
      "grad_norm": 2.388404206275107,
      "learning_rate": 1.4032928308912468e-06,
      "log_odds_chosen": 1.3020508289337158,
      "log_odds_ratio": -0.4385742247104645,
      "logits/chosen": -1.0968749523162842,
      "logits/rejected": -0.989062488079071,
      "logps/chosen": -0.7256835699081421,
      "logps/rejected": -1.671875,
      "loss": 0.8979,
      "nll_loss": 0.8353515863418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07257080078125,
      "rewards/margins": 0.09464111179113388,
      "rewards/rejected": -0.16718749701976776,
      "step": 3250
    },
    {
      "epoch": 0.420699445089689,
      "grad_norm": 2.0449540849753576,
      "learning_rate": 1.4011388882081175e-06,
      "log_odds_chosen": 1.5710937976837158,
      "log_odds_ratio": -0.3470214903354645,
      "logits/chosen": -1.041406273841858,
      "logits/rejected": -0.912109375,
      "logps/chosen": -0.726367175579071,
      "logps/rejected": -1.82421875,
      "loss": 0.8896,
      "nll_loss": 0.884765625,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.07257080078125,
      "rewards/margins": 0.10991211235523224,
      "rewards/rejected": -0.182373046875,
      "step": 3260
    },
    {
      "epoch": 0.42198993418505615,
      "grad_norm": 2.386978412155408,
      "learning_rate": 1.3989948335653378e-06,
      "log_odds_chosen": 1.27880859375,
      "log_odds_ratio": -0.4677734375,
      "logits/chosen": -1.0847656726837158,
      "logits/rejected": -0.992968738079071,
      "logps/chosen": -0.724902331829071,
      "logps/rejected": -1.6062500476837158,
      "loss": 0.8943,
      "nll_loss": 0.861523449420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07248535007238388,
      "rewards/margins": 0.08826293796300888,
      "rewards/rejected": -0.16081543266773224,
      "step": 3270
    },
    {
      "epoch": 0.42328042328042326,
      "grad_norm": 2.207751863200346,
      "learning_rate": 1.3968605915391564e-06,
      "log_odds_chosen": 1.389306664466858,
      "log_odds_ratio": -0.429443359375,
      "logits/chosen": -1.0517578125,
      "logits/rejected": -0.9369140863418579,
      "logps/chosen": -0.690625011920929,
      "logps/rejected": -1.638281226158142,
      "loss": 0.8696,
      "nll_loss": 0.866992175579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06904296576976776,
      "rewards/margins": 0.094818115234375,
      "rewards/rejected": -0.16391602158546448,
      "step": 3280
    },
    {
      "epoch": 0.4245709123757904,
      "grad_norm": 2.3468951536919826,
      "learning_rate": 1.3947360875088132e-06,
      "log_odds_chosen": 1.1168944835662842,
      "log_odds_ratio": -0.47846680879592896,
      "logits/chosen": -1.041015625,
      "logits/rejected": -0.9624999761581421,
      "logps/chosen": -0.804492175579071,
      "logps/rejected": -1.575781226158142,
      "loss": 0.916,
      "nll_loss": 0.866406261920929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.08043213188648224,
      "rewards/margins": 0.077178955078125,
      "rewards/rejected": -0.15756836533546448,
      "step": 3290
    },
    {
      "epoch": 0.4258614014711576,
      "grad_norm": 2.8349529447881587,
      "learning_rate": 1.3926212476455828e-06,
      "log_odds_chosen": 1.4289062023162842,
      "log_odds_ratio": -0.4022460877895355,
      "logits/chosen": -1.0349609851837158,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.735156238079071,
      "logps/rejected": -1.7716796398162842,
      "loss": 0.8626,
      "nll_loss": 0.85546875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07355956733226776,
      "rewards/margins": 0.10382690280675888,
      "rewards/rejected": -0.17724609375,
      "step": 3300
    },
    {
      "epoch": 0.4271518905665247,
      "grad_norm": 2.64944885815083,
      "learning_rate": 1.3905159989019964e-06,
      "log_odds_chosen": 1.3752930164337158,
      "log_odds_ratio": -0.4356445372104645,
      "logits/chosen": -1.068750023841858,
      "logits/rejected": -0.961718738079071,
      "logps/chosen": -0.7344726324081421,
      "logps/rejected": -1.728906273841858,
      "loss": 0.8884,
      "nll_loss": 0.8121093511581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07341308891773224,
      "rewards/margins": 0.09931640326976776,
      "rewards/rejected": -0.17275390028953552,
      "step": 3310
    },
    {
      "epoch": 0.42844237966189186,
      "grad_norm": 2.308972326268489,
      "learning_rate": 1.3884202690012465e-06,
      "log_odds_chosen": 1.452539086341858,
      "log_odds_ratio": -0.38525390625,
      "logits/chosen": -1.0285155773162842,
      "logits/rejected": -0.914257824420929,
      "logps/chosen": -0.651562511920929,
      "logps/rejected": -1.6320312023162842,
      "loss": 0.8809,
      "nll_loss": 0.836621105670929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06514892727136612,
      "rewards/margins": 0.09809570014476776,
      "rewards/rejected": -0.16328124701976776,
      "step": 3320
    },
    {
      "epoch": 0.429732868757259,
      "grad_norm": 2.6230132025993256,
      "learning_rate": 1.3863339864267636e-06,
      "log_odds_chosen": 1.402734398841858,
      "log_odds_ratio": -0.4161132872104645,
      "logits/chosen": -1.0546875,
      "logits/rejected": -0.934765636920929,
      "logps/chosen": -0.715136706829071,
      "logps/rejected": -1.7039062976837158,
      "loss": 0.8845,
      "nll_loss": 0.855175793170929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07152099907398224,
      "rewards/margins": 0.09897460788488388,
      "rewards/rejected": -0.17045898735523224,
      "step": 3330
    },
    {
      "epoch": 0.43102335785262613,
      "grad_norm": 2.4905831461613,
      "learning_rate": 1.3842570804119655e-06,
      "log_odds_chosen": 1.368554711341858,
      "log_odds_ratio": -0.41796875,
      "logits/chosen": -1.027929663658142,
      "logits/rejected": -0.9267578125,
      "logps/chosen": -0.671875,
      "logps/rejected": -1.621484398841858,
      "loss": 0.8595,
      "nll_loss": 0.8238281011581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06717529147863388,
      "rewards/margins": 0.09489746391773224,
      "rewards/rejected": -0.16206054389476776,
      "step": 3340
    },
    {
      "epoch": 0.4323138469479933,
      "grad_norm": 2.341995316203895,
      "learning_rate": 1.3821894809301763e-06,
      "log_odds_chosen": 1.457421898841858,
      "log_odds_ratio": -0.416259765625,
      "logits/chosen": -1.0460937023162842,
      "logits/rejected": -0.947070300579071,
      "logps/chosen": -0.6634765863418579,
      "logps/rejected": -1.702539086341858,
      "loss": 0.8912,
      "nll_loss": 0.8480468988418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06632079929113388,
      "rewards/margins": 0.10387267917394638,
      "rewards/rejected": -0.17026367783546448,
      "step": 3350
    },
    {
      "epoch": 0.43360433604336046,
      "grad_norm": 2.6092076784502956,
      "learning_rate": 1.3801311186847081e-06,
      "log_odds_chosen": 1.3494141101837158,
      "log_odds_ratio": -0.38496094942092896,
      "logits/chosen": -1.0558593273162842,
      "logits/rejected": -0.962890625,
      "logps/chosen": -0.679492175579071,
      "logps/rejected": -1.560937523841858,
      "loss": 0.8882,
      "nll_loss": 0.7876952886581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06796874850988388,
      "rewards/margins": 0.08807373046875,
      "rewards/rejected": -0.15605469048023224,
      "step": 3360
    },
    {
      "epoch": 0.43489482513872757,
      "grad_norm": 2.405800517606485,
      "learning_rate": 1.378081925099109e-06,
      "log_odds_chosen": 1.205078125,
      "log_odds_ratio": -0.4443359375,
      "logits/chosen": -1.0517578125,
      "logits/rejected": -0.9664062261581421,
      "logps/chosen": -0.71435546875,
      "logps/rejected": -1.542578101158142,
      "loss": 0.9128,
      "nll_loss": 0.874804675579071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07143554836511612,
      "rewards/margins": 0.08285827934741974,
      "rewards/rejected": -0.15427246689796448,
      "step": 3370
    },
    {
      "epoch": 0.43618531423409473,
      "grad_norm": 2.614506688632388,
      "learning_rate": 1.376041832307563e-06,
      "log_odds_chosen": 1.4582030773162842,
      "log_odds_ratio": -0.4073730409145355,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.6871093511581421,
      "logps/rejected": -1.6960937976837158,
      "loss": 0.8876,
      "nll_loss": 0.8539062738418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06875000149011612,
      "rewards/margins": 0.10064697265625,
      "rewards/rejected": -0.16940918564796448,
      "step": 3380
    },
    {
      "epoch": 0.43747580332946184,
      "grad_norm": 2.4701290174154407,
      "learning_rate": 1.3740107731454524e-06,
      "log_odds_chosen": 1.7009766101837158,
      "log_odds_ratio": -0.3541503846645355,
      "logits/chosen": -1.071874976158142,
      "logits/rejected": -0.9359375238418579,
      "logps/chosen": -0.6576172113418579,
      "logps/rejected": -1.892187476158142,
      "loss": 0.8854,
      "nll_loss": 0.7962890863418579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06578369438648224,
      "rewards/margins": 0.12352295219898224,
      "rewards/rejected": -0.18925781548023224,
      "step": 3390
    },
    {
      "epoch": 0.438766292424829,
      "grad_norm": 2.89948165195672,
      "learning_rate": 1.3719886811400705e-06,
      "log_odds_chosen": 1.4343750476837158,
      "log_odds_ratio": -0.409912109375,
      "logits/chosen": -1.087499976158142,
      "logits/rejected": -0.9765625,
      "logps/chosen": -0.6800781488418579,
      "logps/rejected": -1.705078125,
      "loss": 0.8913,
      "nll_loss": 0.8365234136581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06795654445886612,
      "rewards/margins": 0.10252685844898224,
      "rewards/rejected": -0.17060546576976776,
      "step": 3400
    },
    {
      "epoch": 0.44005678152019617,
      "grad_norm": 2.4047628121709814,
      "learning_rate": 1.3699754905014834e-06,
      "log_odds_chosen": 1.4521484375,
      "log_odds_ratio": -0.40019530057907104,
      "logits/chosen": -1.049414038658142,
      "logits/rejected": -0.9488281011581421,
      "logps/chosen": -0.715624988079071,
      "logps/rejected": -1.7136719226837158,
      "loss": 0.8862,
      "nll_loss": 0.8431640863418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07155761867761612,
      "rewards/margins": 0.099853515625,
      "rewards/rejected": -0.17136231064796448,
      "step": 3410
    },
    {
      "epoch": 0.4413472706155633,
      "grad_norm": 2.50403398626086,
      "learning_rate": 1.3679711361135388e-06,
      "log_odds_chosen": 1.671289086341858,
      "log_odds_ratio": -0.3609619140625,
      "logits/chosen": -1.0693359375,
      "logits/rejected": -0.969531238079071,
      "logps/chosen": -0.6957031488418579,
      "logps/rejected": -1.9113280773162842,
      "loss": 0.8666,
      "nll_loss": 0.867968738079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06951904296875,
      "rewards/margins": 0.12156982719898224,
      "rewards/rejected": -0.19101563096046448,
      "step": 3420
    },
    {
      "epoch": 0.44263775971093045,
      "grad_norm": 2.3203170376268756,
      "learning_rate": 1.3659755535250212e-06,
      "log_odds_chosen": 1.600000023841858,
      "log_odds_ratio": -0.3653320372104645,
      "logits/chosen": -1.0642578601837158,
      "logits/rejected": -0.9624999761581421,
      "logps/chosen": -0.662304699420929,
      "logps/rejected": -1.786523461341858,
      "loss": 0.9004,
      "nll_loss": 0.819042980670929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06621094048023224,
      "rewards/margins": 0.11255493015050888,
      "rewards/rejected": -0.1788330078125,
      "step": 3430
    },
    {
      "epoch": 0.4439282488062976,
      "grad_norm": 2.464936546022166,
      "learning_rate": 1.3639886789409469e-06,
      "log_odds_chosen": 1.50927734375,
      "log_odds_ratio": -0.3860107362270355,
      "logits/chosen": -1.0626952648162842,
      "logits/rejected": -0.9501953125,
      "logps/chosen": -0.6585937738418579,
      "logps/rejected": -1.712890625,
      "loss": 0.889,
      "nll_loss": 0.8148437738418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06586913764476776,
      "rewards/margins": 0.10545043647289276,
      "rewards/rejected": -0.17136231064796448,
      "step": 3440
    },
    {
      "epoch": 0.4452187379016647,
      "grad_norm": 2.5381578045448725,
      "learning_rate": 1.3620104492139977e-06,
      "log_odds_chosen": 1.592187523841858,
      "log_odds_ratio": -0.3633789122104645,
      "logits/chosen": -1.0470702648162842,
      "logits/rejected": -0.9390624761581421,
      "logps/chosen": -0.7134765386581421,
      "logps/rejected": -1.8232421875,
      "loss": 0.8881,
      "nll_loss": 0.8501952886581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07138671725988388,
      "rewards/margins": 0.11092529445886612,
      "rewards/rejected": -0.1822509765625,
      "step": 3450
    },
    {
      "epoch": 0.4465092269970319,
      "grad_norm": 2.614834944471132,
      "learning_rate": 1.3600408018360918e-06,
      "log_odds_chosen": 1.6653320789337158,
      "log_odds_ratio": -0.43095701932907104,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.943554699420929,
      "logps/chosen": -0.715527355670929,
      "logps/rejected": -1.8957030773162842,
      "loss": 0.8935,
      "nll_loss": 0.890429675579071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07154540717601776,
      "rewards/margins": 0.11787567287683487,
      "rewards/rejected": -0.189453125,
      "step": 3460
    },
    {
      "epoch": 0.447799716092399,
      "grad_norm": 2.3921540414566103,
      "learning_rate": 1.3580796749300878e-06,
      "log_odds_chosen": 1.3718750476837158,
      "log_odds_ratio": -0.4267578125,
      "logits/chosen": -1.066796898841858,
      "logits/rejected": -0.962109386920929,
      "logps/chosen": -0.685742199420929,
      "logps/rejected": -1.6875,
      "loss": 0.8825,
      "nll_loss": 0.8519531488418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06854248046875,
      "rewards/margins": 0.10007324069738388,
      "rewards/rejected": -0.1688232421875,
      "step": 3470
    },
    {
      "epoch": 0.44909020518776616,
      "grad_norm": 2.387319748301138,
      "learning_rate": 1.3561270072416209e-06,
      "log_odds_chosen": 1.295019507408142,
      "log_odds_ratio": -0.4051757752895355,
      "logits/chosen": -1.1248047351837158,
      "logits/rejected": -1.003515601158142,
      "logps/chosen": -0.7017577886581421,
      "logps/rejected": -1.5890624523162842,
      "loss": 0.8777,
      "nll_loss": 0.8128906488418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07021484524011612,
      "rewards/margins": 0.08879394829273224,
      "rewards/rejected": -0.15898437798023224,
      "step": 3480
    },
    {
      "epoch": 0.4503806942831333,
      "grad_norm": 2.859274127179742,
      "learning_rate": 1.3541827381310652e-06,
      "log_odds_chosen": 1.846582055091858,
      "log_odds_ratio": -0.32634276151657104,
      "logits/chosen": -1.1056640148162842,
      "logits/rejected": -0.958984375,
      "logps/chosen": -0.6778320074081421,
      "logps/rejected": -2.012500047683716,
      "loss": 0.8619,
      "nll_loss": 0.8128906488418579,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06773681938648224,
      "rewards/margins": 0.133331298828125,
      "rewards/rejected": -0.20134277641773224,
      "step": 3490
    },
    {
      "epoch": 0.45167118337850043,
      "grad_norm": 2.5841134080316768,
      "learning_rate": 1.3522468075656264e-06,
      "log_odds_chosen": 1.5730469226837158,
      "log_odds_ratio": -0.35527342557907104,
      "logits/chosen": -1.0378906726837158,
      "logits/rejected": -0.911914050579071,
      "logps/chosen": -0.697265625,
      "logps/rejected": -1.818750023841858,
      "loss": 0.862,
      "nll_loss": 0.8358398675918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06966552883386612,
      "rewards/margins": 0.11213378608226776,
      "rewards/rejected": -0.18183593451976776,
      "step": 3500
    },
    {
      "epoch": 0.4529616724738676,
      "grad_norm": 2.5995306083743035,
      "learning_rate": 1.3503191561115553e-06,
      "log_odds_chosen": 1.490966796875,
      "log_odds_ratio": -0.39921873807907104,
      "logits/chosen": -1.0986328125,
      "logits/rejected": -0.991406261920929,
      "logps/chosen": -0.7051757574081421,
      "logps/rejected": -1.782812476158142,
      "loss": 0.849,
      "nll_loss": 0.860156238079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.070556640625,
      "rewards/margins": 0.10781250149011612,
      "rewards/rejected": -0.17824706435203552,
      "step": 3510
    },
    {
      "epoch": 0.45425216156923476,
      "grad_norm": 2.2407538716746496,
      "learning_rate": 1.348399724926484e-06,
      "log_odds_chosen": 1.5166015625,
      "log_odds_ratio": -0.40288084745407104,
      "logits/chosen": -1.059960961341858,
      "logits/rejected": -0.9624999761581421,
      "logps/chosen": -0.719921886920929,
      "logps/rejected": -1.8152344226837158,
      "loss": 0.8958,
      "nll_loss": 0.8470703363418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07196044921875,
      "rewards/margins": 0.109375,
      "rewards/rejected": -0.18129882216453552,
      "step": 3520
    },
    {
      "epoch": 0.45554265066460187,
      "grad_norm": 2.4320311327520505,
      "learning_rate": 1.346488455751882e-06,
      "log_odds_chosen": 1.497656226158142,
      "log_odds_ratio": -0.37409669160842896,
      "logits/chosen": -1.072656273841858,
      "logits/rejected": -0.969921886920929,
      "logps/chosen": -0.676464855670929,
      "logps/rejected": -1.693359375,
      "loss": 0.879,
      "nll_loss": 0.889453113079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06765136867761612,
      "rewards/margins": 0.10167236626148224,
      "rewards/rejected": -0.16933593153953552,
      "step": 3530
    },
    {
      "epoch": 0.45683313975996903,
      "grad_norm": 2.355337746677995,
      "learning_rate": 1.3445852909056286e-06,
      "log_odds_chosen": 1.5068359375,
      "log_odds_ratio": -0.3611816465854645,
      "logits/chosen": -1.095117211341858,
      "logits/rejected": -0.986523449420929,
      "logps/chosen": -0.6737304925918579,
      "logps/rejected": -1.692968726158142,
      "loss": 0.8753,
      "nll_loss": 0.8011718988418579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0673828125,
      "rewards/margins": 0.10189209133386612,
      "rewards/rejected": -0.1693115234375,
      "step": 3540
    },
    {
      "epoch": 0.4581236288553362,
      "grad_norm": 2.431782421623709,
      "learning_rate": 1.3426901732747024e-06,
      "log_odds_chosen": 1.7664062976837158,
      "log_odds_ratio": -0.33642578125,
      "logits/chosen": -1.0701172351837158,
      "logits/rejected": -0.9624999761581421,
      "logps/chosen": -0.666210949420929,
      "logps/rejected": -1.898828148841858,
      "loss": 0.8587,
      "nll_loss": 0.797656238079071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06660155951976776,
      "rewards/margins": 0.12335815280675888,
      "rewards/rejected": -0.18986816704273224,
      "step": 3550
    },
    {
      "epoch": 0.4594141179507033,
      "grad_norm": 2.735889802997097,
      "learning_rate": 1.3408030463079818e-06,
      "log_odds_chosen": 1.4411132335662842,
      "log_odds_ratio": -0.39599609375,
      "logits/chosen": -1.0486328601837158,
      "logits/rejected": -0.9320312738418579,
      "logps/chosen": -0.620410144329071,
      "logps/rejected": -1.619140625,
      "loss": 0.8415,
      "nll_loss": 0.8042968511581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06207275390625,
      "rewards/margins": 0.09987793117761612,
      "rewards/rejected": -0.161865234375,
      "step": 3560
    },
    {
      "epoch": 0.46070460704607047,
      "grad_norm": 2.54944844062609,
      "learning_rate": 1.3389238540091568e-06,
      "log_odds_chosen": 1.4348633289337158,
      "log_odds_ratio": -0.40522462129592896,
      "logits/chosen": -1.0675780773162842,
      "logits/rejected": -0.976757824420929,
      "logps/chosen": -0.722851574420929,
      "logps/rejected": -1.7218749523162842,
      "loss": 0.8534,
      "nll_loss": 0.824414074420929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07220458984375,
      "rewards/margins": 0.09993286430835724,
      "rewards/rejected": -0.1722412109375,
      "step": 3570
    },
    {
      "epoch": 0.4619950961414376,
      "grad_norm": 2.6000977753489347,
      "learning_rate": 1.337052540929751e-06,
      "log_odds_chosen": 1.5007812976837158,
      "log_odds_ratio": -0.3998046815395355,
      "logits/chosen": -1.0896484851837158,
      "logits/rejected": -0.958203136920929,
      "logps/chosen": -0.6898437738418579,
      "logps/rejected": -1.720703125,
      "loss": 0.8574,
      "nll_loss": 0.8167968988418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06895752251148224,
      "rewards/margins": 0.10316161811351776,
      "rewards/rejected": -0.17204590141773224,
      "step": 3580
    },
    {
      "epoch": 0.46328558523680474,
      "grad_norm": 2.708748085895321,
      "learning_rate": 1.33518905216225e-06,
      "log_odds_chosen": 1.475195288658142,
      "log_odds_ratio": -0.37128907442092896,
      "logits/chosen": -1.0626952648162842,
      "logits/rejected": -0.954882800579071,
      "logps/chosen": -0.66845703125,
      "logps/rejected": -1.677734375,
      "loss": 0.8693,
      "nll_loss": 0.8257812261581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06685791164636612,
      "rewards/margins": 0.10090331733226776,
      "rewards/rejected": -0.16765137016773224,
      "step": 3590
    },
    {
      "epoch": 0.4645760743321719,
      "grad_norm": 2.524927077133194,
      "learning_rate": 1.3333333333333332e-06,
      "log_odds_chosen": 1.327539086341858,
      "log_odds_ratio": -0.4326171875,
      "logits/chosen": -1.0851562023162842,
      "logits/rejected": -0.9609375,
      "logps/chosen": -0.6880859136581421,
      "logps/rejected": -1.6130859851837158,
      "loss": 0.866,
      "nll_loss": 0.849414050579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06873778998851776,
      "rewards/margins": 0.09244994819164276,
      "rewards/rejected": -0.1612548828125,
      "step": 3600
    },
    {
      "epoch": 0.465866563427539,
      "grad_norm": 2.2363576172848543,
      "learning_rate": 1.3314853305972122e-06,
      "log_odds_chosen": 1.397558569908142,
      "log_odds_ratio": -0.40083009004592896,
      "logits/chosen": -1.068359375,
      "logits/rejected": -0.9642578363418579,
      "logps/chosen": -0.697460949420929,
      "logps/rejected": -1.684179663658142,
      "loss": 0.8465,
      "nll_loss": 0.80078125,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06977538764476776,
      "rewards/margins": 0.09870605170726776,
      "rewards/rejected": -0.16843262314796448,
      "step": 3610
    },
    {
      "epoch": 0.4671570525229062,
      "grad_norm": 2.6249271120247144,
      "learning_rate": 1.3296449906290671e-06,
      "log_odds_chosen": 1.382568359375,
      "log_odds_ratio": -0.432373046875,
      "logits/chosen": -1.0349609851837158,
      "logits/rejected": -0.936718761920929,
      "logps/chosen": -0.6919921636581421,
      "logps/rejected": -1.6181640625,
      "loss": 0.8689,
      "nll_loss": 0.8109375238418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06923828274011612,
      "rewards/margins": 0.09257812798023224,
      "rewards/rejected": -0.16181640326976776,
      "step": 3620
    },
    {
      "epoch": 0.46844754161827334,
      "grad_norm": 2.578563053191326,
      "learning_rate": 1.3278122606185844e-06,
      "log_odds_chosen": 1.2776367664337158,
      "log_odds_ratio": -0.4400878846645355,
      "logits/chosen": -1.0642578601837158,
      "logits/rejected": -0.948437511920929,
      "logps/chosen": -0.695996105670929,
      "logps/rejected": -1.5724608898162842,
      "loss": 0.8793,
      "nll_loss": 0.835742175579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06954345852136612,
      "rewards/margins": 0.087615966796875,
      "rewards/rejected": -0.15732422471046448,
      "step": 3630
    },
    {
      "epoch": 0.46973803071364045,
      "grad_norm": 2.450028503350631,
      "learning_rate": 1.3259870882635918e-06,
      "log_odds_chosen": 1.767578125,
      "log_odds_ratio": -0.3069824278354645,
      "logits/chosen": -1.065039038658142,
      "logits/rejected": -0.912109375,
      "logps/chosen": -0.6629883050918579,
      "logps/rejected": -1.8777344226837158,
      "loss": 0.8746,
      "nll_loss": 0.8125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06630859524011612,
      "rewards/margins": 0.12153320014476776,
      "rewards/rejected": -0.18779297173023224,
      "step": 3640
    },
    {
      "epoch": 0.4710285198090076,
      "grad_norm": 2.4159409079053042,
      "learning_rate": 1.3241694217637886e-06,
      "log_odds_chosen": 1.747656226158142,
      "log_odds_ratio": -0.33515626192092896,
      "logits/chosen": -1.035742163658142,
      "logits/rejected": -0.8968750238418579,
      "logps/chosen": -0.673535168170929,
      "logps/rejected": -1.9246094226837158,
      "loss": 0.894,
      "nll_loss": 0.836132824420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06730957329273224,
      "rewards/margins": 0.12515869736671448,
      "rewards/rejected": -0.1923828125,
      "step": 3650
    },
    {
      "epoch": 0.4723190089043748,
      "grad_norm": 2.855575988151085,
      "learning_rate": 1.3223592098145723e-06,
      "log_odds_chosen": 1.5417969226837158,
      "log_odds_ratio": -0.3719238340854645,
      "logits/chosen": -1.0554687976837158,
      "logits/rejected": -0.9380859136581421,
      "logps/chosen": -0.6856445074081421,
      "logps/rejected": -1.7882812023162842,
      "loss": 0.841,
      "nll_loss": 0.8037109375,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06853027641773224,
      "rewards/margins": 0.11021728813648224,
      "rewards/rejected": -0.17875976860523224,
      "step": 3660
    },
    {
      "epoch": 0.4736094979997419,
      "grad_norm": 2.696404382752237,
      "learning_rate": 1.3205564016009555e-06,
      "log_odds_chosen": 1.599023461341858,
      "log_odds_ratio": -0.34077149629592896,
      "logits/chosen": -1.076171875,
      "logits/rejected": -0.9462890625,
      "logps/chosen": -0.65380859375,
      "logps/rejected": -1.7345702648162842,
      "loss": 0.823,
      "nll_loss": 0.827929675579071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0654296875,
      "rewards/margins": 0.10802154242992401,
      "rewards/rejected": -0.17341308295726776,
      "step": 3670
    },
    {
      "epoch": 0.47489998709510906,
      "grad_norm": 3.0594616016054825,
      "learning_rate": 1.318760946791574e-06,
      "log_odds_chosen": 1.578710913658142,
      "log_odds_ratio": -0.4005126953125,
      "logits/chosen": -1.0466797351837158,
      "logits/rejected": -0.920117199420929,
      "logps/chosen": -0.6878906488418579,
      "logps/rejected": -1.828515648841858,
      "loss": 0.835,
      "nll_loss": 0.8333984613418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06881103664636612,
      "rewards/margins": 0.1142578125,
      "rewards/rejected": -0.18305663764476776,
      "step": 3680
    },
    {
      "epoch": 0.47619047619047616,
      "grad_norm": 2.588316417436268,
      "learning_rate": 1.316972795532786e-06,
      "log_odds_chosen": 1.4377930164337158,
      "log_odds_ratio": -0.385986328125,
      "logits/chosen": -1.0818359851837158,
      "logits/rejected": -0.986132800579071,
      "logps/chosen": -0.668261706829071,
      "logps/rejected": -1.666406273841858,
      "loss": 0.8314,
      "nll_loss": 0.843554675579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06689453125,
      "rewards/margins": 0.09976806491613388,
      "rewards/rejected": -0.16665038466453552,
      "step": 3690
    },
    {
      "epoch": 0.47748096528584333,
      "grad_norm": 2.3661464569776856,
      "learning_rate": 1.3151918984428582e-06,
      "log_odds_chosen": 1.6865234375,
      "log_odds_ratio": -0.3580078184604645,
      "logits/chosen": -1.033789038658142,
      "logits/rejected": -0.9281250238418579,
      "logps/chosen": -0.63818359375,
      "logps/rejected": -1.832421898841858,
      "loss": 0.8465,
      "nll_loss": 0.7740234136581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06379394233226776,
      "rewards/margins": 0.1195068359375,
      "rewards/rejected": -0.18330077826976776,
      "step": 3700
    },
    {
      "epoch": 0.4787714543812105,
      "grad_norm": 2.440249569130522,
      "learning_rate": 1.313418206606237e-06,
      "log_odds_chosen": 1.474609375,
      "log_odds_ratio": -0.38715821504592896,
      "logits/chosen": -1.02734375,
      "logits/rejected": -0.9224609136581421,
      "logps/chosen": -0.6722656488418579,
      "logps/rejected": -1.7146484851837158,
      "loss": 0.8474,
      "nll_loss": 0.794726550579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06722412258386612,
      "rewards/margins": 0.10429687798023224,
      "rewards/rejected": -0.17138671875,
      "step": 3710
    },
    {
      "epoch": 0.4800619434765776,
      "grad_norm": 2.0848461858769096,
      "learning_rate": 1.3116516715679057e-06,
      "log_odds_chosen": 1.6337890625,
      "log_odds_ratio": -0.3718505799770355,
      "logits/chosen": -1.050195336341858,
      "logits/rejected": -0.930468738079071,
      "logps/chosen": -0.719921886920929,
      "logps/rejected": -1.9093749523162842,
      "loss": 0.8617,
      "nll_loss": 0.8490234613418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07193603366613388,
      "rewards/margins": 0.11881713569164276,
      "rewards/rejected": -0.19086913764476776,
      "step": 3720
    },
    {
      "epoch": 0.48135243257194477,
      "grad_norm": 2.3124162934563413,
      "learning_rate": 1.3098922453278258e-06,
      "log_odds_chosen": 1.3639647960662842,
      "log_odds_ratio": -0.406494140625,
      "logits/chosen": -1.0529296398162842,
      "logits/rejected": -0.9408203363418579,
      "logps/chosen": -0.7173827886581421,
      "logps/rejected": -1.635351538658142,
      "loss": 0.8547,
      "nll_loss": 0.8587890863418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07169189304113388,
      "rewards/margins": 0.09197998046875,
      "rewards/rejected": -0.16357421875,
      "step": 3730
    },
    {
      "epoch": 0.48264292166731193,
      "grad_norm": 2.537999603253272,
      "learning_rate": 1.3081398803354573e-06,
      "log_odds_chosen": 1.875585913658142,
      "log_odds_ratio": -0.3245605528354645,
      "logits/chosen": -1.072851538658142,
      "logits/rejected": -0.901171863079071,
      "logps/chosen": -0.6126953363418579,
      "logps/rejected": -1.950781226158142,
      "loss": 0.8454,
      "nll_loss": 0.8291015625,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06120605394244194,
      "rewards/margins": 0.13389892876148224,
      "rewards/rejected": -0.195068359375,
      "step": 3740
    },
    {
      "epoch": 0.48393341076267904,
      "grad_norm": 2.25452537342582,
      "learning_rate": 1.3063945294843617e-06,
      "log_odds_chosen": 1.2957031726837158,
      "log_odds_ratio": -0.4580078125,
      "logits/chosen": -1.0662109851837158,
      "logits/rejected": -0.9751952886581421,
      "logps/chosen": -0.746289074420929,
      "logps/rejected": -1.6640625,
      "loss": 0.8425,
      "nll_loss": 0.803906261920929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07465820014476776,
      "rewards/margins": 0.09194336086511612,
      "rewards/rejected": -0.16650390625,
      "step": 3750
    },
    {
      "epoch": 0.4852238998580462,
      "grad_norm": 2.2124065539598314,
      "learning_rate": 1.3046561461068843e-06,
      "log_odds_chosen": 1.595312476158142,
      "log_odds_ratio": -0.36894530057907104,
      "logits/chosen": -1.006250023841858,
      "logits/rejected": -0.8863281011581421,
      "logps/chosen": -0.6463867425918579,
      "logps/rejected": -1.7257812023162842,
      "loss": 0.8594,
      "nll_loss": 0.849804699420929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06459961086511612,
      "rewards/margins": 0.10786132514476776,
      "rewards/rejected": -0.17258301377296448,
      "step": 3760
    },
    {
      "epoch": 0.48651438895341337,
      "grad_norm": 2.6375437456816817,
      "learning_rate": 1.3029246839689124e-06,
      "log_odds_chosen": 1.6056640148162842,
      "log_odds_ratio": -0.361328125,
      "logits/chosen": -1.0224609375,
      "logits/rejected": -0.8871093988418579,
      "logps/chosen": -0.670214831829071,
      "logps/rejected": -1.8308594226837158,
      "loss": 0.8577,
      "nll_loss": 0.764453113079071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06700439751148224,
      "rewards/margins": 0.11624755710363388,
      "rewards/rejected": -0.18330077826976776,
      "step": 3770
    },
    {
      "epoch": 0.4878048780487805,
      "grad_norm": 2.4608468226434894,
      "learning_rate": 1.3012000972647109e-06,
      "log_odds_chosen": 1.556884765625,
      "log_odds_ratio": -0.39252930879592896,
      "logits/chosen": -1.060937523841858,
      "logits/rejected": -0.9634765386581421,
      "logps/chosen": -0.7132812738418579,
      "logps/rejected": -1.8605468273162842,
      "loss": 0.854,
      "nll_loss": 0.7972656488418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0712890625,
      "rewards/margins": 0.11475829780101776,
      "rewards/rejected": -0.18601074814796448,
      "step": 3780
    },
    {
      "epoch": 0.48909536714414764,
      "grad_norm": 2.7319155581324743,
      "learning_rate": 1.299482340611832e-06,
      "log_odds_chosen": 1.2722656726837158,
      "log_odds_ratio": -0.44746094942092896,
      "logits/chosen": -0.9912109375,
      "logits/rejected": -0.89453125,
      "logps/chosen": -0.7152343988418579,
      "logps/rejected": -1.6296875476837158,
      "loss": 0.8459,
      "nll_loss": 0.8414062261581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07149658352136612,
      "rewards/margins": 0.09132079780101776,
      "rewards/rejected": -0.16293945908546448,
      "step": 3790
    },
    {
      "epoch": 0.49038585623951475,
      "grad_norm": 2.685035139375476,
      "learning_rate": 1.2977713690461003e-06,
      "log_odds_chosen": 1.466064453125,
      "log_odds_ratio": -0.40571290254592896,
      "logits/chosen": -1.059179663658142,
      "logits/rejected": -0.908007800579071,
      "logps/chosen": -0.689257800579071,
      "logps/rejected": -1.7492187023162842,
      "loss": 0.8568,
      "nll_loss": 0.818359375,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06895752251148224,
      "rewards/margins": 0.10599365085363388,
      "rewards/rejected": -0.17482909560203552,
      "step": 3800
    },
    {
      "epoch": 0.4916763453348819,
      "grad_norm": 3.0637361776928285,
      "learning_rate": 1.296067138016669e-06,
      "log_odds_chosen": 1.38720703125,
      "log_odds_ratio": -0.3975585997104645,
      "logits/chosen": -1.0382812023162842,
      "logits/rejected": -0.9203125238418579,
      "logps/chosen": -0.6732422113418579,
      "logps/rejected": -1.645117163658142,
      "loss": 0.8584,
      "nll_loss": 0.733691394329071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0672607421875,
      "rewards/margins": 0.0970458984375,
      "rewards/rejected": -0.16447754204273224,
      "step": 3810
    },
    {
      "epoch": 0.4929668344302491,
      "grad_norm": 2.4385703263932643,
      "learning_rate": 1.294369603381147e-06,
      "log_odds_chosen": 1.47607421875,
      "log_odds_ratio": -0.39453125,
      "logits/chosen": -1.043359398841858,
      "logits/rejected": -0.9214843511581421,
      "logps/chosen": -0.667187511920929,
      "logps/rejected": -1.747656226158142,
      "loss": 0.8536,
      "nll_loss": 0.8675781488418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06674804538488388,
      "rewards/margins": 0.108184814453125,
      "rewards/rejected": -0.1748046875,
      "step": 3820
    },
    {
      "epoch": 0.4942573235256162,
      "grad_norm": 2.1520107065440723,
      "learning_rate": 1.2926787214007981e-06,
      "log_odds_chosen": 1.470117211341858,
      "log_odds_ratio": -0.39018553495407104,
      "logits/chosen": -1.0505859851837158,
      "logits/rejected": -0.9712890386581421,
      "logps/chosen": -0.6607421636581421,
      "logps/rejected": -1.6603515148162842,
      "loss": 0.8292,
      "nll_loss": 0.7417968511581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06614990532398224,
      "rewards/margins": 0.09978027641773224,
      "rewards/rejected": -0.1658935546875,
      "step": 3830
    },
    {
      "epoch": 0.49554781262098335,
      "grad_norm": 2.900804359228131,
      "learning_rate": 1.2909944487358056e-06,
      "log_odds_chosen": 1.7156250476837158,
      "log_odds_ratio": -0.3372802734375,
      "logits/chosen": -1.0773437023162842,
      "logits/rejected": -0.913867175579071,
      "logps/chosen": -0.6666015386581421,
      "logps/rejected": -1.8777344226837158,
      "loss": 0.8457,
      "nll_loss": 0.835156261920929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06660155951976776,
      "rewards/margins": 0.1212158203125,
      "rewards/rejected": -0.1876220703125,
      "step": 3840
    },
    {
      "epoch": 0.4968383017163505,
      "grad_norm": 2.9133134378304737,
      "learning_rate": 1.2893167424406084e-06,
      "log_odds_chosen": 1.654882788658142,
      "log_odds_ratio": -0.37666016817092896,
      "logits/chosen": -1.055273413658142,
      "logits/rejected": -0.9384765625,
      "logps/chosen": -0.678515613079071,
      "logps/rejected": -1.8828125,
      "loss": 0.8245,
      "nll_loss": 0.7928711175918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06781005859375,
      "rewards/margins": 0.12053222954273224,
      "rewards/rejected": -0.18842773139476776,
      "step": 3850
    },
    {
      "epoch": 0.4981287908117176,
      "grad_norm": 2.2769260083304057,
      "learning_rate": 1.2876455599593008e-06,
      "log_odds_chosen": 1.4285156726837158,
      "log_odds_ratio": -0.42438966035842896,
      "logits/chosen": -1.041406273841858,
      "logits/rejected": -0.949999988079071,
      "logps/chosen": -0.741015613079071,
      "logps/rejected": -1.7472655773162842,
      "loss": 0.8582,
      "nll_loss": 0.8232421875,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07413329929113388,
      "rewards/margins": 0.10067443549633026,
      "rewards/rejected": -0.1746826171875,
      "step": 3860
    },
    {
      "epoch": 0.4994192799070848,
      "grad_norm": 2.336111883760762,
      "learning_rate": 1.285980859121099e-06,
      "log_odds_chosen": 1.479101538658142,
      "log_odds_ratio": -0.4425292909145355,
      "logits/chosen": -1.0134766101837158,
      "logits/rejected": -0.888671875,
      "logps/chosen": -0.686816394329071,
      "logps/rejected": -1.7527344226837158,
      "loss": 0.845,
      "nll_loss": 0.844921886920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06868896633386612,
      "rewards/margins": 0.10646972805261612,
      "rewards/rejected": -0.17529296875,
      "step": 3870
    },
    {
      "epoch": 0.5007097690024519,
      "grad_norm": 2.1732451935507946,
      "learning_rate": 1.2843225981358712e-06,
      "log_odds_chosen": 1.6794922351837158,
      "log_odds_ratio": -0.3609375059604645,
      "logits/chosen": -1.1142578125,
      "logits/rejected": -0.975781261920929,
      "logps/chosen": -0.6463867425918579,
      "logps/rejected": -1.865820288658142,
      "loss": 0.8443,
      "nll_loss": 0.726367175579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06459961086511612,
      "rewards/margins": 0.12210693210363388,
      "rewards/rejected": -0.18654784560203552,
      "step": 3880
    },
    {
      "epoch": 0.5020002580978191,
      "grad_norm": 2.4879510610706483,
      "learning_rate": 1.2826707355897317e-06,
      "log_odds_chosen": 1.265234351158142,
      "log_odds_ratio": -0.443359375,
      "logits/chosen": -1.08203125,
      "logits/rejected": -0.952343761920929,
      "logps/chosen": -0.717578113079071,
      "logps/rejected": -1.644140601158142,
      "loss": 0.8688,
      "nll_loss": 0.8021484613418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07169189304113388,
      "rewards/margins": 0.09269104152917862,
      "rewards/rejected": -0.1644287109375,
      "step": 3890
    },
    {
      "epoch": 0.5032907471931862,
      "grad_norm": 2.797260062678071,
      "learning_rate": 1.281025230440697e-06,
      "log_odds_chosen": 1.797460913658142,
      "log_odds_ratio": -0.31376951932907104,
      "logits/chosen": -1.068359375,
      "logits/rejected": -0.9097656011581421,
      "logps/chosen": -0.67138671875,
      "logps/rejected": -1.959375023841858,
      "loss": 0.8421,
      "nll_loss": 0.7933593988418579,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.067138671875,
      "rewards/margins": 0.12886962294578552,
      "rewards/rejected": -0.19589844346046448,
      "step": 3900
    },
    {
      "epoch": 0.5045812362885533,
      "grad_norm": 2.686603777062713,
      "learning_rate": 1.2793860420144025e-06,
      "log_odds_chosen": 1.578027367591858,
      "log_odds_ratio": -0.387451171875,
      "logits/chosen": -1.0617187023162842,
      "logits/rejected": -0.965624988079071,
      "logps/chosen": -0.6650390625,
      "logps/rejected": -1.7228515148162842,
      "loss": 0.845,
      "nll_loss": 0.821484386920929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06646728515625,
      "rewards/margins": 0.10585327446460724,
      "rewards/rejected": -0.17250975966453552,
      "step": 3910
    },
    {
      "epoch": 0.5058717253839206,
      "grad_norm": 2.5013992929746145,
      "learning_rate": 1.2777531299998798e-06,
      "log_odds_chosen": 1.508398413658142,
      "log_odds_ratio": -0.36689454317092896,
      "logits/chosen": -1.083593726158142,
      "logits/rejected": -0.9361327886581421,
      "logps/chosen": -0.6597656011581421,
      "logps/rejected": -1.696874976158142,
      "loss": 0.8363,
      "nll_loss": 0.786328136920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06594238430261612,
      "rewards/margins": 0.10379638522863388,
      "rewards/rejected": -0.1697998046875,
      "step": 3920
    },
    {
      "epoch": 0.5071622144792877,
      "grad_norm": 2.516345041094057,
      "learning_rate": 1.2761264544453928e-06,
      "log_odds_chosen": 1.5810546875,
      "log_odds_ratio": -0.3462890684604645,
      "logits/chosen": -1.0720703601837158,
      "logits/rejected": -0.939257800579071,
      "logps/chosen": -0.654101550579071,
      "logps/rejected": -1.755468726158142,
      "loss": 0.8278,
      "nll_loss": 0.794726550579071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06544189155101776,
      "rewards/margins": 0.1099853515625,
      "rewards/rejected": -0.17548827826976776,
      "step": 3930
    },
    {
      "epoch": 0.5084527035746548,
      "grad_norm": 2.6692811241221257,
      "learning_rate": 1.2745059757543324e-06,
      "log_odds_chosen": 1.4326171875,
      "log_odds_ratio": -0.40800780057907104,
      "logits/chosen": -1.058984398841858,
      "logits/rejected": -0.947460949420929,
      "logps/chosen": -0.719921886920929,
      "logps/rejected": -1.7101562023162842,
      "loss": 0.8209,
      "nll_loss": 0.813281238079071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07197265326976776,
      "rewards/margins": 0.09909667819738388,
      "rewards/rejected": -0.17106933891773224,
      "step": 3940
    },
    {
      "epoch": 0.5097431926700219,
      "grad_norm": 3.0333885010168897,
      "learning_rate": 1.272891654681168e-06,
      "log_odds_chosen": 1.4445312023162842,
      "log_odds_ratio": -0.4061523377895355,
      "logits/chosen": -1.1062500476837158,
      "logits/rejected": -0.952343761920929,
      "logps/chosen": -0.70703125,
      "logps/rejected": -1.7414062023162842,
      "loss": 0.8376,
      "nll_loss": 0.843554675579071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07078857719898224,
      "rewards/margins": 0.10325928032398224,
      "rewards/rejected": -0.17412109673023224,
      "step": 3950
    },
    {
      "epoch": 0.5110336817653891,
      "grad_norm": 2.2874471381641133,
      "learning_rate": 1.2712834523274563e-06,
      "log_odds_chosen": 1.783300757408142,
      "log_odds_ratio": -0.3387695252895355,
      "logits/chosen": -1.024999976158142,
      "logits/rejected": -0.9214843511581421,
      "logps/chosen": -0.6729491949081421,
      "logps/rejected": -1.9611327648162842,
      "loss": 0.8309,
      "nll_loss": 0.8101562261581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06727294623851776,
      "rewards/margins": 0.12888184189796448,
      "rewards/rejected": -0.19609375298023224,
      "step": 3960
    },
    {
      "epoch": 0.5123241708607562,
      "grad_norm": 2.8071092063209857,
      "learning_rate": 1.2696813301379032e-06,
      "log_odds_chosen": 1.501953125,
      "log_odds_ratio": -0.38056641817092896,
      "logits/chosen": -1.0666015148162842,
      "logits/rejected": -0.9261718988418579,
      "logps/chosen": -0.706250011920929,
      "logps/rejected": -1.7546875476837158,
      "loss": 0.8498,
      "nll_loss": 0.864453136920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07060547173023224,
      "rewards/margins": 0.10499267280101776,
      "rewards/rejected": -0.17546387016773224,
      "step": 3970
    },
    {
      "epoch": 0.5136146599561233,
      "grad_norm": 2.287434230454512,
      "learning_rate": 1.2680852498964829e-06,
      "log_odds_chosen": 1.4835937023162842,
      "log_odds_ratio": -0.39892578125,
      "logits/chosen": -1.0652344226837158,
      "logits/rejected": -0.9144531488418579,
      "logps/chosen": -0.6044921875,
      "logps/rejected": -1.628320336341858,
      "loss": 0.8172,
      "nll_loss": 0.7744140625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06040038913488388,
      "rewards/margins": 0.10217895358800888,
      "rewards/rejected": -0.1627197265625,
      "step": 3980
    },
    {
      "epoch": 0.5149051490514905,
      "grad_norm": 2.8342443441574527,
      "learning_rate": 1.266495173722607e-06,
      "log_odds_chosen": 1.4113280773162842,
      "log_odds_ratio": -0.4195312559604645,
      "logits/chosen": -1.052734375,
      "logits/rejected": -0.9449218511581421,
      "logps/chosen": -0.6669921875,
      "logps/rejected": -1.6388671398162842,
      "loss": 0.8364,
      "nll_loss": 0.8109375238418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06669922173023224,
      "rewards/margins": 0.09712524712085724,
      "rewards/rejected": -0.16384276747703552,
      "step": 3990
    },
    {
      "epoch": 0.5161956381468576,
      "grad_norm": 2.369981891547595,
      "learning_rate": 1.2649110640673517e-06,
      "log_odds_chosen": 1.516210913658142,
      "log_odds_ratio": -0.3704589903354645,
      "logits/chosen": -1.0226562023162842,
      "logits/rejected": -0.906445324420929,
      "logps/chosen": -0.654003918170929,
      "logps/rejected": -1.6925780773162842,
      "loss": 0.8584,
      "nll_loss": 0.7791992425918579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06546630710363388,
      "rewards/margins": 0.10397949069738388,
      "rewards/rejected": -0.1693115234375,
      "step": 4000
    },
    {
      "epoch": 0.5174861272422248,
      "grad_norm": 2.4908159115194737,
      "learning_rate": 1.2633328837097308e-06,
      "log_odds_chosen": 1.726171851158142,
      "log_odds_ratio": -0.36884766817092896,
      "logits/chosen": -1.021484375,
      "logits/rejected": -0.890625,
      "logps/chosen": -0.68212890625,
      "logps/rejected": -1.9501953125,
      "loss": 0.858,
      "nll_loss": 0.853710949420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06816406548023224,
      "rewards/margins": 0.12681274116039276,
      "rewards/rejected": -0.19509276747703552,
      "step": 4010
    },
    {
      "epoch": 0.518776616337592,
      "grad_norm": 2.790745875044973,
      "learning_rate": 1.2617605957530233e-06,
      "log_odds_chosen": 1.6570312976837158,
      "log_odds_ratio": -0.3556152284145355,
      "logits/chosen": -1.0185546875,
      "logits/rejected": -0.871874988079071,
      "logps/chosen": -0.7236328125,
      "logps/rejected": -1.9148437976837158,
      "loss": 0.8242,
      "nll_loss": 0.8251953125,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07236327975988388,
      "rewards/margins": 0.11918945610523224,
      "rewards/rejected": -0.191650390625,
      "step": 4020
    },
    {
      "epoch": 0.5200671054329591,
      "grad_norm": 2.553128604245277,
      "learning_rate": 1.2601941636211516e-06,
      "log_odds_chosen": 1.5212891101837158,
      "log_odds_ratio": -0.38203126192092896,
      "logits/chosen": -1.064453125,
      "logits/rejected": -0.966992199420929,
      "logps/chosen": -0.642871081829071,
      "logps/rejected": -1.650976538658142,
      "loss": 0.8375,
      "nll_loss": 0.731249988079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06424560397863388,
      "rewards/margins": 0.100830078125,
      "rewards/rejected": -0.16513672471046448,
      "step": 4030
    },
    {
      "epoch": 0.5213575945283262,
      "grad_norm": 2.8625778741989163,
      "learning_rate": 1.2586335510551052e-06,
      "log_odds_chosen": 1.6779296398162842,
      "log_odds_ratio": -0.334716796875,
      "logits/chosen": -1.053320288658142,
      "logits/rejected": -0.9447265863418579,
      "logps/chosen": -0.6539062261581421,
      "logps/rejected": -1.8203125,
      "loss": 0.824,
      "nll_loss": 0.7708984613418579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06534423679113388,
      "rewards/margins": 0.11679687350988388,
      "rewards/rejected": -0.18208007514476776,
      "step": 4040
    },
    {
      "epoch": 0.5226480836236934,
      "grad_norm": 2.45624082459909,
      "learning_rate": 1.2570787221094177e-06,
      "log_odds_chosen": 1.403906226158142,
      "log_odds_ratio": -0.41166990995407104,
      "logits/chosen": -1.0935547351837158,
      "logits/rejected": -0.9912109375,
      "logps/chosen": -0.6732422113418579,
      "logps/rejected": -1.6124999523162842,
      "loss": 0.8339,
      "nll_loss": 0.7623046636581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06732177734375,
      "rewards/margins": 0.09393310546875,
      "rewards/rejected": -0.1612548828125,
      "step": 4050
    },
    {
      "epoch": 0.5239385727190605,
      "grad_norm": 2.841956384784609,
      "learning_rate": 1.255529641148689e-06,
      "log_odds_chosen": 1.53564453125,
      "log_odds_ratio": -0.39545899629592896,
      "logits/chosen": -1.060546875,
      "logits/rejected": -0.9681640863418579,
      "logps/chosen": -0.652050793170929,
      "logps/rejected": -1.7101562023162842,
      "loss": 0.7998,
      "nll_loss": 0.7323242425918579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06516113132238388,
      "rewards/margins": 0.10571899265050888,
      "rewards/rejected": -0.17097167670726776,
      "step": 4060
    },
    {
      "epoch": 0.5252290618144276,
      "grad_norm": 2.8140012392029554,
      "learning_rate": 1.2539862728441536e-06,
      "log_odds_chosen": 1.6784179210662842,
      "log_odds_ratio": -0.38007813692092896,
      "logits/chosen": -1.0783202648162842,
      "logits/rejected": -0.9603515863418579,
      "logps/chosen": -0.698535144329071,
      "logps/rejected": -1.9230468273162842,
      "loss": 0.834,
      "nll_loss": 0.80078125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06987304985523224,
      "rewards/margins": 0.12259521335363388,
      "rewards/rejected": -0.19252929091453552,
      "step": 4070
    },
    {
      "epoch": 0.5265195509097949,
      "grad_norm": 2.271853424274781,
      "learning_rate": 1.252448582170299e-06,
      "log_odds_chosen": 1.454687476158142,
      "log_odds_ratio": -0.38823240995407104,
      "logits/chosen": -1.056054711341858,
      "logits/rejected": -0.950976550579071,
      "logps/chosen": -0.628710925579071,
      "logps/rejected": -1.616796851158142,
      "loss": 0.8304,
      "nll_loss": 0.7508789300918579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06287841498851776,
      "rewards/margins": 0.09888305515050888,
      "rewards/rejected": -0.16169433295726776,
      "step": 4080
    },
    {
      "epoch": 0.527810040005162,
      "grad_norm": 2.510336593729211,
      "learning_rate": 1.2509165344015243e-06,
      "log_odds_chosen": 1.474609375,
      "log_odds_ratio": -0.407470703125,
      "logits/chosen": -1.1335937976837158,
      "logits/rejected": -0.9937499761581421,
      "logps/chosen": -0.6634765863418579,
      "logps/rejected": -1.7109375,
      "loss": 0.8174,
      "nll_loss": 0.7255859375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06638183444738388,
      "rewards/margins": 0.10477294772863388,
      "rewards/rejected": -0.17116698622703552,
      "step": 4090
    },
    {
      "epoch": 0.5291005291005291,
      "grad_norm": 2.519643315836529,
      "learning_rate": 1.2493900951088486e-06,
      "log_odds_chosen": 1.761132836341858,
      "log_odds_ratio": -0.363525390625,
      "logits/chosen": -1.0751953125,
      "logits/rejected": -0.936328113079071,
      "logps/chosen": -0.6597656011581421,
      "logps/rejected": -1.9113280773162842,
      "loss": 0.8167,
      "nll_loss": 0.8013671636581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06596679985523224,
      "rewards/margins": 0.12529297173023224,
      "rewards/rejected": -0.19130858778953552,
      "step": 4100
    },
    {
      "epoch": 0.5303910181958963,
      "grad_norm": 2.5901038226180986,
      "learning_rate": 1.2478692301566601e-06,
      "log_odds_chosen": 1.5460937023162842,
      "log_odds_ratio": -0.35676270723342896,
      "logits/chosen": -1.058007836341858,
      "logits/rejected": -0.9224609136581421,
      "logps/chosen": -0.658398449420929,
      "logps/rejected": -1.7179687023162842,
      "loss": 0.8102,
      "nll_loss": 0.8306640386581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06584472954273224,
      "rewards/margins": 0.10590820014476776,
      "rewards/rejected": -0.17165526747703552,
      "step": 4110
    },
    {
      "epoch": 0.5316815072912634,
      "grad_norm": 2.63634578846919,
      "learning_rate": 1.2463539056995116e-06,
      "log_odds_chosen": 1.5275390148162842,
      "log_odds_ratio": -0.39453125,
      "logits/chosen": -1.095117211341858,
      "logits/rejected": -0.9867187738418579,
      "logps/chosen": -0.67919921875,
      "logps/rejected": -1.725000023841858,
      "loss": 0.8347,
      "nll_loss": 0.7583984136581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06788329780101776,
      "rewards/margins": 0.10452880710363388,
      "rewards/rejected": -0.17246094346046448,
      "step": 4120
    },
    {
      "epoch": 0.5329719963866305,
      "grad_norm": 2.562932461591451,
      "learning_rate": 1.2448440881789541e-06,
      "log_odds_chosen": 1.495996117591858,
      "log_odds_ratio": -0.4166015684604645,
      "logits/chosen": -1.0544922351837158,
      "logits/rejected": -0.9576171636581421,
      "logps/chosen": -0.696484386920929,
      "logps/rejected": -1.775781273841858,
      "loss": 0.8334,
      "nll_loss": 0.875683605670929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06964111328125,
      "rewards/margins": 0.10804443061351776,
      "rewards/rejected": -0.17763671278953552,
      "step": 4130
    },
    {
      "epoch": 0.5342624854819977,
      "grad_norm": 2.6174762618546654,
      "learning_rate": 1.2433397443204184e-06,
      "log_odds_chosen": 1.371191382408142,
      "log_odds_ratio": -0.445068359375,
      "logits/chosen": -1.0441405773162842,
      "logits/rejected": -0.9228515625,
      "logps/chosen": -0.739062488079071,
      "logps/rejected": -1.746484398841858,
      "loss": 0.841,
      "nll_loss": 0.826953113079071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07392577826976776,
      "rewards/margins": 0.100677490234375,
      "rewards/rejected": -0.17458495497703552,
      "step": 4140
    },
    {
      "epoch": 0.5355529745773648,
      "grad_norm": 2.493754363400268,
      "learning_rate": 1.2418408411301324e-06,
      "log_odds_chosen": 1.354394555091858,
      "log_odds_ratio": -0.42509764432907104,
      "logits/chosen": -1.0779297351837158,
      "logits/rejected": -0.9624999761581421,
      "logps/chosen": -0.7396484613418579,
      "logps/rejected": -1.6945312023162842,
      "loss": 0.848,
      "nll_loss": 0.7964843511581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07406006008386612,
      "rewards/margins": 0.09549560397863388,
      "rewards/rejected": -0.16962890326976776,
      "step": 4150
    },
    {
      "epoch": 0.536843463672732,
      "grad_norm": 2.5670495101005577,
      "learning_rate": 1.2403473458920844e-06,
      "log_odds_chosen": 1.3720703125,
      "log_odds_ratio": -0.45048826932907104,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.918164074420929,
      "logps/chosen": -0.7123047113418579,
      "logps/rejected": -1.708593726158142,
      "loss": 0.8393,
      "nll_loss": 0.8578125238418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07121582329273224,
      "rewards/margins": 0.09967041015625,
      "rewards/rejected": -0.17087402939796448,
      "step": 4160
    },
    {
      "epoch": 0.538133952768099,
      "grad_norm": 3.1208171665293474,
      "learning_rate": 1.2388592261650217e-06,
      "log_odds_chosen": 1.7294921875,
      "log_odds_ratio": -0.33740234375,
      "logits/chosen": -1.0986328125,
      "logits/rejected": -0.9439452886581421,
      "logps/chosen": -0.7085937261581421,
      "logps/rejected": -1.9230468273162842,
      "loss": 0.8435,
      "nll_loss": 0.771679699420929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.07094726711511612,
      "rewards/margins": 0.12132568657398224,
      "rewards/rejected": -0.192138671875,
      "step": 4170
    },
    {
      "epoch": 0.5394244418634663,
      "grad_norm": 2.8264845447788827,
      "learning_rate": 1.2373764497794918e-06,
      "log_odds_chosen": 1.4679687023162842,
      "log_odds_ratio": -0.3703857362270355,
      "logits/chosen": -1.0427734851837158,
      "logits/rejected": -0.9117187261581421,
      "logps/chosen": -0.6846679449081421,
      "logps/rejected": -1.69921875,
      "loss": 0.8496,
      "nll_loss": 0.837597668170929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06840820610523224,
      "rewards/margins": 0.10140381008386612,
      "rewards/rejected": -0.16982421278953552,
      "step": 4180
    },
    {
      "epoch": 0.5407149309588334,
      "grad_norm": 3.018862763439587,
      "learning_rate": 1.2358989848349217e-06,
      "log_odds_chosen": 1.456762671470642,
      "log_odds_ratio": -0.42255860567092896,
      "logits/chosen": -1.009765625,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.701464831829071,
      "logps/rejected": -1.7058594226837158,
      "loss": 0.819,
      "nll_loss": 0.8031250238418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07010497897863388,
      "rewards/margins": 0.10056152194738388,
      "rewards/rejected": -0.1705322265625,
      "step": 4190
    },
    {
      "epoch": 0.5420054200542005,
      "grad_norm": 2.435208998660067,
      "learning_rate": 1.2344267996967353e-06,
      "log_odds_chosen": 1.3777344226837158,
      "log_odds_ratio": -0.43876951932907104,
      "logits/chosen": -1.083593726158142,
      "logits/rejected": -0.9691406488418579,
      "logps/chosen": -0.702343761920929,
      "logps/rejected": -1.7091796398162842,
      "loss": 0.8115,
      "nll_loss": 0.7728515863418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07026366889476776,
      "rewards/margins": 0.10076294094324112,
      "rewards/rejected": -0.17094726860523224,
      "step": 4200
    },
    {
      "epoch": 0.5432959091495677,
      "grad_norm": 2.699440157377714,
      "learning_rate": 1.2329598629935076e-06,
      "log_odds_chosen": 1.4982421398162842,
      "log_odds_ratio": -0.36884766817092896,
      "logits/chosen": -1.0828125476837158,
      "logits/rejected": -0.9498046636581421,
      "logps/chosen": -0.6668945550918579,
      "logps/rejected": -1.6912109851837158,
      "loss": 0.8388,
      "nll_loss": 0.7666015625,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06671142578125,
      "rewards/margins": 0.10252074897289276,
      "rewards/rejected": -0.16914062201976776,
      "step": 4210
    },
    {
      "epoch": 0.5445863982449348,
      "grad_norm": 2.695435340393188,
      "learning_rate": 1.2314981436141583e-06,
      "log_odds_chosen": 1.3699462413787842,
      "log_odds_ratio": -0.4139648377895355,
      "logits/chosen": -1.133203148841858,
      "logits/rejected": -0.99609375,
      "logps/chosen": -0.68408203125,
      "logps/rejected": -1.630468726158142,
      "loss": 0.807,
      "nll_loss": 0.7509765625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06828613579273224,
      "rewards/margins": 0.09479675441980362,
      "rewards/rejected": -0.16318359971046448,
      "step": 4220
    },
    {
      "epoch": 0.5458768873403019,
      "grad_norm": 2.627157806650008,
      "learning_rate": 1.2300416107051802e-06,
      "log_odds_chosen": 1.381738305091858,
      "log_odds_ratio": -0.39599609375,
      "logits/chosen": -1.104882836341858,
      "logits/rejected": -0.9986327886581421,
      "logps/chosen": -0.703417956829071,
      "logps/rejected": -1.6746094226837158,
      "loss": 0.8135,
      "nll_loss": 0.7591797113418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07030029594898224,
      "rewards/margins": 0.09711913764476776,
      "rewards/rejected": -0.16745606064796448,
      "step": 4230
    },
    {
      "epoch": 0.5471673764356692,
      "grad_norm": 2.571691767693838,
      "learning_rate": 1.2285902336679024e-06,
      "log_odds_chosen": 1.5400390625,
      "log_odds_ratio": -0.384521484375,
      "logits/chosen": -1.058203101158142,
      "logits/rejected": -0.9498046636581421,
      "logps/chosen": -0.6913086175918579,
      "logps/rejected": -1.776953101158142,
      "loss": 0.822,
      "nll_loss": 0.748730480670929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06912841647863388,
      "rewards/margins": 0.10844726860523224,
      "rewards/rejected": -0.17768554389476776,
      "step": 4240
    },
    {
      "epoch": 0.5484578655310363,
      "grad_norm": 2.728944464231484,
      "learning_rate": 1.2271439821557926e-06,
      "log_odds_chosen": 1.962890625,
      "log_odds_ratio": -0.28620606660842896,
      "logits/chosen": -1.062890648841858,
      "logits/rejected": -0.9501953125,
      "logps/chosen": -0.6533203125,
      "logps/rejected": -2.026171922683716,
      "loss": 0.8116,
      "nll_loss": 0.7978515625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0653076171875,
      "rewards/margins": 0.13734130561351776,
      "rewards/rejected": -0.20283202826976776,
      "step": 4250
    },
    {
      "epoch": 0.5497483546264034,
      "grad_norm": 2.5477231544551326,
      "learning_rate": 1.225702826071791e-06,
      "log_odds_chosen": 1.6251952648162842,
      "log_odds_ratio": -0.375732421875,
      "logits/chosen": -1.0539062023162842,
      "logits/rejected": -0.9263671636581421,
      "logps/chosen": -0.7210937738418579,
      "logps/rejected": -1.892187476158142,
      "loss": 0.8288,
      "nll_loss": 0.80859375,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07204589992761612,
      "rewards/margins": 0.11710204929113388,
      "rewards/rejected": -0.18911132216453552,
      "step": 4260
    },
    {
      "epoch": 0.5510388437217706,
      "grad_norm": 2.729350231046786,
      "learning_rate": 1.2242667355656797e-06,
      "log_odds_chosen": 1.6650390625,
      "log_odds_ratio": -0.35380858182907104,
      "logits/chosen": -1.0310547351837158,
      "logits/rejected": -0.908984363079071,
      "logps/chosen": -0.657031238079071,
      "logps/rejected": -1.778906226158142,
      "loss": 0.8193,
      "nll_loss": 0.8363281488418579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06572265923023224,
      "rewards/margins": 0.11223144829273224,
      "rewards/rejected": -0.17795410752296448,
      "step": 4270
    },
    {
      "epoch": 0.5523293328171377,
      "grad_norm": 2.5689875286830786,
      "learning_rate": 1.2228356810314862e-06,
      "log_odds_chosen": 1.3904297351837158,
      "log_odds_ratio": -0.4107421934604645,
      "logits/chosen": -1.0710937976837158,
      "logits/rejected": -0.9517577886581421,
      "logps/chosen": -0.6927734613418579,
      "logps/rejected": -1.658203125,
      "loss": 0.8223,
      "nll_loss": 0.7867187261581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06932373344898224,
      "rewards/margins": 0.09654541313648224,
      "rewards/rejected": -0.1658935546875,
      "step": 4280
    },
    {
      "epoch": 0.5536198219125048,
      "grad_norm": 3.0683278381855357,
      "learning_rate": 1.2214096331049186e-06,
      "log_odds_chosen": 1.407324194908142,
      "log_odds_ratio": -0.41791993379592896,
      "logits/chosen": -1.0310547351837158,
      "logits/rejected": -0.8951171636581421,
      "logps/chosen": -0.7149413824081421,
      "logps/rejected": -1.731835961341858,
      "loss": 0.8254,
      "nll_loss": 0.782421886920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07148437201976776,
      "rewards/margins": 0.10164642333984375,
      "rewards/rejected": -0.17326660454273224,
      "step": 4290
    },
    {
      "epoch": 0.554910311007872,
      "grad_norm": 2.5536267450479797,
      "learning_rate": 1.2199885626608373e-06,
      "log_odds_chosen": 1.701171875,
      "log_odds_ratio": -0.3185058534145355,
      "logits/chosen": -1.076171875,
      "logits/rejected": -0.9281250238418579,
      "logps/chosen": -0.614062488079071,
      "logps/rejected": -1.8191406726837158,
      "loss": 0.8159,
      "nll_loss": 0.771777331829071,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06136474758386612,
      "rewards/margins": 0.12060546875,
      "rewards/rejected": -0.18198242783546448,
      "step": 4300
    },
    {
      "epoch": 0.5562008001032391,
      "grad_norm": 2.9907974812986162,
      "learning_rate": 1.2185724408107546e-06,
      "log_odds_chosen": 1.59423828125,
      "log_odds_ratio": -0.3929199278354645,
      "logits/chosen": -1.0851562023162842,
      "logits/rejected": -0.9771484136581421,
      "logps/chosen": -0.682421863079071,
      "logps/rejected": -1.8386719226837158,
      "loss": 0.8127,
      "nll_loss": 0.8182617425918579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06822510063648224,
      "rewards/margins": 0.11567382514476776,
      "rewards/rejected": -0.18386229872703552,
      "step": 4310
    },
    {
      "epoch": 0.5574912891986062,
      "grad_norm": 2.68979547446551,
      "learning_rate": 1.2171612389003689e-06,
      "log_odds_chosen": 1.5945312976837158,
      "log_odds_ratio": -0.3840087950229645,
      "logits/chosen": -1.0617187023162842,
      "logits/rejected": -0.9296875,
      "logps/chosen": -0.687207043170929,
      "logps/rejected": -1.829687476158142,
      "loss": 0.8192,
      "nll_loss": 0.8287109136581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.0687255859375,
      "rewards/margins": 0.11431884765625,
      "rewards/rejected": -0.18303222954273224,
      "step": 4320
    },
    {
      "epoch": 0.5587817782939735,
      "grad_norm": 2.3331277594341255,
      "learning_rate": 1.2157549285071297e-06,
      "log_odds_chosen": 1.5499999523162842,
      "log_odds_ratio": -0.36347657442092896,
      "logits/chosen": -1.084375023841858,
      "logits/rejected": -0.9537109136581421,
      "logps/chosen": -0.630078136920929,
      "logps/rejected": -1.715234398841858,
      "loss": 0.8275,
      "nll_loss": 0.807812511920929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06298828125,
      "rewards/margins": 0.10853882133960724,
      "rewards/rejected": -0.17153319716453552,
      "step": 4330
    },
    {
      "epoch": 0.5600722673893406,
      "grad_norm": 3.0712809645567165,
      "learning_rate": 1.2143534814378327e-06,
      "log_odds_chosen": 1.8313477039337158,
      "log_odds_ratio": -0.348876953125,
      "logits/chosen": -1.0583984851837158,
      "logits/rejected": -0.911914050579071,
      "logps/chosen": -0.6224609613418579,
      "logps/rejected": -1.9148437976837158,
      "loss": 0.8105,
      "nll_loss": 0.7513672113418579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06226806715130806,
      "rewards/margins": 0.12917479872703552,
      "rewards/rejected": -0.1915283203125,
      "step": 4340
    },
    {
      "epoch": 0.5613627564847077,
      "grad_norm": 3.420143891725679,
      "learning_rate": 1.2129568697262454e-06,
      "log_odds_chosen": 1.52734375,
      "log_odds_ratio": -0.3955078125,
      "logits/chosen": -1.0935547351837158,
      "logits/rejected": -0.9527343511581421,
      "logps/chosen": -0.6781250238418579,
      "logps/rejected": -1.7890625,
      "loss": 0.8151,
      "nll_loss": 0.7271484136581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06783447414636612,
      "rewards/margins": 0.11091308295726776,
      "rewards/rejected": -0.1787109375,
      "step": 4350
    },
    {
      "epoch": 0.5626532455800749,
      "grad_norm": 2.4403306773999267,
      "learning_rate": 1.2115650656307653e-06,
      "log_odds_chosen": 1.5642578601837158,
      "log_odds_ratio": -0.36406248807907104,
      "logits/chosen": -1.120507836341858,
      "logits/rejected": -0.943359375,
      "logps/chosen": -0.6029297113418579,
      "logps/rejected": -1.6687500476837158,
      "loss": 0.8116,
      "nll_loss": 0.7587890625,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06027831882238388,
      "rewards/margins": 0.10664062201976776,
      "rewards/rejected": -0.16691894829273224,
      "step": 4360
    },
    {
      "epoch": 0.563943734675442,
      "grad_norm": 2.541357491002039,
      "learning_rate": 1.210178041632103e-06,
      "log_odds_chosen": 1.634667992591858,
      "log_odds_ratio": -0.3770507872104645,
      "logits/chosen": -1.0978515148162842,
      "logits/rejected": -0.976367175579071,
      "logps/chosen": -0.730273425579071,
      "logps/rejected": -1.924218773841858,
      "loss": 0.824,
      "nll_loss": 0.7452148199081421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07308349758386612,
      "rewards/margins": 0.11942748725414276,
      "rewards/rejected": -0.1923828125,
      "step": 4370
    },
    {
      "epoch": 0.5652342237708091,
      "grad_norm": 2.555237812714047,
      "learning_rate": 1.2087957704309988e-06,
      "log_odds_chosen": 1.7000000476837158,
      "log_odds_ratio": -0.35688477754592896,
      "logits/chosen": -1.0994141101837158,
      "logits/rejected": -0.9482421875,
      "logps/chosen": -0.6533203125,
      "logps/rejected": -1.8624999523162842,
      "loss": 0.8244,
      "nll_loss": 0.76806640625,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06533203274011612,
      "rewards/margins": 0.12099609524011612,
      "rewards/rejected": -0.186279296875,
      "step": 4380
    },
    {
      "epoch": 0.5665247128661762,
      "grad_norm": 3.062472795740199,
      "learning_rate": 1.2074182249459642e-06,
      "log_odds_chosen": 1.7843749523162842,
      "log_odds_ratio": -0.37004393339157104,
      "logits/chosen": -1.1134765148162842,
      "logits/rejected": -0.9703124761581421,
      "logps/chosen": -0.671191394329071,
      "logps/rejected": -1.913671851158142,
      "loss": 0.7927,
      "nll_loss": 0.753222644329071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06716308742761612,
      "rewards/margins": 0.12406005710363388,
      "rewards/rejected": -0.19121094048023224,
      "step": 4390
    },
    {
      "epoch": 0.5678152019615434,
      "grad_norm": 2.6674158419062257,
      "learning_rate": 1.2060453783110545e-06,
      "log_odds_chosen": 1.5673828125,
      "log_odds_ratio": -0.37939453125,
      "logits/chosen": -1.064843773841858,
      "logits/rejected": -0.963085949420929,
      "logps/chosen": -0.6728515625,
      "logps/rejected": -1.8019530773162842,
      "loss": 0.8111,
      "nll_loss": 0.811718761920929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06732177734375,
      "rewards/margins": 0.11286620795726776,
      "rewards/rejected": -0.18022461235523224,
      "step": 4400
    },
    {
      "epoch": 0.5691056910569106,
      "grad_norm": 2.781680314718773,
      "learning_rate": 1.2046772038736682e-06,
      "log_odds_chosen": 1.748632788658142,
      "log_odds_ratio": -0.37596434354782104,
      "logits/chosen": -1.0763671398162842,
      "logits/rejected": -0.951953113079071,
      "logps/chosen": -0.656542956829071,
      "logps/rejected": -1.898828148841858,
      "loss": 0.8159,
      "nll_loss": 0.77587890625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06556396186351776,
      "rewards/margins": 0.12434692680835724,
      "rewards/rejected": -0.18991699814796448,
      "step": 4410
    },
    {
      "epoch": 0.5703961801522777,
      "grad_norm": 2.478422881654229,
      "learning_rate": 1.2033136751923736e-06,
      "log_odds_chosen": 1.6130859851837158,
      "log_odds_ratio": -0.36799317598342896,
      "logits/chosen": -1.098046898841858,
      "logits/rejected": -0.998242199420929,
      "logps/chosen": -0.709765613079071,
      "logps/rejected": -1.860937476158142,
      "loss": 0.7885,
      "nll_loss": 0.77099609375,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07094726711511612,
      "rewards/margins": 0.11507568508386612,
      "rewards/rejected": -0.18608398735523224,
      "step": 4420
    },
    {
      "epoch": 0.5716866692476449,
      "grad_norm": 2.37015989295487,
      "learning_rate": 1.201954766034762e-06,
      "log_odds_chosen": 1.6650390625,
      "log_odds_ratio": -0.3645996153354645,
      "logits/chosen": -1.062890648841858,
      "logits/rejected": -0.921093761920929,
      "logps/chosen": -0.6529296636581421,
      "logps/rejected": -1.8250000476837158,
      "loss": 0.7961,
      "nll_loss": 0.7408202886581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06528320163488388,
      "rewards/margins": 0.117156982421875,
      "rewards/rejected": -0.18234863877296448,
      "step": 4430
    },
    {
      "epoch": 0.572977158343012,
      "grad_norm": 2.9239831445939073,
      "learning_rate": 1.2006004503753285e-06,
      "log_odds_chosen": 1.6628906726837158,
      "log_odds_ratio": -0.3294433653354645,
      "logits/chosen": -1.09375,
      "logits/rejected": -0.9556640386581421,
      "logps/chosen": -0.6851562261581421,
      "logps/rejected": -1.873437523841858,
      "loss": 0.8023,
      "nll_loss": 0.785351574420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06851806491613388,
      "rewards/margins": 0.11865234375,
      "rewards/rejected": -0.18720702826976776,
      "step": 4440
    },
    {
      "epoch": 0.5742676474383791,
      "grad_norm": 2.375563007078648,
      "learning_rate": 1.1992507023933782e-06,
      "log_odds_chosen": 1.8977539539337158,
      "log_odds_ratio": -0.31779783964157104,
      "logits/chosen": -1.047265648841858,
      "logits/rejected": -0.9339843988418579,
      "logps/chosen": -0.6534179449081421,
      "logps/rejected": -2.0464844703674316,
      "loss": 0.7947,
      "nll_loss": 0.7865234613418579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06535644829273224,
      "rewards/margins": 0.13934937119483948,
      "rewards/rejected": -0.20468750596046448,
      "step": 4450
    },
    {
      "epoch": 0.5755581365337463,
      "grad_norm": 2.2317243837413674,
      "learning_rate": 1.1979054964709597e-06,
      "log_odds_chosen": 1.3166015148162842,
      "log_odds_ratio": -0.4186035096645355,
      "logits/chosen": -1.008398413658142,
      "logits/rejected": -0.909375011920929,
      "logps/chosen": -0.6908203363418579,
      "logps/rejected": -1.617578148841858,
      "loss": 0.8126,
      "nll_loss": 0.8441406488418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06905517727136612,
      "rewards/margins": 0.09271240234375,
      "rewards/rejected": -0.161865234375,
      "step": 4460
    },
    {
      "epoch": 0.5768486256291134,
      "grad_norm": 3.7191539156806703,
      "learning_rate": 1.1965648071908207e-06,
      "log_odds_chosen": 1.4542968273162842,
      "log_odds_ratio": -0.3818359375,
      "logits/chosen": -1.037695288658142,
      "logits/rejected": -0.919921875,
      "logps/chosen": -0.666308581829071,
      "logps/rejected": -1.646875023841858,
      "loss": 0.8108,
      "nll_loss": 0.7959960699081421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06661377102136612,
      "rewards/margins": 0.09799804538488388,
      "rewards/rejected": -0.16469725966453552,
      "step": 4470
    },
    {
      "epoch": 0.5781391147244805,
      "grad_norm": 2.8503517746973457,
      "learning_rate": 1.1952286093343935e-06,
      "log_odds_chosen": 1.696679711341858,
      "log_odds_ratio": -0.35478514432907104,
      "logits/chosen": -1.046289086341858,
      "logits/rejected": -0.926562488079071,
      "logps/chosen": -0.648632824420929,
      "logps/rejected": -1.841406226158142,
      "loss": 0.8225,
      "nll_loss": 0.7744140625,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06483153998851776,
      "rewards/margins": 0.11924438178539276,
      "rewards/rejected": -0.18422850966453552,
      "step": 4480
    },
    {
      "epoch": 0.5794296038198478,
      "grad_norm": 2.585180427331984,
      "learning_rate": 1.1938968778798005e-06,
      "log_odds_chosen": 1.467626929283142,
      "log_odds_ratio": -0.426025390625,
      "logits/chosen": -1.002539038658142,
      "logits/rejected": -0.9146484136581421,
      "logps/chosen": -0.7562500238418579,
      "logps/rejected": -1.837499976158142,
      "loss": 0.8129,
      "nll_loss": 0.856640636920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07561035454273224,
      "rewards/margins": 0.108184814453125,
      "rewards/rejected": -0.183837890625,
      "step": 4490
    },
    {
      "epoch": 0.5807200929152149,
      "grad_norm": 2.6609971078692523,
      "learning_rate": 1.1925695879998878e-06,
      "log_odds_chosen": 1.96484375,
      "log_odds_ratio": -0.31585693359375,
      "logits/chosen": -1.0439453125,
      "logits/rejected": -0.908984363079071,
      "logps/chosen": -0.613085925579071,
      "logps/rejected": -2.0269532203674316,
      "loss": 0.8115,
      "nll_loss": 0.7330077886581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06136474758386612,
      "rewards/margins": 0.14143677055835724,
      "rewards/rejected": -0.2025146484375,
      "step": 4500
    },
    {
      "epoch": 0.582010582010582,
      "grad_norm": 2.931675679314555,
      "learning_rate": 1.1912467150602794e-06,
      "log_odds_chosen": 1.408593773841858,
      "log_odds_ratio": -0.43525391817092896,
      "logits/chosen": -1.0050780773162842,
      "logits/rejected": -0.9052734375,
      "logps/chosen": -0.67919921875,
      "logps/rejected": -1.667578101158142,
      "loss": 0.8014,
      "nll_loss": 0.736328125,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06788329780101776,
      "rewards/margins": 0.098846435546875,
      "rewards/rejected": -0.16684570908546448,
      "step": 4510
    },
    {
      "epoch": 0.5833010711059492,
      "grad_norm": 3.1176342735076887,
      "learning_rate": 1.189928234617459e-06,
      "log_odds_chosen": 1.2903320789337158,
      "log_odds_ratio": -0.43730467557907104,
      "logits/chosen": -1.077539086341858,
      "logits/rejected": -0.957812488079071,
      "logps/chosen": -0.673828125,
      "logps/rejected": -1.5695312023162842,
      "loss": 0.8029,
      "nll_loss": 0.6988281011581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06733398139476776,
      "rewards/margins": 0.08950195461511612,
      "rewards/rejected": -0.15683594346046448,
      "step": 4520
    },
    {
      "epoch": 0.5845915602013163,
      "grad_norm": 2.8697748950496123,
      "learning_rate": 1.1886141224168716e-06,
      "log_odds_chosen": 1.423437476158142,
      "log_odds_ratio": -0.380615234375,
      "logits/chosen": -1.0398437976837158,
      "logits/rejected": -0.9345703125,
      "logps/chosen": -0.6371093988418579,
      "logps/rejected": -1.6066405773162842,
      "loss": 0.8044,
      "nll_loss": 0.716601550579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06374511867761612,
      "rewards/margins": 0.09685058891773224,
      "rewards/rejected": -0.16064453125,
      "step": 4530
    },
    {
      "epoch": 0.5858820492966834,
      "grad_norm": 2.563017515620488,
      "learning_rate": 1.1873043543910495e-06,
      "log_odds_chosen": 2.0609374046325684,
      "log_odds_ratio": -0.2711425721645355,
      "logits/chosen": -1.055273413658142,
      "logits/rejected": -0.8707031011581421,
      "logps/chosen": -0.6650390625,
      "logps/rejected": -2.1664061546325684,
      "loss": 0.8136,
      "nll_loss": 0.8121093511581421,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06650390475988388,
      "rewards/margins": 0.15021972358226776,
      "rewards/rejected": -0.21669921278953552,
      "step": 4540
    },
    {
      "epoch": 0.5871725383920506,
      "grad_norm": 2.7549349105182146,
      "learning_rate": 1.1859989066577617e-06,
      "log_odds_chosen": 1.484960913658142,
      "log_odds_ratio": -0.4012695252895355,
      "logits/chosen": -1.0703125,
      "logits/rejected": -0.9458984136581421,
      "logps/chosen": -0.740234375,
      "logps/rejected": -1.785546898841858,
      "loss": 0.8076,
      "nll_loss": 0.7388671636581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07403564453125,
      "rewards/margins": 0.1043701171875,
      "rewards/rejected": -0.1783447265625,
      "step": 4550
    },
    {
      "epoch": 0.5884630274874177,
      "grad_norm": 2.7308110606952307,
      "learning_rate": 1.1846977555181846e-06,
      "log_odds_chosen": 1.36328125,
      "log_odds_ratio": -0.44697266817092896,
      "logits/chosen": -1.0478515625,
      "logits/rejected": -0.9273437261581421,
      "logps/chosen": -0.769726574420929,
      "logps/rejected": -1.758203148841858,
      "loss": 0.7692,
      "nll_loss": 0.794628918170929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07700195163488388,
      "rewards/margins": 0.09886474907398224,
      "rewards/rejected": -0.17578125,
      "step": 4560
    },
    {
      "epoch": 0.5897535165827849,
      "grad_norm": 2.998268667010775,
      "learning_rate": 1.1834008774550946e-06,
      "log_odds_chosen": 1.7587890625,
      "log_odds_ratio": -0.308837890625,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.9437500238418579,
      "logps/chosen": -0.590624988079071,
      "logps/rejected": -1.7804687023162842,
      "loss": 0.8052,
      "nll_loss": 0.723437488079071,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.05904541164636612,
      "rewards/margins": 0.11892089992761612,
      "rewards/rejected": -0.17824706435203552,
      "step": 4570
    },
    {
      "epoch": 0.5910440056781521,
      "grad_norm": 3.3977858664782508,
      "learning_rate": 1.1821082491310835e-06,
      "log_odds_chosen": 1.5441405773162842,
      "log_odds_ratio": -0.402587890625,
      "logits/chosen": -1.065039038658142,
      "logits/rejected": -0.968945324420929,
      "logps/chosen": -0.688281238079071,
      "logps/rejected": -1.736718773841858,
      "loss": 0.7907,
      "nll_loss": 0.7083984613418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06879882514476776,
      "rewards/margins": 0.10482177883386612,
      "rewards/rejected": -0.17368164658546448,
      "step": 4580
    },
    {
      "epoch": 0.5923344947735192,
      "grad_norm": 2.5423121222842027,
      "learning_rate": 1.1808198473867937e-06,
      "log_odds_chosen": 1.6710937023162842,
      "log_odds_ratio": -0.348388671875,
      "logits/chosen": -1.0613281726837158,
      "logits/rejected": -0.913867175579071,
      "logps/chosen": -0.6923828125,
      "logps/rejected": -1.8791015148162842,
      "loss": 0.7884,
      "nll_loss": 0.821093738079071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06923828274011612,
      "rewards/margins": 0.11862792819738388,
      "rewards/rejected": -0.18776854872703552,
      "step": 4590
    },
    {
      "epoch": 0.5936249838688863,
      "grad_norm": 2.8453736572409505,
      "learning_rate": 1.179535649239177e-06,
      "log_odds_chosen": 1.3488280773162842,
      "log_odds_ratio": -0.42485350370407104,
      "logits/chosen": -1.058984398841858,
      "logits/rejected": -0.947460949420929,
      "logps/chosen": -0.639843761920929,
      "logps/rejected": -1.502343773841858,
      "loss": 0.8133,
      "nll_loss": 0.775390625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06403808295726776,
      "rewards/margins": 0.08615722507238388,
      "rewards/rejected": -0.15017089247703552,
      "step": 4600
    },
    {
      "epoch": 0.5949154729642534,
      "grad_norm": 2.84924883381114,
      "learning_rate": 1.178255631879771e-06,
      "log_odds_chosen": 1.649023413658142,
      "log_odds_ratio": -0.34965819120407104,
      "logits/chosen": -1.067968726158142,
      "logits/rejected": -0.9408203363418579,
      "logps/chosen": -0.7030273675918579,
      "logps/rejected": -1.8761718273162842,
      "loss": 0.7709,
      "nll_loss": 0.74560546875,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07022704929113388,
      "rewards/margins": 0.11745605617761612,
      "rewards/rejected": -0.18771973252296448,
      "step": 4610
    },
    {
      "epoch": 0.5962059620596206,
      "grad_norm": 2.778480911509227,
      "learning_rate": 1.1769797726729992e-06,
      "log_odds_chosen": 1.4357421398162842,
      "log_odds_ratio": -0.3775390684604645,
      "logits/chosen": -1.101953148841858,
      "logits/rejected": -0.943359375,
      "logps/chosen": -0.667675793170929,
      "logps/rejected": -1.6640625,
      "loss": 0.8047,
      "nll_loss": 0.733691394329071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06673584133386612,
      "rewards/margins": 0.09979858249425888,
      "rewards/rejected": -0.16647949814796448,
      "step": 4620
    },
    {
      "epoch": 0.5974964511549877,
      "grad_norm": 2.603887585552828,
      "learning_rate": 1.1757080491544881e-06,
      "log_odds_chosen": 1.495849609375,
      "log_odds_ratio": -0.41694337129592896,
      "logits/chosen": -1.115234375,
      "logits/rejected": -0.9322265386581421,
      "logps/chosen": -0.711132824420929,
      "logps/rejected": -1.8054687976837158,
      "loss": 0.7904,
      "nll_loss": 0.750683605670929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07111816108226776,
      "rewards/margins": 0.10943603515625,
      "rewards/rejected": -0.18059082329273224,
      "step": 4630
    },
    {
      "epoch": 0.5987869402503548,
      "grad_norm": 2.7101653088673996,
      "learning_rate": 1.1744404390294068e-06,
      "log_odds_chosen": 1.3629882335662842,
      "log_odds_ratio": -0.439697265625,
      "logits/chosen": -1.051171898841858,
      "logits/rejected": -0.942578136920929,
      "logps/chosen": -0.720996081829071,
      "logps/rejected": -1.6513671875,
      "loss": 0.7894,
      "nll_loss": 0.75244140625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07211913913488388,
      "rewards/margins": 0.09307861328125,
      "rewards/rejected": -0.16520996391773224,
      "step": 4640
    },
    {
      "epoch": 0.6000774293457221,
      "grad_norm": 3.027946585381608,
      "learning_rate": 1.1731769201708264e-06,
      "log_odds_chosen": 1.595312476158142,
      "log_odds_ratio": -0.3604492247104645,
      "logits/chosen": -1.097070336341858,
      "logits/rejected": -0.980273425579071,
      "logps/chosen": -0.7100585699081421,
      "logps/rejected": -1.85546875,
      "loss": 0.7839,
      "nll_loss": 0.723339855670929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.0709228515625,
      "rewards/margins": 0.11455078423023224,
      "rewards/rejected": -0.18547363579273224,
      "step": 4650
    },
    {
      "epoch": 0.6013679184410892,
      "grad_norm": 2.624646140842192,
      "learning_rate": 1.1719174706180952e-06,
      "log_odds_chosen": 1.586572289466858,
      "log_odds_ratio": -0.3775390684604645,
      "logits/chosen": -1.1005859375,
      "logits/rejected": -0.9521484375,
      "logps/chosen": -0.667773425579071,
      "logps/rejected": -1.755468726158142,
      "loss": 0.8038,
      "nll_loss": 0.6923828125,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06683349609375,
      "rewards/margins": 0.10871734470129013,
      "rewards/rejected": -0.17558594048023224,
      "step": 4660
    },
    {
      "epoch": 0.6026584075364563,
      "grad_norm": 2.473588141923909,
      "learning_rate": 1.1706620685752386e-06,
      "log_odds_chosen": 1.6318359375,
      "log_odds_ratio": -0.35297852754592896,
      "logits/chosen": -1.1328125,
      "logits/rejected": -0.9652343988418579,
      "logps/chosen": -0.66796875,
      "logps/rejected": -1.861328125,
      "loss": 0.789,
      "nll_loss": 0.7818359136581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06674804538488388,
      "rewards/margins": 0.11918945610523224,
      "rewards/rejected": -0.18623046576976776,
      "step": 4670
    },
    {
      "epoch": 0.6039488966318235,
      "grad_norm": 2.9213573094851233,
      "learning_rate": 1.1694106924093723e-06,
      "log_odds_chosen": 1.5164062976837158,
      "log_odds_ratio": -0.40351563692092896,
      "logits/chosen": -1.1160156726837158,
      "logits/rejected": -0.9849609136581421,
      "logps/chosen": -0.704882800579071,
      "logps/rejected": -1.842187523841858,
      "loss": 0.7776,
      "nll_loss": 0.740039050579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07044677436351776,
      "rewards/margins": 0.11375732719898224,
      "rewards/rejected": -0.18430176377296448,
      "step": 4680
    },
    {
      "epoch": 0.6052393857271906,
      "grad_norm": 2.6745440469844457,
      "learning_rate": 1.1681633206491381e-06,
      "log_odds_chosen": 1.5261719226837158,
      "log_odds_ratio": -0.3719726502895355,
      "logits/chosen": -1.05078125,
      "logits/rejected": -0.949999988079071,
      "logps/chosen": -0.6439453363418579,
      "logps/rejected": -1.728906273841858,
      "loss": 0.7956,
      "nll_loss": 0.8287109136581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06436767429113388,
      "rewards/margins": 0.10837402194738388,
      "rewards/rejected": -0.17277832329273224,
      "step": 4690
    },
    {
      "epoch": 0.6065298748225577,
      "grad_norm": 3.088620320675354,
      "learning_rate": 1.1669199319831564e-06,
      "log_odds_chosen": 1.6409180164337158,
      "log_odds_ratio": -0.35942381620407104,
      "logits/chosen": -1.034765601158142,
      "logits/rejected": -0.881054699420929,
      "logps/chosen": -0.695996105670929,
      "logps/rejected": -1.845703125,
      "loss": 0.7829,
      "nll_loss": 0.7865234613418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06960449367761612,
      "rewards/margins": 0.11484527587890625,
      "rewards/rejected": -0.18433837592601776,
      "step": 4700
    },
    {
      "epoch": 0.6078203639179249,
      "grad_norm": 2.9099802787061257,
      "learning_rate": 1.1656805052584958e-06,
      "log_odds_chosen": 1.651098608970642,
      "log_odds_ratio": -0.3760742247104645,
      "logits/chosen": -1.033593773841858,
      "logits/rejected": -0.9208984375,
      "logps/chosen": -0.671875,
      "logps/rejected": -1.840234398841858,
      "loss": 0.7932,
      "nll_loss": 0.799609363079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06721191108226776,
      "rewards/margins": 0.11670837551355362,
      "rewards/rejected": -0.1839599609375,
      "step": 4710
    },
    {
      "epoch": 0.609110853013292,
      "grad_norm": 2.8188320454083438,
      "learning_rate": 1.164445019479164e-06,
      "log_odds_chosen": 1.658593773841858,
      "log_odds_ratio": -0.31962889432907104,
      "logits/chosen": -1.0324218273162842,
      "logits/rejected": -0.911914050579071,
      "logps/chosen": -0.68896484375,
      "logps/rejected": -1.818945288658142,
      "loss": 0.785,
      "nll_loss": 0.7911132574081421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06894531100988388,
      "rewards/margins": 0.11296387016773224,
      "rewards/rejected": -0.18186035752296448,
      "step": 4720
    },
    {
      "epoch": 0.6104013421086592,
      "grad_norm": 2.769084171678557,
      "learning_rate": 1.1632134538046105e-06,
      "log_odds_chosen": 1.6095702648162842,
      "log_odds_ratio": -0.38066405057907104,
      "logits/chosen": -1.077734351158142,
      "logits/rejected": -0.9476562738418579,
      "logps/chosen": -0.6900390386581421,
      "logps/rejected": -1.862890601158142,
      "loss": 0.7881,
      "nll_loss": 0.772265613079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06901855766773224,
      "rewards/margins": 0.11738280951976776,
      "rewards/rejected": -0.18649902939796448,
      "step": 4730
    },
    {
      "epoch": 0.6116918312040264,
      "grad_norm": 2.618826587783404,
      "learning_rate": 1.1619857875482536e-06,
      "log_odds_chosen": 1.522558569908142,
      "log_odds_ratio": -0.3857421875,
      "logits/chosen": -1.0388672351837158,
      "logits/rejected": -0.9517577886581421,
      "logps/chosen": -0.6416015625,
      "logps/rejected": -1.680273413658142,
      "loss": 0.7752,
      "nll_loss": 0.709667980670929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06417236477136612,
      "rewards/margins": 0.10374756157398224,
      "rewards/rejected": -0.16789551079273224,
      "step": 4740
    },
    {
      "epoch": 0.6129823202993935,
      "grad_norm": 2.923965708778238,
      "learning_rate": 1.1607620001760185e-06,
      "log_odds_chosen": 1.446874976158142,
      "log_odds_ratio": -0.3583984375,
      "logits/chosen": -1.058984398841858,
      "logits/rejected": -0.921679675579071,
      "logps/chosen": -0.640820324420929,
      "logps/rejected": -1.616796851158142,
      "loss": 0.8022,
      "nll_loss": 0.78076171875,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.0640869140625,
      "rewards/margins": 0.09755859524011612,
      "rewards/rejected": -0.16169433295726776,
      "step": 4750
    },
    {
      "epoch": 0.6142728093947606,
      "grad_norm": 2.9066327989771725,
      "learning_rate": 1.1595420713048968e-06,
      "log_odds_chosen": 1.440820336341858,
      "log_odds_ratio": -0.36577147245407104,
      "logits/chosen": -1.090429663658142,
      "logits/rejected": -0.9232422113418579,
      "logps/chosen": -0.7119140625,
      "logps/rejected": -1.7566406726837158,
      "loss": 0.7912,
      "nll_loss": 0.7835937738418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07120361179113388,
      "rewards/margins": 0.10449828952550888,
      "rewards/rejected": -0.17573241889476776,
      "step": 4760
    },
    {
      "epoch": 0.6155632984901278,
      "grad_norm": 2.6052395508547668,
      "learning_rate": 1.1583259807015182e-06,
      "log_odds_chosen": 1.665686011314392,
      "log_odds_ratio": -0.3516174256801605,
      "logits/chosen": -1.058007836341858,
      "logits/rejected": -0.949414074420929,
      "logps/chosen": -0.649609386920929,
      "logps/rejected": -1.8185546398162842,
      "loss": 0.7802,
      "nll_loss": 0.728710949420929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06499023735523224,
      "rewards/margins": 0.11692504584789276,
      "rewards/rejected": -0.18190917372703552,
      "step": 4770
    },
    {
      "epoch": 0.6168537875854949,
      "grad_norm": 3.2417036336670075,
      "learning_rate": 1.1571137082807434e-06,
      "log_odds_chosen": 1.639257788658142,
      "log_odds_ratio": -0.3416503965854645,
      "logits/chosen": -1.0529296398162842,
      "logits/rejected": -0.9107421636581421,
      "logps/chosen": -0.65478515625,
      "logps/rejected": -1.787695288658142,
      "loss": 0.7818,
      "nll_loss": 0.758007824420929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06549072265625,
      "rewards/margins": 0.11316528171300888,
      "rewards/rejected": -0.17868652939796448,
      "step": 4780
    },
    {
      "epoch": 0.618144276680862,
      "grad_norm": 3.168159776438873,
      "learning_rate": 1.155905234104269e-06,
      "log_odds_chosen": 1.686914086341858,
      "log_odds_ratio": -0.3873046934604645,
      "logits/chosen": -1.080468773841858,
      "logits/rejected": -0.9564453363418579,
      "logps/chosen": -0.6709960699081421,
      "logps/rejected": -1.8953125476837158,
      "loss": 0.7949,
      "nll_loss": 0.744140625,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06710205227136612,
      "rewards/margins": 0.12241210788488388,
      "rewards/rejected": -0.18947753310203552,
      "step": 4790
    },
    {
      "epoch": 0.6194347657762292,
      "grad_norm": 2.77999880854307,
      "learning_rate": 1.1547005383792516e-06,
      "log_odds_chosen": 1.5480468273162842,
      "log_odds_ratio": -0.35786134004592896,
      "logits/chosen": -1.0378906726837158,
      "logits/rejected": -0.9208984375,
      "logps/chosen": -0.645312488079071,
      "logps/rejected": -1.7062499523162842,
      "loss": 0.7885,
      "nll_loss": 0.7734375,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06456299126148224,
      "rewards/margins": 0.10602416843175888,
      "rewards/rejected": -0.17048339545726776,
      "step": 4800
    },
    {
      "epoch": 0.6207252548715964,
      "grad_norm": 3.7390493659567228,
      "learning_rate": 1.1534996014569446e-06,
      "log_odds_chosen": 1.58837890625,
      "log_odds_ratio": -0.394775390625,
      "logits/chosen": -1.099218726158142,
      "logits/rejected": -0.9632812738418579,
      "logps/chosen": -0.7021484375,
      "logps/rejected": -1.810937523841858,
      "loss": 0.7825,
      "nll_loss": 0.759570300579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07021484524011612,
      "rewards/margins": 0.11072997748851776,
      "rewards/rejected": -0.18110351264476776,
      "step": 4810
    },
    {
      "epoch": 0.6220157439669635,
      "grad_norm": 2.6458758104660998,
      "learning_rate": 1.1523024038313547e-06,
      "log_odds_chosen": 1.397363305091858,
      "log_odds_ratio": -0.4107421934604645,
      "logits/chosen": -1.0478515625,
      "logits/rejected": -0.9164062738418579,
      "logps/chosen": -0.7001953125,
      "logps/rejected": -1.6628906726837158,
      "loss": 0.8098,
      "nll_loss": 0.855273425579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07001952826976776,
      "rewards/margins": 0.09619140625,
      "rewards/rejected": -0.166259765625,
      "step": 4820
    },
    {
      "epoch": 0.6233062330623306,
      "grad_norm": 2.6337348872576096,
      "learning_rate": 1.1511089261379083e-06,
      "log_odds_chosen": 1.6943359375,
      "log_odds_ratio": -0.35693359375,
      "logits/chosen": -1.046289086341858,
      "logits/rejected": -0.905078113079071,
      "logps/chosen": -0.718457043170929,
      "logps/rejected": -1.899023413658142,
      "loss": 0.7852,
      "nll_loss": 0.7708984613418579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07186279445886612,
      "rewards/margins": 0.11810302734375,
      "rewards/rejected": -0.18994140625,
      "step": 4830
    },
    {
      "epoch": 0.6245967221576978,
      "grad_norm": 2.8172275490589285,
      "learning_rate": 1.149919149152138e-06,
      "log_odds_chosen": 1.3449218273162842,
      "log_odds_ratio": -0.39677733182907104,
      "logits/chosen": -1.0363280773162842,
      "logits/rejected": -0.9244140386581421,
      "logps/chosen": -0.65625,
      "logps/rejected": -1.5802733898162842,
      "loss": 0.7838,
      "nll_loss": 0.7412109375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06563720852136612,
      "rewards/margins": 0.0924072265625,
      "rewards/rejected": -0.1580810546875,
      "step": 4840
    },
    {
      "epoch": 0.6258872112530649,
      "grad_norm": 3.174370005501859,
      "learning_rate": 1.148733053788381e-06,
      "log_odds_chosen": 1.5949218273162842,
      "log_odds_ratio": -0.3863281309604645,
      "logits/chosen": -1.063085913658142,
      "logits/rejected": -0.953906238079071,
      "logps/chosen": -0.6045898199081421,
      "logps/rejected": -1.6921875476837158,
      "loss": 0.7702,
      "nll_loss": 0.682421863079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06049804762005806,
      "rewards/margins": 0.10883788764476776,
      "rewards/rejected": -0.16943359375,
      "step": 4850
    },
    {
      "epoch": 0.627177700348432,
      "grad_norm": 2.6536963295161544,
      "learning_rate": 1.1475506210984938e-06,
      "log_odds_chosen": 1.5634765625,
      "log_odds_ratio": -0.3743652403354645,
      "logits/chosen": -1.048828125,
      "logits/rejected": -0.9408203363418579,
      "logps/chosen": -0.6724609136581421,
      "logps/rejected": -1.7501952648162842,
      "loss": 0.8019,
      "nll_loss": 0.7201172113418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.0672607421875,
      "rewards/margins": 0.10780028998851776,
      "rewards/rejected": -0.17507323622703552,
      "step": 4860
    },
    {
      "epoch": 0.6284681894437992,
      "grad_norm": 3.4595481857433628,
      "learning_rate": 1.1463718322705807e-06,
      "log_odds_chosen": 1.6159179210662842,
      "log_odds_ratio": -0.3487792909145355,
      "logits/chosen": -1.0574219226837158,
      "logits/rejected": -0.9312499761581421,
      "logps/chosen": -0.631054699420929,
      "logps/rejected": -1.711328148841858,
      "loss": 0.7894,
      "nll_loss": 0.7562500238418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06308593600988388,
      "rewards/margins": 0.108123779296875,
      "rewards/rejected": -0.17104491591453552,
      "step": 4870
    },
    {
      "epoch": 0.6297586785391663,
      "grad_norm": 2.8249700166237455,
      "learning_rate": 1.1451966686277364e-06,
      "log_odds_chosen": 1.859960913658142,
      "log_odds_ratio": -0.3294433653354645,
      "logits/chosen": -1.0466797351837158,
      "logits/rejected": -0.938281238079071,
      "logps/chosen": -0.647656261920929,
      "logps/rejected": -1.958593726158142,
      "loss": 0.7611,
      "nll_loss": 0.736621081829071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.0648193359375,
      "rewards/margins": 0.13112792372703552,
      "rewards/rejected": -0.19584961235523224,
      "step": 4880
    },
    {
      "epoch": 0.6310491676345334,
      "grad_norm": 3.339443745317083,
      "learning_rate": 1.1440251116268034e-06,
      "log_odds_chosen": 1.5737793445587158,
      "log_odds_ratio": -0.4090332090854645,
      "logits/chosen": -1.0595703125,
      "logits/rejected": -0.938671886920929,
      "logps/chosen": -0.6519531011581421,
      "logps/rejected": -1.7412109375,
      "loss": 0.768,
      "nll_loss": 0.7064453363418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06517334282398224,
      "rewards/margins": 0.10889282077550888,
      "rewards/rejected": -0.17404785752296448,
      "step": 4890
    },
    {
      "epoch": 0.6323396567299007,
      "grad_norm": 2.862161607019979,
      "learning_rate": 1.1428571428571428e-06,
      "log_odds_chosen": 1.658203125,
      "log_odds_ratio": -0.36474609375,
      "logits/chosen": -1.0822265148162842,
      "logits/rejected": -0.9521484375,
      "logps/chosen": -0.660937488079071,
      "logps/rejected": -1.87109375,
      "loss": 0.7737,
      "nll_loss": 0.7474609613418579,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06613769382238388,
      "rewards/margins": 0.12086181342601776,
      "rewards/rejected": -0.18691405653953552,
      "step": 4900
    },
    {
      "epoch": 0.6336301458252678,
      "grad_norm": 2.813618329536153,
      "learning_rate": 1.14169274403942e-06,
      "log_odds_chosen": 1.607421875,
      "log_odds_ratio": -0.34028321504592896,
      "logits/chosen": -1.0568358898162842,
      "logits/rejected": -0.9365234375,
      "logps/chosen": -0.625195324420929,
      "logps/rejected": -1.7332031726837158,
      "loss": 0.7638,
      "nll_loss": 0.689648449420929,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06252441555261612,
      "rewards/margins": 0.11065673828125,
      "rewards/rejected": -0.1732177734375,
      "step": 4910
    },
    {
      "epoch": 0.6349206349206349,
      "grad_norm": 2.9747268625817087,
      "learning_rate": 1.140531897024402e-06,
      "log_odds_chosen": 1.707421898841858,
      "log_odds_ratio": -0.36066895723342896,
      "logits/chosen": -1.047460913658142,
      "logits/rejected": -0.947070300579071,
      "logps/chosen": -0.7005859613418579,
      "logps/rejected": -1.9249999523162842,
      "loss": 0.7974,
      "nll_loss": 0.7451171875,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07005615532398224,
      "rewards/margins": 0.12235717475414276,
      "rewards/rejected": -0.19233398139476776,
      "step": 4920
    },
    {
      "epoch": 0.6362111240160021,
      "grad_norm": 2.7411912981653863,
      "learning_rate": 1.13937458379177e-06,
      "log_odds_chosen": 1.506250023841858,
      "log_odds_ratio": -0.38188475370407104,
      "logits/chosen": -1.0478515625,
      "logits/rejected": -0.927734375,
      "logps/chosen": -0.6703125238418579,
      "logps/rejected": -1.6873047351837158,
      "loss": 0.7894,
      "nll_loss": 0.7330077886581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06699218600988388,
      "rewards/margins": 0.10186767578125,
      "rewards/rejected": -0.16887207329273224,
      "step": 4930
    },
    {
      "epoch": 0.6375016131113692,
      "grad_norm": 3.159163712528672,
      "learning_rate": 1.1382207864489444e-06,
      "log_odds_chosen": 1.7470703125,
      "log_odds_ratio": -0.31645506620407104,
      "logits/chosen": -1.0666015148162842,
      "logits/rejected": -0.9281250238418579,
      "logps/chosen": -0.6659179925918579,
      "logps/rejected": -1.9148437976837158,
      "loss": 0.7669,
      "nll_loss": 0.7080078125,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.06658935546875,
      "rewards/margins": 0.12468262016773224,
      "rewards/rejected": -0.19118651747703552,
      "step": 4940
    },
    {
      "epoch": 0.6387921022067363,
      "grad_norm": 2.9443713415533215,
      "learning_rate": 1.1370704872299223e-06,
      "log_odds_chosen": 1.572656273841858,
      "log_odds_ratio": -0.35600584745407104,
      "logits/chosen": -1.081445336341858,
      "logits/rejected": -0.954296886920929,
      "logps/chosen": -0.6904296875,
      "logps/rejected": -1.800390601158142,
      "loss": 0.7673,
      "nll_loss": 0.701464831829071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06907959282398224,
      "rewards/margins": 0.11091308295726776,
      "rewards/rejected": -0.18000487983226776,
      "step": 4950
    },
    {
      "epoch": 0.6400825913021035,
      "grad_norm": 2.61708933698978,
      "learning_rate": 1.1359236684941295e-06,
      "log_odds_chosen": 1.934179663658142,
      "log_odds_ratio": -0.3379882872104645,
      "logits/chosen": -1.0828125476837158,
      "logits/rejected": -0.949999988079071,
      "logps/chosen": -0.6802734136581421,
      "logps/rejected": -2.1175780296325684,
      "loss": 0.7816,
      "nll_loss": 0.7237304449081421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06806640326976776,
      "rewards/margins": 0.14395752549171448,
      "rewards/rejected": -0.21198730170726776,
      "step": 4960
    },
    {
      "epoch": 0.6413730803974707,
      "grad_norm": 2.9332457520925215,
      "learning_rate": 1.1347803127252839e-06,
      "log_odds_chosen": 1.4188964366912842,
      "log_odds_ratio": -0.4136718809604645,
      "logits/chosen": -1.052148461341858,
      "logits/rejected": -0.912890613079071,
      "logps/chosen": -0.680468738079071,
      "logps/rejected": -1.6689453125,
      "loss": 0.773,
      "nll_loss": 0.7642577886581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06807861477136612,
      "rewards/margins": 0.09895477443933487,
      "rewards/rejected": -0.1668701171875,
      "step": 4970
    },
    {
      "epoch": 0.6426635694928378,
      "grad_norm": 2.820687045244134,
      "learning_rate": 1.1336404025302715e-06,
      "log_odds_chosen": 1.697167992591858,
      "log_odds_ratio": -0.380859375,
      "logits/chosen": -1.0623047351837158,
      "logits/rejected": -0.930859386920929,
      "logps/chosen": -0.6659179925918579,
      "logps/rejected": -1.859765648841858,
      "loss": 0.7504,
      "nll_loss": 0.740234375,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06663818657398224,
      "rewards/margins": 0.11922607570886612,
      "rewards/rejected": -0.185791015625,
      "step": 4980
    },
    {
      "epoch": 0.643954058588205,
      "grad_norm": 2.4466912415668576,
      "learning_rate": 1.1325039206380352e-06,
      "log_odds_chosen": 1.5244140625,
      "log_odds_ratio": -0.411865234375,
      "logits/chosen": -1.112695336341858,
      "logits/rejected": -0.982421875,
      "logps/chosen": -0.7085937261581421,
      "logps/rejected": -1.807031273841858,
      "loss": 0.8118,
      "nll_loss": 0.773242175579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07086181640625,
      "rewards/margins": 0.10972900688648224,
      "rewards/rejected": -0.18068847060203552,
      "step": 4990
    },
    {
      "epoch": 0.6452445476835721,
      "grad_norm": 2.658787259667725,
      "learning_rate": 1.131370849898476e-06,
      "log_odds_chosen": 1.668359398841858,
      "log_odds_ratio": -0.34619140625,
      "logits/chosen": -1.088281273841858,
      "logits/rejected": -0.9173828363418579,
      "logps/chosen": -0.7139648199081421,
      "logps/rejected": -1.9386718273162842,
      "loss": 0.7741,
      "nll_loss": 0.8089843988418579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.07135009765625,
      "rewards/margins": 0.12263183295726776,
      "rewards/rejected": -0.19384765625,
      "step": 5000
    },
    {
      "epoch": 0.6452445476835721,
      "eval_log_odds_chosen": 1.1819437742233276,
      "eval_log_odds_ratio": -0.47540709376335144,
      "eval_logits/chosen": -0.8723087906837463,
      "eval_logits/rejected": -0.7670671343803406,
      "eval_logps/chosen": -0.7900527119636536,
      "eval_logps/rejected": -1.678329348564148,
      "eval_loss": 1.2826783657073975,
      "eval_nll_loss": 1.2283066511154175,
      "eval_rewards/accuracies": 0.7319767475128174,
      "eval_rewards/chosen": -0.07900191843509674,
      "eval_rewards/margins": 0.08882816880941391,
      "eval_rewards/rejected": -0.16783475875854492,
      "eval_runtime": 722.4069,
      "eval_samples_per_second": 76.181,
      "eval_steps_per_second": 1.19,
      "step": 5000
    },
    {
      "epoch": 0.6465350367789392,
      "grad_norm": 2.96817850794281,
      "learning_rate": 1.130241173281366e-06,
      "log_odds_chosen": 1.589453101158142,
      "log_odds_ratio": -0.3560546934604645,
      "logits/chosen": -1.0734374523162842,
      "logits/rejected": -0.9371093511581421,
      "logps/chosen": -0.6514648199081421,
      "logps/rejected": -1.736328125,
      "loss": 0.7621,
      "nll_loss": 0.76513671875,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06513671576976776,
      "rewards/margins": 0.10853271186351776,
      "rewards/rejected": -0.17368164658546448,
      "step": 5010
    },
    {
      "epoch": 0.6478255258743063,
      "grad_norm": 2.811919167288791,
      "learning_rate": 1.1291148738752732e-06,
      "log_odds_chosen": 1.713623046875,
      "log_odds_ratio": -0.3540283143520355,
      "logits/chosen": -1.0587890148162842,
      "logits/rejected": -0.944140613079071,
      "logps/chosen": -0.649218738079071,
      "logps/rejected": -1.8271484375,
      "loss": 0.7562,
      "nll_loss": 0.683789074420929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06490478664636612,
      "rewards/margins": 0.11788024753332138,
      "rewards/rejected": -0.18295899033546448,
      "step": 5020
    },
    {
      "epoch": 0.6491160149696735,
      "grad_norm": 2.85188980331095,
      "learning_rate": 1.1279919348864981e-06,
      "log_odds_chosen": 1.5341796875,
      "log_odds_ratio": -0.37871092557907104,
      "logits/chosen": -1.0966796875,
      "logits/rejected": -0.9701172113418579,
      "logps/chosen": -0.702343761920929,
      "logps/rejected": -1.787500023841858,
      "loss": 0.763,
      "nll_loss": 0.7529296875,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07022704929113388,
      "rewards/margins": 0.10855712741613388,
      "rewards/rejected": -0.17875976860523224,
      "step": 5030
    },
    {
      "epoch": 0.6504065040650406,
      "grad_norm": 2.7402965352841138,
      "learning_rate": 1.126872339638022e-06,
      "log_odds_chosen": 1.6334960460662842,
      "log_odds_ratio": -0.3376708924770355,
      "logits/chosen": -1.0656249523162842,
      "logits/rejected": -0.958984375,
      "logps/chosen": -0.6405273675918579,
      "logps/rejected": -1.763281226158142,
      "loss": 0.7758,
      "nll_loss": 0.68994140625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06407471001148224,
      "rewards/margins": 0.112213134765625,
      "rewards/rejected": -0.17634277045726776,
      "step": 5040
    },
    {
      "epoch": 0.6516969931604077,
      "grad_norm": 3.1992683704457607,
      "learning_rate": 1.1257560715684668e-06,
      "log_odds_chosen": 1.767578125,
      "log_odds_ratio": -0.3187011778354645,
      "logits/chosen": -1.056249976158142,
      "logits/rejected": -0.9105468988418579,
      "logps/chosen": -0.6646484136581421,
      "logps/rejected": -1.912109375,
      "loss": 0.7665,
      "nll_loss": 0.7427734136581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06649170070886612,
      "rewards/margins": 0.12473144382238388,
      "rewards/rejected": -0.1912841796875,
      "step": 5050
    },
    {
      "epoch": 0.652987482255775,
      "grad_norm": 2.801955167694493,
      "learning_rate": 1.1246431142310665e-06,
      "log_odds_chosen": 1.8708984851837158,
      "log_odds_ratio": -0.3615966737270355,
      "logits/chosen": -1.063085913658142,
      "logits/rejected": -0.9644531011581421,
      "logps/chosen": -0.670117199420929,
      "logps/rejected": -2.0367188453674316,
      "loss": 0.7643,
      "nll_loss": 0.7386718988418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06700439751148224,
      "rewards/margins": 0.13680419325828552,
      "rewards/rejected": -0.20380859076976776,
      "step": 5060
    },
    {
      "epoch": 0.6542779713511421,
      "grad_norm": 2.649685214893808,
      "learning_rate": 1.1235334512926484e-06,
      "log_odds_chosen": 1.759374976158142,
      "log_odds_ratio": -0.36787110567092896,
      "logits/chosen": -1.082617163658142,
      "logits/rejected": -0.9332031011581421,
      "logps/chosen": -0.6333984136581421,
      "logps/rejected": -1.896484375,
      "loss": 0.7732,
      "nll_loss": 0.72705078125,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06334228813648224,
      "rewards/margins": 0.12631836533546448,
      "rewards/rejected": -0.18964843451976776,
      "step": 5070
    },
    {
      "epoch": 0.6555684604465092,
      "grad_norm": 2.917944180970189,
      "learning_rate": 1.1224270665326274e-06,
      "log_odds_chosen": 1.631445288658142,
      "log_odds_ratio": -0.37836915254592896,
      "logits/chosen": -1.0955078601837158,
      "logits/rejected": -0.9564453363418579,
      "logps/chosen": -0.705273449420929,
      "logps/rejected": -1.8781249523162842,
      "loss": 0.7448,
      "nll_loss": 0.7412109375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07060547173023224,
      "rewards/margins": 0.11728515475988388,
      "rewards/rejected": -0.18779297173023224,
      "step": 5080
    },
    {
      "epoch": 0.6568589495418764,
      "grad_norm": 2.563754068314291,
      "learning_rate": 1.12132394384201e-06,
      "log_odds_chosen": 1.6876952648162842,
      "log_odds_ratio": -0.37202149629592896,
      "logits/chosen": -1.0830078125,
      "logits/rejected": -0.960742175579071,
      "logps/chosen": -0.655468761920929,
      "logps/rejected": -1.8796875476837158,
      "loss": 0.7574,
      "nll_loss": 0.7455078363418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06553955376148224,
      "rewards/margins": 0.12236328423023224,
      "rewards/rejected": -0.1881103515625,
      "step": 5090
    },
    {
      "epoch": 0.6581494386372435,
      "grad_norm": 2.5701981380020187,
      "learning_rate": 1.1202240672224076e-06,
      "log_odds_chosen": 1.486425757408142,
      "log_odds_ratio": -0.4286132752895355,
      "logits/chosen": -1.0896484851837158,
      "logits/rejected": -0.950976550579071,
      "logps/chosen": -0.716601550579071,
      "logps/rejected": -1.765625,
      "loss": 0.7705,
      "nll_loss": 0.776171863079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07158203423023224,
      "rewards/margins": 0.10505370795726776,
      "rewards/rejected": -0.17666015028953552,
      "step": 5100
    },
    {
      "epoch": 0.6594399277326106,
      "grad_norm": 2.597489690840765,
      "learning_rate": 1.1191274207850654e-06,
      "log_odds_chosen": 1.712890625,
      "log_odds_ratio": -0.34288328886032104,
      "logits/chosen": -1.1179687976837158,
      "logits/rejected": -0.981249988079071,
      "logps/chosen": -0.6356445550918579,
      "logps/rejected": -1.8132812976837158,
      "loss": 0.7657,
      "nll_loss": 0.695019543170929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06353759765625,
      "rewards/margins": 0.117828369140625,
      "rewards/rejected": -0.181396484375,
      "step": 5110
    },
    {
      "epoch": 0.6607304168279778,
      "grad_norm": 2.3515241379387666,
      "learning_rate": 1.1180339887498948e-06,
      "log_odds_chosen": 1.642675757408142,
      "log_odds_ratio": -0.3730224668979645,
      "logits/chosen": -1.060546875,
      "logits/rejected": -0.927734375,
      "logps/chosen": -0.6900390386581421,
      "logps/rejected": -1.89453125,
      "loss": 0.7561,
      "nll_loss": 0.743945300579071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06898193061351776,
      "rewards/margins": 0.120635986328125,
      "rewards/rejected": -0.18959960341453552,
      "step": 5120
    },
    {
      "epoch": 0.662020905923345,
      "grad_norm": 2.458448942659749,
      "learning_rate": 1.1169437554445213e-06,
      "log_odds_chosen": 1.729394555091858,
      "log_odds_ratio": -0.342041015625,
      "logits/chosen": -1.066015601158142,
      "logits/rejected": -0.961132824420929,
      "logps/chosen": -0.6646484136581421,
      "logps/rejected": -1.876953125,
      "loss": 0.7784,
      "nll_loss": 0.739941418170929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06647948920726776,
      "rewards/margins": 0.12115478515625,
      "rewards/rejected": -0.18771973252296448,
      "step": 5130
    },
    {
      "epoch": 0.6633113950187121,
      "grad_norm": 2.620483110523257,
      "learning_rate": 1.1158567053033413e-06,
      "log_odds_chosen": 1.6248047351837158,
      "log_odds_ratio": -0.37744140625,
      "logits/chosen": -1.065820336341858,
      "logits/rejected": -0.9048827886581421,
      "logps/chosen": -0.6869140863418579,
      "logps/rejected": -1.858007788658142,
      "loss": 0.7617,
      "nll_loss": 0.7489258050918579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06870117038488388,
      "rewards/margins": 0.1170654296875,
      "rewards/rejected": -0.18571777641773224,
      "step": 5140
    },
    {
      "epoch": 0.6646018841140793,
      "grad_norm": 3.232846397563988,
      "learning_rate": 1.1147728228665882e-06,
      "log_odds_chosen": 1.710351586341858,
      "log_odds_ratio": -0.328125,
      "logits/chosen": -1.0744140148162842,
      "logits/rejected": -0.9310547113418579,
      "logps/chosen": -0.667187511920929,
      "logps/rejected": -1.8605468273162842,
      "loss": 0.7761,
      "nll_loss": 0.729199230670929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06668701022863388,
      "rewards/margins": 0.11936035007238388,
      "rewards/rejected": -0.18598632514476776,
      "step": 5150
    },
    {
      "epoch": 0.6658923732094464,
      "grad_norm": 2.7764638414644,
      "learning_rate": 1.1136920927794092e-06,
      "log_odds_chosen": 1.625585913658142,
      "log_odds_ratio": -0.35371094942092896,
      "logits/chosen": -1.061914086341858,
      "logits/rejected": -0.950976550579071,
      "logps/chosen": -0.7378906011581421,
      "logps/rejected": -1.843359351158142,
      "loss": 0.7869,
      "nll_loss": 0.785937488079071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07380370795726776,
      "rewards/margins": 0.110437773168087,
      "rewards/rejected": -0.18439941108226776,
      "step": 5160
    },
    {
      "epoch": 0.6671828623048135,
      "grad_norm": 2.555005046984885,
      "learning_rate": 1.1126144997909508e-06,
      "log_odds_chosen": 1.4792969226837158,
      "log_odds_ratio": -0.4107910096645355,
      "logits/chosen": -1.064062476158142,
      "logits/rejected": -0.9453125,
      "logps/chosen": -0.678906261920929,
      "logps/rejected": -1.7353515625,
      "loss": 0.7688,
      "nll_loss": 0.781933605670929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06790771335363388,
      "rewards/margins": 0.105621337890625,
      "rewards/rejected": -0.1734619140625,
      "step": 5170
    },
    {
      "epoch": 0.6684733514001807,
      "grad_norm": 3.3075710251783828,
      "learning_rate": 1.1115400287534568e-06,
      "log_odds_chosen": 1.618994116783142,
      "log_odds_ratio": -0.3758789002895355,
      "logits/chosen": -1.0798828601837158,
      "logits/rejected": -0.9478515386581421,
      "logps/chosen": -0.6797851324081421,
      "logps/rejected": -1.827734351158142,
      "loss": 0.7652,
      "nll_loss": 0.7412109375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06794433295726776,
      "rewards/margins": 0.11481323093175888,
      "rewards/rejected": -0.182861328125,
      "step": 5180
    },
    {
      "epoch": 0.6697638404955478,
      "grad_norm": 2.7011200797287196,
      "learning_rate": 1.110468664621372e-06,
      "log_odds_chosen": 1.575585961341858,
      "log_odds_ratio": -0.39460450410842896,
      "logits/chosen": -1.088476538658142,
      "logits/rejected": -0.981640636920929,
      "logps/chosen": -0.6756836175918579,
      "logps/rejected": -1.7410156726837158,
      "loss": 0.75,
      "nll_loss": 0.706250011920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06755371391773224,
      "rewards/margins": 0.10646667331457138,
      "rewards/rejected": -0.17402343451976776,
      "step": 5190
    },
    {
      "epoch": 0.6710543295909149,
      "grad_norm": 2.9125077163037445,
      "learning_rate": 1.1094003924504583e-06,
      "log_odds_chosen": 1.5625,
      "log_odds_ratio": -0.41389161348342896,
      "logits/chosen": -1.095312476158142,
      "logits/rejected": -0.9765625,
      "logps/chosen": -0.671093761920929,
      "logps/rejected": -1.7999999523162842,
      "loss": 0.7613,
      "nll_loss": 0.7216796875,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06712646782398224,
      "rewards/margins": 0.11262817680835724,
      "rewards/rejected": -0.1798095703125,
      "step": 5200
    },
    {
      "epoch": 0.6723448186862822,
      "grad_norm": 3.2194271911489354,
      "learning_rate": 1.1083351973969191e-06,
      "log_odds_chosen": 1.737207055091858,
      "log_odds_ratio": -0.3533691465854645,
      "logits/chosen": -1.029296875,
      "logits/rejected": -0.913867175579071,
      "logps/chosen": -0.615527331829071,
      "logps/rejected": -1.849609375,
      "loss": 0.7574,
      "nll_loss": 0.6998046636581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06154785305261612,
      "rewards/margins": 0.12363281100988388,
      "rewards/rejected": -0.18520507216453552,
      "step": 5210
    },
    {
      "epoch": 0.6736353077816493,
      "grad_norm": 2.7403893753454063,
      "learning_rate": 1.107273064716533e-06,
      "log_odds_chosen": 1.654882788658142,
      "log_odds_ratio": -0.34492188692092896,
      "logits/chosen": -1.0544922351837158,
      "logits/rejected": -0.94140625,
      "logps/chosen": -0.6546875238418579,
      "logps/rejected": -1.789453148841858,
      "loss": 0.7687,
      "nll_loss": 0.754101574420929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06545410305261612,
      "rewards/margins": 0.11346435546875,
      "rewards/rejected": -0.17885741591453552,
      "step": 5220
    },
    {
      "epoch": 0.6749257968770164,
      "grad_norm": 2.511674207636408,
      "learning_rate": 1.1062139797637962e-06,
      "log_odds_chosen": 1.6931641101837158,
      "log_odds_ratio": -0.3992675840854645,
      "logits/chosen": -1.065820336341858,
      "logits/rejected": -0.956835925579071,
      "logps/chosen": -0.683398425579071,
      "logps/rejected": -1.911718726158142,
      "loss": 0.745,
      "nll_loss": 0.73291015625,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06826172024011612,
      "rewards/margins": 0.12294922024011612,
      "rewards/rejected": -0.19121094048023224,
      "step": 5230
    },
    {
      "epoch": 0.6762162859723835,
      "grad_norm": 2.4836262888960783,
      "learning_rate": 1.1051579279910751e-06,
      "log_odds_chosen": 1.591796875,
      "log_odds_ratio": -0.3714355528354645,
      "logits/chosen": -1.108789086341858,
      "logits/rejected": -0.947460949420929,
      "logps/chosen": -0.6878906488418579,
      "logps/rejected": -1.763281226158142,
      "loss": 0.7482,
      "nll_loss": 0.740429699420929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06875000149011612,
      "rewards/margins": 0.10759887844324112,
      "rewards/rejected": -0.17622070014476776,
      "step": 5240
    },
    {
      "epoch": 0.6775067750677507,
      "grad_norm": 2.9565318152148397,
      "learning_rate": 1.1041048949477667e-06,
      "log_odds_chosen": 1.770898461341858,
      "log_odds_ratio": -0.3408203125,
      "logits/chosen": -1.079492211341858,
      "logits/rejected": -0.9267578125,
      "logps/chosen": -0.646777331829071,
      "logps/rejected": -1.920507788658142,
      "loss": 0.7595,
      "nll_loss": 0.740039050579071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06464843451976776,
      "rewards/margins": 0.12723389267921448,
      "rewards/rejected": -0.1920166015625,
      "step": 5250
    },
    {
      "epoch": 0.6787972641631178,
      "grad_norm": 3.198778685770453,
      "learning_rate": 1.1030548662794673e-06,
      "log_odds_chosen": 1.680273413658142,
      "log_odds_ratio": -0.34575194120407104,
      "logits/chosen": -1.0427734851837158,
      "logits/rejected": -0.902539074420929,
      "logps/chosen": -0.6688476800918579,
      "logps/rejected": -1.853515625,
      "loss": 0.786,
      "nll_loss": 0.7474609613418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06684570014476776,
      "rewards/margins": 0.11850585788488388,
      "rewards/rejected": -0.18537597358226776,
      "step": 5260
    },
    {
      "epoch": 0.6800877532584849,
      "grad_norm": 3.2081299677508746,
      "learning_rate": 1.102007827727152e-06,
      "log_odds_chosen": 1.7068359851837158,
      "log_odds_ratio": -0.37666016817092896,
      "logits/chosen": -1.068750023841858,
      "logits/rejected": -0.9404296875,
      "logps/chosen": -0.665332019329071,
      "logps/rejected": -1.890234351158142,
      "loss": 0.7428,
      "nll_loss": 0.672167956829071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06644286960363388,
      "rewards/margins": 0.12259521335363388,
      "rewards/rejected": -0.18906250596046448,
      "step": 5270
    },
    {
      "epoch": 0.6813782423538521,
      "grad_norm": 2.9344853982663732,
      "learning_rate": 1.1009637651263607e-06,
      "log_odds_chosen": 1.5712890625,
      "log_odds_ratio": -0.3630615174770355,
      "logits/chosen": -1.078515648841858,
      "logits/rejected": -0.9476562738418579,
      "logps/chosen": -0.66015625,
      "logps/rejected": -1.766015648841858,
      "loss": 0.7516,
      "nll_loss": 0.679492175579071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06601562350988388,
      "rewards/margins": 0.11061401665210724,
      "rewards/rejected": -0.17670898139476776,
      "step": 5280
    },
    {
      "epoch": 0.6826687314492192,
      "grad_norm": 3.3827529915287258,
      "learning_rate": 1.0999226644063927e-06,
      "log_odds_chosen": 1.802148461341858,
      "log_odds_ratio": -0.33544921875,
      "logits/chosen": -1.0841796398162842,
      "logits/rejected": -0.9300781488418579,
      "logps/chosen": -0.6299804449081421,
      "logps/rejected": -1.8972656726837158,
      "loss": 0.7739,
      "nll_loss": 0.6732422113418579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06296386569738388,
      "rewards/margins": 0.12666015326976776,
      "rewards/rejected": -0.18955078721046448,
      "step": 5290
    },
    {
      "epoch": 0.6839592205445864,
      "grad_norm": 2.6026331806863063,
      "learning_rate": 1.0988845115895123e-06,
      "log_odds_chosen": 1.7971680164337158,
      "log_odds_ratio": -0.3687988221645355,
      "logits/chosen": -1.086328148841858,
      "logits/rejected": -0.9615234136581421,
      "logps/chosen": -0.6332031488418579,
      "logps/rejected": -1.945703148841858,
      "loss": 0.7466,
      "nll_loss": 0.728710949420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06331787258386612,
      "rewards/margins": 0.13127441704273224,
      "rewards/rejected": -0.194580078125,
      "step": 5300
    },
    {
      "epoch": 0.6852497096399536,
      "grad_norm": 2.4788295936433893,
      "learning_rate": 1.0978492927901574e-06,
      "log_odds_chosen": 1.643945336341858,
      "log_odds_ratio": -0.3677734434604645,
      "logits/chosen": -1.0705077648162842,
      "logits/rejected": -0.922656238079071,
      "logps/chosen": -0.6102539300918579,
      "logps/rejected": -1.7351562976837158,
      "loss": 0.7593,
      "nll_loss": 0.7113281488418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06096191331744194,
      "rewards/margins": 0.1124267578125,
      "rewards/rejected": -0.17348632216453552,
      "step": 5310
    },
    {
      "epoch": 0.6865401987353207,
      "grad_norm": 2.917959083349945,
      "learning_rate": 1.0968169942141634e-06,
      "log_odds_chosen": 1.569921851158142,
      "log_odds_ratio": -0.3680175840854645,
      "logits/chosen": -1.121679663658142,
      "logits/rejected": -0.9697265625,
      "logps/chosen": -0.6563476324081421,
      "logps/rejected": -1.76171875,
      "loss": 0.748,
      "nll_loss": 0.6561523675918579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06568603217601776,
      "rewards/margins": 0.1107177734375,
      "rewards/rejected": -0.17641600966453552,
      "step": 5320
    },
    {
      "epoch": 0.6878306878306878,
      "grad_norm": 3.027015046411249,
      "learning_rate": 1.0957876021579874e-06,
      "log_odds_chosen": 1.654296875,
      "log_odds_ratio": -0.3812499940395355,
      "logits/chosen": -1.023828148841858,
      "logits/rejected": -0.914257824420929,
      "logps/chosen": -0.65869140625,
      "logps/rejected": -1.858007788658142,
      "loss": 0.7755,
      "nll_loss": 0.7530273199081421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06580810248851776,
      "rewards/margins": 0.12014160305261612,
      "rewards/rejected": -0.18586425483226776,
      "step": 5330
    },
    {
      "epoch": 0.689121176926055,
      "grad_norm": 2.824707143891119,
      "learning_rate": 1.0947611030079466e-06,
      "log_odds_chosen": 1.9675781726837158,
      "log_odds_ratio": -0.3235839903354645,
      "logits/chosen": -1.0515625476837158,
      "logits/rejected": -0.915820300579071,
      "logps/chosen": -0.677441418170929,
      "logps/rejected": -2.1109375953674316,
      "loss": 0.7396,
      "nll_loss": 0.732421875,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06771240383386612,
      "rewards/margins": 0.14340820908546448,
      "rewards/rejected": -0.21103516221046448,
      "step": 5340
    },
    {
      "epoch": 0.6904116660214221,
      "grad_norm": 2.524046771628622,
      "learning_rate": 1.0937374832394612e-06,
      "log_odds_chosen": 1.644921898841858,
      "log_odds_ratio": -0.3836425840854645,
      "logits/chosen": -1.0476562976837158,
      "logits/rejected": -0.937695324420929,
      "logps/chosen": -0.7162109613418579,
      "logps/rejected": -1.9167969226837158,
      "loss": 0.771,
      "nll_loss": 0.7198241949081421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07163085788488388,
      "rewards/margins": 0.11995849758386612,
      "rewards/rejected": -0.19143065810203552,
      "step": 5350
    },
    {
      "epoch": 0.6917021551167892,
      "grad_norm": 2.810855527924549,
      "learning_rate": 1.092716729416306e-06,
      "log_odds_chosen": 1.697167992591858,
      "log_odds_ratio": -0.3415283262729645,
      "logits/chosen": -1.0496094226837158,
      "logits/rejected": -0.9248046875,
      "logps/chosen": -0.630664050579071,
      "logps/rejected": -1.8312499523162842,
      "loss": 0.7594,
      "nll_loss": 0.688281238079071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06312255561351776,
      "rewards/margins": 0.11989746242761612,
      "rewards/rejected": -0.18312987685203552,
      "step": 5360
    },
    {
      "epoch": 0.6929926442121565,
      "grad_norm": 2.6791242288340933,
      "learning_rate": 1.0916988281898703e-06,
      "log_odds_chosen": 1.4255859851837158,
      "log_odds_ratio": -0.3965820372104645,
      "logits/chosen": -1.102148413658142,
      "logits/rejected": -0.9671875238418579,
      "logps/chosen": -0.634472668170929,
      "logps/rejected": -1.5947265625,
      "loss": 0.7629,
      "nll_loss": 0.6998046636581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06343994289636612,
      "rewards/margins": 0.09598388522863388,
      "rewards/rejected": -0.15939942002296448,
      "step": 5370
    },
    {
      "epoch": 0.6942831333075236,
      "grad_norm": 3.142093653309373,
      "learning_rate": 1.0906837662984237e-06,
      "log_odds_chosen": 1.618749976158142,
      "log_odds_ratio": -0.42333984375,
      "logits/chosen": -1.1134765148162842,
      "logits/rejected": -0.9873046875,
      "logps/chosen": -0.687792956829071,
      "logps/rejected": -1.8425781726837158,
      "loss": 0.7472,
      "nll_loss": 0.72265625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0687255859375,
      "rewards/margins": 0.11567382514476776,
      "rewards/rejected": -0.184326171875,
      "step": 5380
    },
    {
      "epoch": 0.6955736224028907,
      "grad_norm": 3.0416531575315724,
      "learning_rate": 1.089671530566391e-06,
      "log_odds_chosen": 1.7824218273162842,
      "log_odds_ratio": -0.3616699278354645,
      "logits/chosen": -1.080078125,
      "logits/rejected": -0.950976550579071,
      "logps/chosen": -0.6539062261581421,
      "logps/rejected": -1.9187500476837158,
      "loss": 0.7681,
      "nll_loss": 0.7120116949081421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06533203274011612,
      "rewards/margins": 0.12650147080421448,
      "rewards/rejected": -0.19189453125,
      "step": 5390
    },
    {
      "epoch": 0.6968641114982579,
      "grad_norm": 2.855616969551232,
      "learning_rate": 1.0886621079036346e-06,
      "log_odds_chosen": 1.85546875,
      "log_odds_ratio": -0.32377928495407104,
      "logits/chosen": -1.117578148841858,
      "logits/rejected": -0.9937499761581421,
      "logps/chosen": -0.671582043170929,
      "logps/rejected": -2.024218797683716,
      "loss": 0.7578,
      "nll_loss": 0.707226574420929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06712646782398224,
      "rewards/margins": 0.13503417372703552,
      "rewards/rejected": -0.20219726860523224,
      "step": 5400
    },
    {
      "epoch": 0.698154600593625,
      "grad_norm": 2.8295504671882457,
      "learning_rate": 1.0876554853047417e-06,
      "log_odds_chosen": 1.44775390625,
      "log_odds_ratio": -0.3916015625,
      "logits/chosen": -1.0603516101837158,
      "logits/rejected": -0.9638671875,
      "logps/chosen": -0.677734375,
      "logps/rejected": -1.626562476158142,
      "loss": 0.7516,
      "nll_loss": 0.687695324420929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.0677490234375,
      "rewards/margins": 0.09488067775964737,
      "rewards/rejected": -0.16269531846046448,
      "step": 5410
    },
    {
      "epoch": 0.6994450896889921,
      "grad_norm": 2.884567667275289,
      "learning_rate": 1.0866516498483225e-06,
      "log_odds_chosen": 1.6345703601837158,
      "log_odds_ratio": -0.3551269471645355,
      "logits/chosen": -1.0828125476837158,
      "logits/rejected": -0.9330078363418579,
      "logps/chosen": -0.695117175579071,
      "logps/rejected": -1.837499976158142,
      "loss": 0.7601,
      "nll_loss": 0.7618163824081421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06955566257238388,
      "rewards/margins": 0.11419677734375,
      "rewards/rejected": -0.18369141221046448,
      "step": 5420
    },
    {
      "epoch": 0.7007355787843593,
      "grad_norm": 3.5786279835890773,
      "learning_rate": 1.0856505886963116e-06,
      "log_odds_chosen": 1.7800781726837158,
      "log_odds_ratio": -0.29926759004592896,
      "logits/chosen": -1.041601538658142,
      "logits/rejected": -0.9267578125,
      "logps/chosen": -0.648632824420929,
      "logps/rejected": -1.880859375,
      "loss": 0.7718,
      "nll_loss": 0.7201172113418579,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06480713188648224,
      "rewards/margins": 0.12338867038488388,
      "rewards/rejected": -0.18815918266773224,
      "step": 5430
    },
    {
      "epoch": 0.7020260678797264,
      "grad_norm": 4.467778455795293,
      "learning_rate": 1.0846522890932808e-06,
      "log_odds_chosen": 1.744726538658142,
      "log_odds_ratio": -0.37861329317092896,
      "logits/chosen": -1.092187523841858,
      "logits/rejected": -0.9623047113418579,
      "logps/chosen": -0.7010742425918579,
      "logps/rejected": -1.9871094226837158,
      "loss": 0.7626,
      "nll_loss": 0.728515625,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.070068359375,
      "rewards/margins": 0.12871094048023224,
      "rewards/rejected": -0.19868163764476776,
      "step": 5440
    },
    {
      "epoch": 0.7033165569750935,
      "grad_norm": 2.6075092885892195,
      "learning_rate": 1.0836567383657542e-06,
      "log_odds_chosen": 1.715234398841858,
      "log_odds_ratio": -0.36406248807907104,
      "logits/chosen": -1.090429663658142,
      "logits/rejected": -0.931445300579071,
      "logps/chosen": -0.621874988079071,
      "logps/rejected": -1.828515648841858,
      "loss": 0.7589,
      "nll_loss": 0.67333984375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06219482421875,
      "rewards/margins": 0.12058105319738388,
      "rewards/rejected": -0.1827392578125,
      "step": 5450
    },
    {
      "epoch": 0.7046070460704607,
      "grad_norm": 3.095769944619177,
      "learning_rate": 1.0826639239215334e-06,
      "log_odds_chosen": 1.7082030773162842,
      "log_odds_ratio": -0.328369140625,
      "logits/chosen": -1.1142578125,
      "logits/rejected": -0.962695300579071,
      "logps/chosen": -0.6285156011581421,
      "logps/rejected": -1.8371093273162842,
      "loss": 0.7439,
      "nll_loss": 0.658203125,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06285400688648224,
      "rewards/margins": 0.12066650390625,
      "rewards/rejected": -0.18364258110523224,
      "step": 5460
    },
    {
      "epoch": 0.7058975351658279,
      "grad_norm": 3.2087442258892724,
      "learning_rate": 1.0816738332490292e-06,
      "log_odds_chosen": 2.091992139816284,
      "log_odds_ratio": -0.3034912049770355,
      "logits/chosen": -1.0988280773162842,
      "logits/rejected": -0.9691406488418579,
      "logps/chosen": -0.6197265386581421,
      "logps/rejected": -2.128124952316284,
      "loss": 0.7185,
      "nll_loss": 0.758593738079071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06196289137005806,
      "rewards/margins": 0.15106201171875,
      "rewards/rejected": -0.21291503310203552,
      "step": 5470
    },
    {
      "epoch": 0.707188024261195,
      "grad_norm": 3.249714513845728,
      "learning_rate": 1.0806864539165982e-06,
      "log_odds_chosen": 1.7615234851837158,
      "log_odds_ratio": -0.34648436307907104,
      "logits/chosen": -1.150781273841858,
      "logits/rejected": -1.0224609375,
      "logps/chosen": -0.637499988079071,
      "logps/rejected": -1.8562500476837158,
      "loss": 0.7363,
      "nll_loss": 0.666796863079071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06375732272863388,
      "rewards/margins": 0.12176513671875,
      "rewards/rejected": -0.185546875,
      "step": 5480
    },
    {
      "epoch": 0.7084785133565621,
      "grad_norm": 3.1727913279801085,
      "learning_rate": 1.0797017735718878e-06,
      "log_odds_chosen": 1.5412108898162842,
      "log_odds_ratio": -0.3921875059604645,
      "logits/chosen": -1.146484375,
      "logits/rejected": -1.008398413658142,
      "logps/chosen": -0.6572265625,
      "logps/rejected": -1.7644531726837158,
      "loss": 0.733,
      "nll_loss": 0.65478515625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06564941257238388,
      "rewards/margins": 0.11074218899011612,
      "rewards/rejected": -0.17636719346046448,
      "step": 5490
    },
    {
      "epoch": 0.7097690024519293,
      "grad_norm": 2.968295014175309,
      "learning_rate": 1.0787197799411874e-06,
      "log_odds_chosen": 1.7663085460662842,
      "log_odds_ratio": -0.3144897520542145,
      "logits/chosen": -1.050195336341858,
      "logits/rejected": -0.9384765625,
      "logps/chosen": -0.6356445550918579,
      "logps/rejected": -1.853906273841858,
      "loss": 0.7346,
      "nll_loss": 0.7333984375,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06362304836511612,
      "rewards/margins": 0.12160644680261612,
      "rewards/rejected": -0.18532714247703552,
      "step": 5500
    },
    {
      "epoch": 0.7110594915472964,
      "grad_norm": 2.9604172132733733,
      "learning_rate": 1.0777404608287846e-06,
      "log_odds_chosen": 1.554101586341858,
      "log_odds_ratio": -0.39960938692092896,
      "logits/chosen": -1.130273461341858,
      "logits/rejected": -0.9957031011581421,
      "logps/chosen": -0.7109375,
      "logps/rejected": -1.818359375,
      "loss": 0.7688,
      "nll_loss": 0.720703125,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07109375298023224,
      "rewards/margins": 0.11077880859375,
      "rewards/rejected": -0.18183593451976776,
      "step": 5510
    },
    {
      "epoch": 0.7123499806426635,
      "grad_norm": 3.2413622905289414,
      "learning_rate": 1.0767638041163309e-06,
      "log_odds_chosen": 1.767187476158142,
      "log_odds_ratio": -0.34296876192092896,
      "logits/chosen": -1.0595703125,
      "logits/rejected": -0.9410156011581421,
      "logps/chosen": -0.6329101324081421,
      "logps/rejected": -1.873632788658142,
      "loss": 0.742,
      "nll_loss": 0.686230480670929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06334228813648224,
      "rewards/margins": 0.12395019829273224,
      "rewards/rejected": -0.18723145127296448,
      "step": 5520
    },
    {
      "epoch": 0.7136404697380307,
      "grad_norm": 13.130170555910942,
      "learning_rate": 1.0757897977622107e-06,
      "log_odds_chosen": 1.6541016101837158,
      "log_odds_ratio": -0.3766845762729645,
      "logits/chosen": -1.0896484851837158,
      "logits/rejected": -0.9654296636581421,
      "logps/chosen": -0.604687511920929,
      "logps/rejected": -1.7248046398162842,
      "loss": 0.7352,
      "nll_loss": 0.6336914300918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06044922024011612,
      "rewards/margins": 0.11195068061351776,
      "rewards/rejected": -0.17246094346046448,
      "step": 5530
    },
    {
      "epoch": 0.7149309588333979,
      "grad_norm": 3.839447404914077,
      "learning_rate": 1.074818429800918e-06,
      "log_odds_chosen": 1.814453125,
      "log_odds_ratio": -0.3204101622104645,
      "logits/chosen": -1.083398461341858,
      "logits/rejected": -0.9652343988418579,
      "logps/chosen": -0.5970703363418579,
      "logps/rejected": -1.8484375476837158,
      "loss": 0.7203,
      "nll_loss": 0.694140613079071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.05965576320886612,
      "rewards/margins": 0.12531737983226776,
      "rewards/rejected": -0.18491211533546448,
      "step": 5540
    },
    {
      "epoch": 0.716221447928765,
      "grad_norm": 3.2571608027852132,
      "learning_rate": 1.073849688342439e-06,
      "log_odds_chosen": 1.4376952648162842,
      "log_odds_ratio": -0.3887695372104645,
      "logits/chosen": -1.032812476158142,
      "logits/rejected": -0.9468749761581421,
      "logps/chosen": -0.6913086175918579,
      "logps/rejected": -1.696679711341858,
      "loss": 0.7612,
      "nll_loss": 0.6937500238418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06916503608226776,
      "rewards/margins": 0.10040283203125,
      "rewards/rejected": -0.16962890326976776,
      "step": 5550
    },
    {
      "epoch": 0.7175119370241322,
      "grad_norm": 2.805392628110819,
      "learning_rate": 1.07288356157164e-06,
      "log_odds_chosen": 1.724609375,
      "log_odds_ratio": -0.34370118379592896,
      "logits/chosen": -1.095703125,
      "logits/rejected": -0.983593761920929,
      "logps/chosen": -0.6763671636581421,
      "logps/rejected": -1.91015625,
      "loss": 0.7378,
      "nll_loss": 0.720898449420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06761474907398224,
      "rewards/margins": 0.12342528998851776,
      "rewards/rejected": -0.19106444716453552,
      "step": 5560
    },
    {
      "epoch": 0.7188024261194993,
      "grad_norm": 3.0838413876153066,
      "learning_rate": 1.0719200377476648e-06,
      "log_odds_chosen": 1.808984398841858,
      "log_odds_ratio": -0.3412109315395355,
      "logits/chosen": -1.0509765148162842,
      "logits/rejected": -0.894335925579071,
      "logps/chosen": -0.6396484375,
      "logps/rejected": -1.9482421875,
      "loss": 0.7293,
      "nll_loss": 0.6871093511581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06395263969898224,
      "rewards/margins": 0.13059082627296448,
      "rewards/rejected": -0.19462890923023224,
      "step": 5570
    },
    {
      "epoch": 0.7200929152148664,
      "grad_norm": 4.240431508880807,
      "learning_rate": 1.0709591052033317e-06,
      "log_odds_chosen": 1.7999999523162842,
      "log_odds_ratio": -0.3349609375,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.883984386920929,
      "logps/chosen": -0.698046863079071,
      "logps/rejected": -1.974218726158142,
      "loss": 0.7593,
      "nll_loss": 0.8212890625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06979980319738388,
      "rewards/margins": 0.12755127251148224,
      "rewards/rejected": -0.197265625,
      "step": 5580
    },
    {
      "epoch": 0.7213834043102336,
      "grad_norm": 3.1931111979703757,
      "learning_rate": 1.0700007523445435e-06,
      "log_odds_chosen": 1.666601538658142,
      "log_odds_ratio": -0.39580076932907104,
      "logits/chosen": -1.096093773841858,
      "logits/rejected": -0.9466797113418579,
      "logps/chosen": -0.685351550579071,
      "logps/rejected": -1.8937499523162842,
      "loss": 0.7273,
      "nll_loss": 0.689453125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06851806491613388,
      "rewards/margins": 0.12093506008386612,
      "rewards/rejected": -0.18935546278953552,
      "step": 5590
    },
    {
      "epoch": 0.7226738934056007,
      "grad_norm": 2.8651837677727494,
      "learning_rate": 1.0690449676496976e-06,
      "log_odds_chosen": 1.8474609851837158,
      "log_odds_ratio": -0.31494140625,
      "logits/chosen": -1.1013672351837158,
      "logits/rejected": -0.9791015386581421,
      "logps/chosen": -0.6484375,
      "logps/rejected": -1.99609375,
      "loss": 0.7214,
      "nll_loss": 0.6524413824081421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06485595554113388,
      "rewards/margins": 0.13480225205421448,
      "rewards/rejected": -0.19968262314796448,
      "step": 5600
    },
    {
      "epoch": 0.7239643825009678,
      "grad_norm": 4.094545456320464,
      "learning_rate": 1.0680917396691054e-06,
      "log_odds_chosen": 1.569726586341858,
      "log_odds_ratio": -0.39277344942092896,
      "logits/chosen": -1.06640625,
      "logits/rejected": -0.928906261920929,
      "logps/chosen": -0.691113293170929,
      "logps/rejected": -1.8097655773162842,
      "loss": 0.7447,
      "nll_loss": 0.7476562261581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06906738132238388,
      "rewards/margins": 0.11171875149011612,
      "rewards/rejected": -0.1810302734375,
      "step": 5610
    },
    {
      "epoch": 0.7252548715963351,
      "grad_norm": 2.7807794491307734,
      "learning_rate": 1.0671410570244164e-06,
      "log_odds_chosen": 1.7819335460662842,
      "log_odds_ratio": -0.3267578184604645,
      "logits/chosen": -1.1240234375,
      "logits/rejected": -0.9662109613418579,
      "logps/chosen": -0.6791015863418579,
      "logps/rejected": -1.984765648841858,
      "loss": 0.7372,
      "nll_loss": 0.685546875,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06789550930261612,
      "rewards/margins": 0.13052979111671448,
      "rewards/rejected": -0.1982421875,
      "step": 5620
    },
    {
      "epoch": 0.7265453606917022,
      "grad_norm": 3.014101944445689,
      "learning_rate": 1.0661929084080466e-06,
      "log_odds_chosen": 1.631738305091858,
      "log_odds_ratio": -0.37482911348342896,
      "logits/chosen": -1.115625023841858,
      "logits/rejected": -0.985546886920929,
      "logps/chosen": -0.643359363079071,
      "logps/rejected": -1.788476586341858,
      "loss": 0.7401,
      "nll_loss": 0.6533203125,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06427001953125,
      "rewards/margins": 0.11446838080883026,
      "rewards/rejected": -0.17878417670726776,
      "step": 5630
    },
    {
      "epoch": 0.7278358497870693,
      "grad_norm": 2.598811079610422,
      "learning_rate": 1.0652472825826149e-06,
      "log_odds_chosen": 1.9080078601837158,
      "log_odds_ratio": -0.3142333924770355,
      "logits/chosen": -1.066015601158142,
      "logits/rejected": -0.9605468511581421,
      "logps/chosen": -0.6209961175918579,
      "logps/rejected": -1.9435546398162842,
      "loss": 0.7404,
      "nll_loss": 0.6924804449081421,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.06206054612994194,
      "rewards/margins": 0.13237304985523224,
      "rewards/rejected": -0.19453124701976776,
      "step": 5640
    },
    {
      "epoch": 0.7291263388824365,
      "grad_norm": 3.3451009930037117,
      "learning_rate": 1.0643041683803828e-06,
      "log_odds_chosen": 1.828125,
      "log_odds_ratio": -0.34541016817092896,
      "logits/chosen": -1.0535156726837158,
      "logits/rejected": -0.95703125,
      "logps/chosen": -0.6499999761581421,
      "logps/rejected": -1.941015601158142,
      "loss": 0.7467,
      "nll_loss": 0.7108398675918579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06501464545726776,
      "rewards/margins": 0.12908935546875,
      "rewards/rejected": -0.19394531846046448,
      "step": 5650
    },
    {
      "epoch": 0.7304168279778036,
      "grad_norm": 2.7464516410129907,
      "learning_rate": 1.063363554702701e-06,
      "log_odds_chosen": 1.5814940929412842,
      "log_odds_ratio": -0.4044433534145355,
      "logits/chosen": -1.0734374523162842,
      "logits/rejected": -0.9585937261581421,
      "logps/chosen": -0.6971679925918579,
      "logps/rejected": -1.781640648841858,
      "loss": 0.7591,
      "nll_loss": 0.6976562738418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.0697021484375,
      "rewards/margins": 0.1085052490234375,
      "rewards/rejected": -0.17822265625,
      "step": 5660
    },
    {
      "epoch": 0.7317073170731707,
      "grad_norm": 2.8402480693726386,
      "learning_rate": 1.0624254305194609e-06,
      "log_odds_chosen": 1.826562523841858,
      "log_odds_ratio": -0.3216796815395355,
      "logits/chosen": -1.065820336341858,
      "logits/rejected": -0.946093738079071,
      "logps/chosen": -0.6797851324081421,
      "logps/rejected": -1.962499976158142,
      "loss": 0.7326,
      "nll_loss": 0.68408203125,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06794433295726776,
      "rewards/margins": 0.12830810248851776,
      "rewards/rejected": -0.19624023139476776,
      "step": 5670
    },
    {
      "epoch": 0.7329978061685378,
      "grad_norm": 2.826870966441485,
      "learning_rate": 1.0614897848685505e-06,
      "log_odds_chosen": 1.5666992664337158,
      "log_odds_ratio": -0.397705078125,
      "logits/chosen": -1.0888671875,
      "logits/rejected": -0.943164050579071,
      "logps/chosen": -0.6494140625,
      "logps/rejected": -1.747460961341858,
      "loss": 0.7535,
      "nll_loss": 0.6923828125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06495361030101776,
      "rewards/margins": 0.10983886569738388,
      "rewards/rejected": -0.17485351860523224,
      "step": 5680
    },
    {
      "epoch": 0.734288295263905,
      "grad_norm": 3.0199352010639218,
      "learning_rate": 1.0605566068553173e-06,
      "log_odds_chosen": 1.62939453125,
      "log_odds_ratio": -0.3932128846645355,
      "logits/chosen": -1.118554711341858,
      "logits/rejected": -1.0046875476837158,
      "logps/chosen": -0.657910168170929,
      "logps/rejected": -1.7976562976837158,
      "loss": 0.7128,
      "nll_loss": 0.642871081829071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06575927883386612,
      "rewards/margins": 0.11390991508960724,
      "rewards/rejected": -0.17978516221046448,
      "step": 5690
    },
    {
      "epoch": 0.7355787843592722,
      "grad_norm": 4.08023481374344,
      "learning_rate": 1.0596258856520351e-06,
      "log_odds_chosen": 1.7211425304412842,
      "log_odds_ratio": -0.34223634004592896,
      "logits/chosen": -1.0929687023162842,
      "logits/rejected": -0.9722656011581421,
      "logps/chosen": -0.6656249761581421,
      "logps/rejected": -1.828125,
      "loss": 0.7336,
      "nll_loss": 0.739062488079071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06658935546875,
      "rewards/margins": 0.11611328274011612,
      "rewards/rejected": -0.182861328125,
      "step": 5700
    },
    {
      "epoch": 0.7368692734546393,
      "grad_norm": 2.834983271027768,
      "learning_rate": 1.0586976104973764e-06,
      "log_odds_chosen": 2.0953125953674316,
      "log_odds_ratio": -0.2857910096645355,
      "logits/chosen": -1.126562476158142,
      "logits/rejected": -0.9371093511581421,
      "logps/chosen": -0.645800769329071,
      "logps/rejected": -2.2015624046325684,
      "loss": 0.7379,
      "nll_loss": 0.66455078125,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06456299126148224,
      "rewards/margins": 0.1556396484375,
      "rewards/rejected": -0.22021484375,
      "step": 5710
    },
    {
      "epoch": 0.7381597625500065,
      "grad_norm": 2.6346579493423103,
      "learning_rate": 1.05777177069589e-06,
      "log_odds_chosen": 1.781835913658142,
      "log_odds_ratio": -0.34418946504592896,
      "logits/chosen": -1.083593726158142,
      "logits/rejected": -0.965624988079071,
      "logps/chosen": -0.6493164300918579,
      "logps/rejected": -1.9035155773162842,
      "loss": 0.7392,
      "nll_loss": 0.646777331829071,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06485595554113388,
      "rewards/margins": 0.12543945014476776,
      "rewards/rejected": -0.19033202528953552,
      "step": 5720
    },
    {
      "epoch": 0.7394502516453736,
      "grad_norm": 3.403491675566296,
      "learning_rate": 1.0568483556174834e-06,
      "log_odds_chosen": 1.822851538658142,
      "log_odds_ratio": -0.3280273377895355,
      "logits/chosen": -1.0537109375,
      "logits/rejected": -0.933398425579071,
      "logps/chosen": -0.6441406011581421,
      "logps/rejected": -1.9621093273162842,
      "loss": 0.7262,
      "nll_loss": 0.6763671636581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06436767429113388,
      "rewards/margins": 0.1317138671875,
      "rewards/rejected": -0.19614258408546448,
      "step": 5730
    },
    {
      "epoch": 0.7407407407407407,
      "grad_norm": 2.621050095641156,
      "learning_rate": 1.055927354696909e-06,
      "log_odds_chosen": 1.6974608898162842,
      "log_odds_ratio": -0.3336425721645355,
      "logits/chosen": -1.152929663658142,
      "logits/rejected": -1.0056641101837158,
      "logps/chosen": -0.621386706829071,
      "logps/rejected": -1.782812476158142,
      "loss": 0.7085,
      "nll_loss": 0.6043945550918579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06215820461511612,
      "rewards/margins": 0.11617126315832138,
      "rewards/rejected": -0.17836913466453552,
      "step": 5740
    },
    {
      "epoch": 0.7420312298361079,
      "grad_norm": 3.0997824342029965,
      "learning_rate": 1.0550087574332592e-06,
      "log_odds_chosen": 1.7863280773162842,
      "log_odds_ratio": -0.33161622285842896,
      "logits/chosen": -1.0304687023162842,
      "logits/rejected": -0.909375011920929,
      "logps/chosen": -0.596972644329071,
      "logps/rejected": -1.8896484375,
      "loss": 0.738,
      "nll_loss": 0.6978515386581421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05972900241613388,
      "rewards/margins": 0.129150390625,
      "rewards/rejected": -0.18874511122703552,
      "step": 5750
    },
    {
      "epoch": 0.743321718931475,
      "grad_norm": 3.1195051113944854,
      "learning_rate": 1.0540925533894598e-06,
      "log_odds_chosen": 1.7726562023162842,
      "log_odds_ratio": -0.3448242247104645,
      "logits/chosen": -1.0808594226837158,
      "logits/rejected": -0.969921886920929,
      "logps/chosen": -0.653613269329071,
      "logps/rejected": -1.915624976158142,
      "loss": 0.7258,
      "nll_loss": 0.696484386920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06540527194738388,
      "rewards/margins": 0.126220703125,
      "rewards/rejected": -0.19155272841453552,
      "step": 5760
    },
    {
      "epoch": 0.7446122080268421,
      "grad_norm": 2.804481304375375,
      "learning_rate": 1.053178732191775e-06,
      "log_odds_chosen": 1.821313500404358,
      "log_odds_ratio": -0.358642578125,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.9535156488418579,
      "logps/chosen": -0.611523449420929,
      "logps/rejected": -1.890625,
      "loss": 0.7301,
      "nll_loss": 0.6083008050918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06116943433880806,
      "rewards/margins": 0.12788239121437073,
      "rewards/rejected": -0.18918457627296448,
      "step": 5770
    },
    {
      "epoch": 0.7459026971222094,
      "grad_norm": 2.56355935460926,
      "learning_rate": 1.0522672835293127e-06,
      "log_odds_chosen": 1.736914038658142,
      "log_odds_ratio": -0.37749022245407104,
      "logits/chosen": -1.061132788658142,
      "logits/rejected": -0.9515625238418579,
      "logps/chosen": -0.6878906488418579,
      "logps/rejected": -1.9503905773162842,
      "loss": 0.7373,
      "nll_loss": 0.7939453125,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06878662109375,
      "rewards/margins": 0.12626953423023224,
      "rewards/rejected": -0.19516602158546448,
      "step": 5780
    },
    {
      "epoch": 0.7471931862175765,
      "grad_norm": 2.54328669599345,
      "learning_rate": 1.0513581971535365e-06,
      "log_odds_chosen": 1.8630859851837158,
      "log_odds_ratio": -0.33549803495407104,
      "logits/chosen": -1.1404297351837158,
      "logits/rejected": -0.9580078125,
      "logps/chosen": -0.642382800579071,
      "logps/rejected": -1.988671898841858,
      "loss": 0.7435,
      "nll_loss": 0.679980456829071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06427001953125,
      "rewards/margins": 0.13475342094898224,
      "rewards/rejected": -0.19890137016773224,
      "step": 5790
    },
    {
      "epoch": 0.7484836753129436,
      "grad_norm": 2.9414468001130185,
      "learning_rate": 1.0504514628777803e-06,
      "log_odds_chosen": 1.6482422351837158,
      "log_odds_ratio": -0.38282471895217896,
      "logits/chosen": -1.0710937976837158,
      "logits/rejected": -0.984375,
      "logps/chosen": -0.648144543170929,
      "logps/rejected": -1.8035156726837158,
      "loss": 0.7364,
      "nll_loss": 0.647656261920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06491699069738388,
      "rewards/margins": 0.11558838188648224,
      "rewards/rejected": -0.18039551377296448,
      "step": 5800
    },
    {
      "epoch": 0.7497741644083108,
      "grad_norm": 2.8771256436526946,
      "learning_rate": 1.0495470705767713e-06,
      "log_odds_chosen": 1.82958984375,
      "log_odds_ratio": -0.339111328125,
      "logits/chosen": -1.103124976158142,
      "logits/rejected": -0.984375,
      "logps/chosen": -0.6136718988418579,
      "logps/rejected": -1.880273461341858,
      "loss": 0.7048,
      "nll_loss": 0.612109363079071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06141357496380806,
      "rewards/margins": 0.12665405869483948,
      "rewards/rejected": -0.18801268935203552,
      "step": 5810
    },
    {
      "epoch": 0.7510646535036779,
      "grad_norm": 3.4092278434406458,
      "learning_rate": 1.0486450101861527e-06,
      "log_odds_chosen": 1.4625976085662842,
      "log_odds_ratio": -0.45849609375,
      "logits/chosen": -1.080468773841858,
      "logits/rejected": -0.966992199420929,
      "logps/chosen": -0.7142578363418579,
      "logps/rejected": -1.751562476158142,
      "loss": 0.7167,
      "nll_loss": 0.7420898675918579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07145996391773224,
      "rewards/margins": 0.10368652641773224,
      "rewards/rejected": -0.17514649033546448,
      "step": 5820
    },
    {
      "epoch": 0.752355142599045,
      "grad_norm": 2.9461062393146533,
      "learning_rate": 1.0477452717020143e-06,
      "log_odds_chosen": 1.6555907726287842,
      "log_odds_ratio": -0.3524169921875,
      "logits/chosen": -1.1238281726837158,
      "logits/rejected": -1.015234351158142,
      "logps/chosen": -0.6908203363418579,
      "logps/rejected": -1.8671875,
      "loss": 0.7466,
      "nll_loss": 0.7313476800918579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06907959282398224,
      "rewards/margins": 0.117401123046875,
      "rewards/rejected": -0.18659667670726776,
      "step": 5830
    },
    {
      "epoch": 0.7536456316944122,
      "grad_norm": 3.720889726037911,
      "learning_rate": 1.0468478451804272e-06,
      "log_odds_chosen": 1.669921875,
      "log_odds_ratio": -0.3515625,
      "logits/chosen": -1.130859375,
      "logits/rejected": -0.9830077886581421,
      "logps/chosen": -0.67333984375,
      "logps/rejected": -1.8837890625,
      "loss": 0.7161,
      "nll_loss": 0.7291015386581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06730957329273224,
      "rewards/margins": 0.12104491889476776,
      "rewards/rejected": -0.18840332329273224,
      "step": 5840
    },
    {
      "epoch": 0.7549361207897793,
      "grad_norm": 3.3561547211615816,
      "learning_rate": 1.0459527207369814e-06,
      "log_odds_chosen": 1.6535155773162842,
      "log_odds_ratio": -0.36909180879592896,
      "logits/chosen": -1.072656273841858,
      "logits/rejected": -0.941601574420929,
      "logps/chosen": -0.649218738079071,
      "logps/rejected": -1.835546851158142,
      "loss": 0.7135,
      "nll_loss": 0.711718738079071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06490478664636612,
      "rewards/margins": 0.11868896335363388,
      "rewards/rejected": -0.18369141221046448,
      "step": 5850
    },
    {
      "epoch": 0.7562266098851465,
      "grad_norm": 2.6377642047838874,
      "learning_rate": 1.0450598885463281e-06,
      "log_odds_chosen": 1.8869140148162842,
      "log_odds_ratio": -0.2914062440395355,
      "logits/chosen": -1.1326172351837158,
      "logits/rejected": -0.9462890625,
      "logps/chosen": -0.664843738079071,
      "logps/rejected": -2.012890577316284,
      "loss": 0.7206,
      "nll_loss": 0.65087890625,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06641845405101776,
      "rewards/margins": 0.13477782905101776,
      "rewards/rejected": -0.201171875,
      "step": 5860
    },
    {
      "epoch": 0.7575170989805137,
      "grad_norm": 3.0802154842618528,
      "learning_rate": 1.0441693388417282e-06,
      "log_odds_chosen": 1.806640625,
      "log_odds_ratio": -0.3306640684604645,
      "logits/chosen": -1.0556640625,
      "logits/rejected": -0.965624988079071,
      "logps/chosen": -0.629687488079071,
      "logps/rejected": -1.9201171398162842,
      "loss": 0.7204,
      "nll_loss": 0.666796863079071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06295166164636612,
      "rewards/margins": 0.12906494736671448,
      "rewards/rejected": -0.19204100966453552,
      "step": 5870
    },
    {
      "epoch": 0.7588075880758808,
      "grad_norm": 2.9283993731543068,
      "learning_rate": 1.0432810619146023e-06,
      "log_odds_chosen": 2.1566405296325684,
      "log_odds_ratio": -0.26618653535842896,
      "logits/chosen": -1.054101586341858,
      "logits/rejected": -0.9175781011581421,
      "logps/chosen": -0.578808605670929,
      "logps/rejected": -2.0835938453674316,
      "loss": 0.732,
      "nll_loss": 0.671679675579071,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.05789794772863388,
      "rewards/margins": 0.15031738579273224,
      "rewards/rejected": -0.2083740234375,
      "step": 5880
    },
    {
      "epoch": 0.7600980771712479,
      "grad_norm": 2.805003415330904,
      "learning_rate": 1.042395048114086e-06,
      "log_odds_chosen": 1.590429663658142,
      "log_odds_ratio": -0.34814453125,
      "logits/chosen": -1.089257836341858,
      "logits/rejected": -0.945117175579071,
      "logps/chosen": -0.617968738079071,
      "logps/rejected": -1.732421875,
      "loss": 0.7388,
      "nll_loss": 0.70654296875,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06177978590130806,
      "rewards/margins": 0.11156006157398224,
      "rewards/rejected": -0.17341308295726776,
      "step": 5890
    },
    {
      "epoch": 0.761388566266615,
      "grad_norm": 3.1469936058228756,
      "learning_rate": 1.041511287846591e-06,
      "log_odds_chosen": 1.830468773841858,
      "log_odds_ratio": -0.3866943418979645,
      "logits/chosen": -1.0857422351837158,
      "logits/rejected": -0.962695300579071,
      "logps/chosen": -0.672070324420929,
      "logps/rejected": -2.0082030296325684,
      "loss": 0.74,
      "nll_loss": 0.702929675579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06724853813648224,
      "rewards/margins": 0.13371582329273224,
      "rewards/rejected": -0.20083007216453552,
      "step": 5900
    },
    {
      "epoch": 0.7626790553619822,
      "grad_norm": 2.7492680405494383,
      "learning_rate": 1.0406297715753674e-06,
      "log_odds_chosen": 1.8037109375,
      "log_odds_ratio": -0.32988280057907104,
      "logits/chosen": -1.097265601158142,
      "logits/rejected": -0.9468749761581421,
      "logps/chosen": -0.6253906488418579,
      "logps/rejected": -1.9054687023162842,
      "loss": 0.7243,
      "nll_loss": 0.69091796875,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06253661960363388,
      "rewards/margins": 0.12803955376148224,
      "rewards/rejected": -0.19052734971046448,
      "step": 5910
    },
    {
      "epoch": 0.7639695444573493,
      "grad_norm": 2.716808412584315,
      "learning_rate": 1.0397504898200726e-06,
      "log_odds_chosen": 1.604882836341858,
      "log_odds_ratio": -0.3529296815395355,
      "logits/chosen": -1.055078148841858,
      "logits/rejected": -0.9400390386581421,
      "logps/chosen": -0.6845703125,
      "logps/rejected": -1.837499976158142,
      "loss": 0.7222,
      "nll_loss": 0.698925793170929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06844482570886612,
      "rewards/margins": 0.11536864936351776,
      "rewards/rejected": -0.18378905951976776,
      "step": 5920
    },
    {
      "epoch": 0.7652600335527164,
      "grad_norm": 2.8644281224900694,
      "learning_rate": 1.0388734331563415e-06,
      "log_odds_chosen": 1.7993652820587158,
      "log_odds_ratio": -0.3543945252895355,
      "logits/chosen": -1.053125023841858,
      "logits/rejected": -0.9330078363418579,
      "logps/chosen": -0.62158203125,
      "logps/rejected": -1.895117163658142,
      "loss": 0.7191,
      "nll_loss": 0.630175769329071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06220703199505806,
      "rewards/margins": 0.12738037109375,
      "rewards/rejected": -0.18952636420726776,
      "step": 5930
    },
    {
      "epoch": 0.7665505226480837,
      "grad_norm": 3.392049153174081,
      "learning_rate": 1.037998592215364e-06,
      "log_odds_chosen": 1.7550780773162842,
      "log_odds_ratio": -0.34941405057907104,
      "logits/chosen": -1.0128905773162842,
      "logits/rejected": -0.912890613079071,
      "logps/chosen": -0.6366211175918579,
      "logps/rejected": -1.861718773841858,
      "loss": 0.7479,
      "nll_loss": 0.7010742425918579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06375732272863388,
      "rewards/margins": 0.12249755859375,
      "rewards/rejected": -0.18625488877296448,
      "step": 5940
    },
    {
      "epoch": 0.7678410117434508,
      "grad_norm": 3.319121255543529,
      "learning_rate": 1.037125957683463e-06,
      "log_odds_chosen": 1.4994628429412842,
      "log_odds_ratio": -0.3983398377895355,
      "logits/chosen": -1.110742211341858,
      "logits/rejected": -1.01953125,
      "logps/chosen": -0.654296875,
      "logps/rejected": -1.6955077648162842,
      "loss": 0.6963,
      "nll_loss": 0.638964831829071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06544189155101776,
      "rewards/margins": 0.10404662787914276,
      "rewards/rejected": -0.16960449516773224,
      "step": 5950
    },
    {
      "epoch": 0.7691315008388179,
      "grad_norm": 3.187100729428803,
      "learning_rate": 1.0362555203016794e-06,
      "log_odds_chosen": 1.622656226158142,
      "log_odds_ratio": -0.38020020723342896,
      "logits/chosen": -1.070898413658142,
      "logits/rejected": -0.9779297113418579,
      "logps/chosen": -0.6631835699081421,
      "logps/rejected": -1.7960937023162842,
      "loss": 0.7085,
      "nll_loss": 0.639843761920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06638183444738388,
      "rewards/margins": 0.11322021484375,
      "rewards/rejected": -0.17954102158546448,
      "step": 5960
    },
    {
      "epoch": 0.7704219899341851,
      "grad_norm": 2.8399278727218364,
      "learning_rate": 1.035387270865359e-06,
      "log_odds_chosen": 1.778906226158142,
      "log_odds_ratio": -0.34736329317092896,
      "logits/chosen": -1.068750023841858,
      "logits/rejected": -0.930859386920929,
      "logps/chosen": -0.669628918170929,
      "logps/rejected": -1.9453125,
      "loss": 0.7306,
      "nll_loss": 0.72802734375,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06695556640625,
      "rewards/margins": 0.127716064453125,
      "rewards/rejected": -0.19453124701976776,
      "step": 5970
    },
    {
      "epoch": 0.7717124790295522,
      "grad_norm": 2.6199785418755295,
      "learning_rate": 1.0345212002237434e-06,
      "log_odds_chosen": 1.604882836341858,
      "log_odds_ratio": -0.38505858182907104,
      "logits/chosen": -1.051367163658142,
      "logits/rejected": -0.9384765625,
      "logps/chosen": -0.683886706829071,
      "logps/rejected": -1.815039038658142,
      "loss": 0.7253,
      "nll_loss": 0.716113269329071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06837157905101776,
      "rewards/margins": 0.11282195895910263,
      "rewards/rejected": -0.18129882216453552,
      "step": 5980
    },
    {
      "epoch": 0.7730029681249193,
      "grad_norm": 3.825421703994586,
      "learning_rate": 1.0336572992795644e-06,
      "log_odds_chosen": 1.720703125,
      "log_odds_ratio": -0.36376953125,
      "logits/chosen": -1.0724608898162842,
      "logits/rejected": -0.9595702886581421,
      "logps/chosen": -0.628222644329071,
      "logps/rejected": -1.820703148841858,
      "loss": 0.7243,
      "nll_loss": 0.6485351324081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06278076022863388,
      "rewards/margins": 0.119384765625,
      "rewards/rejected": -0.18222656846046448,
      "step": 5990
    },
    {
      "epoch": 0.7742934572202865,
      "grad_norm": 3.9464816148722104,
      "learning_rate": 1.0327955589886444e-06,
      "log_odds_chosen": 1.773828148841858,
      "log_odds_ratio": -0.3427490293979645,
      "logits/chosen": -1.111914038658142,
      "logits/rejected": -0.9990234375,
      "logps/chosen": -0.6646484136581421,
      "logps/rejected": -1.91015625,
      "loss": 0.7285,
      "nll_loss": 0.7178710699081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06643066555261612,
      "rewards/margins": 0.12451171875,
      "rewards/rejected": -0.19111327826976776,
      "step": 6000
    },
    {
      "epoch": 0.7755839463156536,
      "grad_norm": 3.562687023105448,
      "learning_rate": 1.0319359703594971e-06,
      "log_odds_chosen": 1.9665038585662842,
      "log_odds_ratio": -0.3184570372104645,
      "logits/chosen": -1.1150391101837158,
      "logits/rejected": -0.9712890386581421,
      "logps/chosen": -0.630078136920929,
      "logps/rejected": -1.986328125,
      "loss": 0.7036,
      "nll_loss": 0.7333008050918579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06301269680261612,
      "rewards/margins": 0.13568726181983948,
      "rewards/rejected": -0.19895020127296448,
      "step": 6010
    },
    {
      "epoch": 0.7768744354110207,
      "grad_norm": 2.951593785486138,
      "learning_rate": 1.0310785244529341e-06,
      "log_odds_chosen": 1.6570312976837158,
      "log_odds_ratio": -0.37089842557907104,
      "logits/chosen": -1.1326172351837158,
      "logits/rejected": -0.9720703363418579,
      "logps/chosen": -0.6304687261581421,
      "logps/rejected": -1.7560546398162842,
      "loss": 0.7168,
      "nll_loss": 0.636425793170929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06304931640625,
      "rewards/margins": 0.11245117336511612,
      "rewards/rejected": -0.17546387016773224,
      "step": 6020
    },
    {
      "epoch": 0.778164924506388,
      "grad_norm": 3.5350560181472517,
      "learning_rate": 1.0302232123816746e-06,
      "log_odds_chosen": 1.712304711341858,
      "log_odds_ratio": -0.3182128965854645,
      "logits/chosen": -1.095703125,
      "logits/rejected": -0.9652343988418579,
      "logps/chosen": -0.61962890625,
      "logps/rejected": -1.8037109375,
      "loss": 0.718,
      "nll_loss": 0.632128894329071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06191406399011612,
      "rewards/margins": 0.11845703423023224,
      "rewards/rejected": -0.18037109076976776,
      "step": 6030
    },
    {
      "epoch": 0.7794554136017551,
      "grad_norm": 3.3060002466880714,
      "learning_rate": 1.0293700253099576e-06,
      "log_odds_chosen": 1.899023413658142,
      "log_odds_ratio": -0.3123535215854645,
      "logits/chosen": -1.1162109375,
      "logits/rejected": -0.9466797113418579,
      "logps/chosen": -0.638867199420929,
      "logps/rejected": -1.9953124523162842,
      "loss": 0.7167,
      "nll_loss": 0.677050769329071,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06390380859375,
      "rewards/margins": 0.1353759765625,
      "rewards/rejected": -0.19926758110523224,
      "step": 6040
    },
    {
      "epoch": 0.7807459026971222,
      "grad_norm": 3.1603305769278616,
      "learning_rate": 1.02851895445316e-06,
      "log_odds_chosen": 1.530175805091858,
      "log_odds_ratio": -0.43486326932907104,
      "logits/chosen": -1.056640625,
      "logits/rejected": -0.9781249761581421,
      "logps/chosen": -0.6834961175918579,
      "logps/rejected": -1.783789038658142,
      "loss": 0.7193,
      "nll_loss": 0.6685546636581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.068359375,
      "rewards/margins": 0.11005248874425888,
      "rewards/rejected": -0.17841796576976776,
      "step": 6050
    },
    {
      "epoch": 0.7820363917924894,
      "grad_norm": 3.035670952181801,
      "learning_rate": 1.0276699910774159e-06,
      "log_odds_chosen": 1.8585937023162842,
      "log_odds_ratio": -0.328369140625,
      "logits/chosen": -1.0578124523162842,
      "logits/rejected": -0.9339843988418579,
      "logps/chosen": -0.6380859613418579,
      "logps/rejected": -1.9695312976837158,
      "loss": 0.7204,
      "nll_loss": 0.6499999761581421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06380615383386612,
      "rewards/margins": 0.13325195014476776,
      "rewards/rejected": -0.19692382216453552,
      "step": 6060
    },
    {
      "epoch": 0.7833268808878565,
      "grad_norm": 3.3186113789561027,
      "learning_rate": 1.0268231264992398e-06,
      "log_odds_chosen": 1.954687476158142,
      "log_odds_ratio": -0.3033447265625,
      "logits/chosen": -1.0539062023162842,
      "logits/rejected": -0.947265625,
      "logps/chosen": -0.691601574420929,
      "logps/rejected": -2.091015577316284,
      "loss": 0.7338,
      "nll_loss": 0.7430664300918579,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06925048679113388,
      "rewards/margins": 0.13984374701976776,
      "rewards/rejected": -0.20908203721046448,
      "step": 6070
    },
    {
      "epoch": 0.7846173699832236,
      "grad_norm": 3.3134286260312025,
      "learning_rate": 1.0259783520851542e-06,
      "log_odds_chosen": 1.8019530773162842,
      "log_odds_ratio": -0.3388671875,
      "logits/chosen": -1.069726586341858,
      "logits/rejected": -0.9736328125,
      "logps/chosen": -0.6429687738418579,
      "logps/rejected": -1.9171874523162842,
      "loss": 0.747,
      "nll_loss": 0.752734363079071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06427001953125,
      "rewards/margins": 0.12736205756664276,
      "rewards/rejected": -0.19162598252296448,
      "step": 6080
    },
    {
      "epoch": 0.7859078590785907,
      "grad_norm": 3.617256692232442,
      "learning_rate": 1.0251356592513193e-06,
      "log_odds_chosen": 2.0826172828674316,
      "log_odds_ratio": -0.26835936307907104,
      "logits/chosen": -1.1357421875,
      "logits/rejected": -0.962695300579071,
      "logps/chosen": -0.587597668170929,
      "logps/rejected": -2.049609422683716,
      "loss": 0.7088,
      "nll_loss": 0.646289050579071,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05878906324505806,
      "rewards/margins": 0.14625243842601776,
      "rewards/rejected": -0.20515136420726776,
      "step": 6090
    },
    {
      "epoch": 0.787198348173958,
      "grad_norm": 3.0825001103033975,
      "learning_rate": 1.0242950394631678e-06,
      "log_odds_chosen": 1.7628905773162842,
      "log_odds_ratio": -0.3384765684604645,
      "logits/chosen": -1.108007788658142,
      "logits/rejected": -0.971484363079071,
      "logps/chosen": -0.651074230670929,
      "logps/rejected": -1.888671875,
      "loss": 0.7173,
      "nll_loss": 0.648730456829071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06513671576976776,
      "rewards/margins": 0.12371826171875,
      "rewards/rejected": -0.18881836533546448,
      "step": 6100
    },
    {
      "epoch": 0.7884888372693251,
      "grad_norm": 3.345519586017487,
      "learning_rate": 1.0234564842350404e-06,
      "log_odds_chosen": 1.655859351158142,
      "log_odds_ratio": -0.34624022245407104,
      "logits/chosen": -1.1193358898162842,
      "logits/rejected": -0.986328125,
      "logps/chosen": -0.634472668170929,
      "logps/rejected": -1.7683594226837158,
      "loss": 0.7113,
      "nll_loss": 0.6591796875,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06337890774011612,
      "rewards/margins": 0.11346435546875,
      "rewards/rejected": -0.17670898139476776,
      "step": 6110
    },
    {
      "epoch": 0.7897793263646922,
      "grad_norm": 3.426462060978761,
      "learning_rate": 1.0226199851298272e-06,
      "log_odds_chosen": 1.8566405773162842,
      "log_odds_ratio": -0.302001953125,
      "logits/chosen": -1.1111328601837158,
      "logits/rejected": -0.973828136920929,
      "logps/chosen": -0.6143554449081421,
      "logps/rejected": -1.9054687023162842,
      "loss": 0.6986,
      "nll_loss": 0.6669921875,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06143798679113388,
      "rewards/margins": 0.12917479872703552,
      "rewards/rejected": -0.19077149033546448,
      "step": 6120
    },
    {
      "epoch": 0.7910698154600594,
      "grad_norm": 3.193070688697007,
      "learning_rate": 1.0217855337586106e-06,
      "log_odds_chosen": 2.004687547683716,
      "log_odds_ratio": -0.28608399629592896,
      "logits/chosen": -1.0886719226837158,
      "logits/rejected": -0.9691406488418579,
      "logps/chosen": -0.623828113079071,
      "logps/rejected": -2.026562452316284,
      "loss": 0.6958,
      "nll_loss": 0.7030273675918579,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06229247897863388,
      "rewards/margins": 0.14020995795726776,
      "rewards/rejected": -0.20253905653953552,
      "step": 6130
    },
    {
      "epoch": 0.7923603045554265,
      "grad_norm": 2.629224439384389,
      "learning_rate": 1.0209531217803119e-06,
      "log_odds_chosen": 1.8927733898162842,
      "log_odds_ratio": -0.3099609315395355,
      "logits/chosen": -1.1052734851837158,
      "logits/rejected": -0.951367199420929,
      "logps/chosen": -0.63134765625,
      "logps/rejected": -1.979882836341858,
      "loss": 0.7054,
      "nll_loss": 0.669140636920929,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06308593600988388,
      "rewards/margins": 0.13492432236671448,
      "rewards/rejected": -0.19804687798023224,
      "step": 6140
    },
    {
      "epoch": 0.7936507936507936,
      "grad_norm": 3.8516009074738466,
      "learning_rate": 1.0201227409013412e-06,
      "log_odds_chosen": 1.819726586341858,
      "log_odds_ratio": -0.32792967557907104,
      "logits/chosen": -1.069726586341858,
      "logits/rejected": -0.9488281011581421,
      "logps/chosen": -0.6749023199081421,
      "logps/rejected": -2.002734422683716,
      "loss": 0.7046,
      "nll_loss": 0.691210925579071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06755371391773224,
      "rewards/margins": 0.13264159858226776,
      "rewards/rejected": -0.20009765028953552,
      "step": 6150
    },
    {
      "epoch": 0.7949412827461608,
      "grad_norm": 3.9336650420621146,
      "learning_rate": 1.0192943828752509e-06,
      "log_odds_chosen": 1.720703125,
      "log_odds_ratio": -0.36127930879592896,
      "logits/chosen": -1.1173827648162842,
      "logits/rejected": -0.971875011920929,
      "logps/chosen": -0.6309570074081421,
      "logps/rejected": -1.82421875,
      "loss": 0.7125,
      "nll_loss": 0.659863293170929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.0631103515625,
      "rewards/margins": 0.1192626953125,
      "rewards/rejected": -0.18227538466453552,
      "step": 6160
    },
    {
      "epoch": 0.7962317718415279,
      "grad_norm": 2.957166972123149,
      "learning_rate": 1.0184680395023912e-06,
      "log_odds_chosen": 2.052929639816284,
      "log_odds_ratio": -0.2884277403354645,
      "logits/chosen": -1.112695336341858,
      "logits/rejected": -0.9830077886581421,
      "logps/chosen": -0.6078125238418579,
      "logps/rejected": -2.0464844703674316,
      "loss": 0.703,
      "nll_loss": 0.695117175579071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.060791015625,
      "rewards/margins": 0.14375610649585724,
      "rewards/rejected": -0.20476074516773224,
      "step": 6170
    },
    {
      "epoch": 0.797522260936895,
      "grad_norm": 2.9294936254528214,
      "learning_rate": 1.0176437026295688e-06,
      "log_odds_chosen": 1.5285155773162842,
      "log_odds_ratio": -0.37055665254592896,
      "logits/chosen": -1.11328125,
      "logits/rejected": -0.9775390625,
      "logps/chosen": -0.6664062738418579,
      "logps/rejected": -1.750390648841858,
      "loss": 0.7289,
      "nll_loss": 0.6841796636581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06668701022863388,
      "rewards/margins": 0.10834960639476776,
      "rewards/rejected": -0.17499999701976776,
      "step": 6180
    },
    {
      "epoch": 0.7988127500322623,
      "grad_norm": 2.8958642382458173,
      "learning_rate": 1.0168213641497094e-06,
      "log_odds_chosen": 1.7605469226837158,
      "log_odds_ratio": -0.3466552793979645,
      "logits/chosen": -1.108007788658142,
      "logits/rejected": -0.9580078125,
      "logps/chosen": -0.6904296875,
      "logps/rejected": -2.0023436546325684,
      "loss": 0.7,
      "nll_loss": 0.6751953363418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06899414211511612,
      "rewards/margins": 0.13124999403953552,
      "rewards/rejected": -0.2003173828125,
      "step": 6190
    },
    {
      "epoch": 0.8001032391276294,
      "grad_norm": 3.0356649159367177,
      "learning_rate": 1.016001016001524e-06,
      "log_odds_chosen": 1.750585913658142,
      "log_odds_ratio": -0.3458496034145355,
      "logits/chosen": -1.111328125,
      "logits/rejected": -0.9658203125,
      "logps/chosen": -0.68310546875,
      "logps/rejected": -1.937109351158142,
      "loss": 0.7229,
      "nll_loss": 0.660937488079071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06821288913488388,
      "rewards/margins": 0.12559814751148224,
      "rewards/rejected": -0.19389648735523224,
      "step": 6200
    },
    {
      "epoch": 0.8013937282229965,
      "grad_norm": 2.6486305055448276,
      "learning_rate": 1.0151826501691747e-06,
      "log_odds_chosen": 1.624609351158142,
      "log_odds_ratio": -0.34208983182907104,
      "logits/chosen": -1.1193358898162842,
      "logits/rejected": -0.976757824420929,
      "logps/chosen": -0.6849609613418579,
      "logps/rejected": -1.806249976158142,
      "loss": 0.696,
      "nll_loss": 0.623339831829071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06850586086511612,
      "rewards/margins": 0.11225585639476776,
      "rewards/rejected": -0.18071289360523224,
      "step": 6210
    },
    {
      "epoch": 0.8026842173183637,
      "grad_norm": 3.4371386222749347,
      "learning_rate": 1.0143662586819475e-06,
      "log_odds_chosen": 1.892968773841858,
      "log_odds_ratio": -0.3365234434604645,
      "logits/chosen": -1.0763671398162842,
      "logits/rejected": -0.949023425579071,
      "logps/chosen": -0.6551758050918579,
      "logps/rejected": -2.001953125,
      "loss": 0.7138,
      "nll_loss": 0.6937500238418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06547851860523224,
      "rewards/margins": 0.13482666015625,
      "rewards/rejected": -0.20024414360523224,
      "step": 6220
    },
    {
      "epoch": 0.8039747064137308,
      "grad_norm": 3.199309619561159,
      "learning_rate": 1.0135518336139257e-06,
      "log_odds_chosen": 1.7546875476837158,
      "log_odds_ratio": -0.34345704317092896,
      "logits/chosen": -1.1064453125,
      "logits/rejected": -1.0060546398162842,
      "logps/chosen": -0.620410144329071,
      "logps/rejected": -1.8542969226837158,
      "loss": 0.7195,
      "nll_loss": 0.6494140625,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06207275390625,
      "rewards/margins": 0.12333984673023224,
      "rewards/rejected": -0.18552246689796448,
      "step": 6230
    },
    {
      "epoch": 0.8052651955090979,
      "grad_norm": 2.9830728871551155,
      "learning_rate": 1.0127393670836666e-06,
      "log_odds_chosen": 1.882226586341858,
      "log_odds_ratio": -0.3099609315395355,
      "logits/chosen": -1.0900390148162842,
      "logits/rejected": -0.9564453363418579,
      "logps/chosen": -0.633007824420929,
      "logps/rejected": -1.9951171875,
      "loss": 0.7297,
      "nll_loss": 0.722363293170929,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06333007663488388,
      "rewards/margins": 0.13613280653953552,
      "rewards/rejected": -0.19953612983226776,
      "step": 6240
    },
    {
      "epoch": 0.8065556846044651,
      "grad_norm": 3.0571249058546224,
      "learning_rate": 1.0119288512538813e-06,
      "log_odds_chosen": 1.409082055091858,
      "log_odds_ratio": -0.40961915254592896,
      "logits/chosen": -1.0695312023162842,
      "logits/rejected": -0.9814453125,
      "logps/chosen": -0.6810547113418579,
      "logps/rejected": -1.6544921398162842,
      "loss": 0.7115,
      "nll_loss": 0.687792956829071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06810303032398224,
      "rewards/margins": 0.09751281887292862,
      "rewards/rejected": -0.1656494140625,
      "step": 6250
    },
    {
      "epoch": 0.8078461736998322,
      "grad_norm": 4.1693590495984365,
      "learning_rate": 1.0111202783311173e-06,
      "log_odds_chosen": 1.709375023841858,
      "log_odds_ratio": -0.361083984375,
      "logits/chosen": -1.0927734375,
      "logits/rejected": -0.943359375,
      "logps/chosen": -0.642871081829071,
      "logps/rejected": -1.7980468273162842,
      "loss": 0.7254,
      "nll_loss": 0.6836913824081421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06429443508386612,
      "rewards/margins": 0.11549071967601776,
      "rewards/rejected": -0.17961426079273224,
      "step": 6260
    },
    {
      "epoch": 0.8091366627951994,
      "grad_norm": 3.23914621558742,
      "learning_rate": 1.010313640565443e-06,
      "log_odds_chosen": 1.524999976158142,
      "log_odds_ratio": -0.363525390625,
      "logits/chosen": -1.115234375,
      "logits/rejected": -0.958203136920929,
      "logps/chosen": -0.64599609375,
      "logps/rejected": -1.6990234851837158,
      "loss": 0.7062,
      "nll_loss": 0.6961914300918579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.064697265625,
      "rewards/margins": 0.10520019382238388,
      "rewards/rejected": -0.169921875,
      "step": 6270
    },
    {
      "epoch": 0.8104271518905666,
      "grad_norm": 3.882171450135503,
      "learning_rate": 1.0095089302501373e-06,
      "log_odds_chosen": 1.7810547351837158,
      "log_odds_ratio": -0.3147216737270355,
      "logits/chosen": -1.083398461341858,
      "logits/rejected": -0.9789062738418579,
      "logps/chosen": -0.62841796875,
      "logps/rejected": -1.844335913658142,
      "loss": 0.6854,
      "nll_loss": 0.683398425579071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06289062649011612,
      "rewards/margins": 0.12165527045726776,
      "rewards/rejected": -0.1846923828125,
      "step": 6280
    },
    {
      "epoch": 0.8117176409859337,
      "grad_norm": 4.000946323414211,
      "learning_rate": 1.0087061397213787e-06,
      "log_odds_chosen": 1.718847632408142,
      "log_odds_ratio": -0.34467774629592896,
      "logits/chosen": -1.080664038658142,
      "logits/rejected": -0.9945312738418579,
      "logps/chosen": -0.64501953125,
      "logps/rejected": -1.79296875,
      "loss": 0.7007,
      "nll_loss": 0.655957043170929,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06452636420726776,
      "rewards/margins": 0.11481933295726776,
      "rewards/rejected": -0.17927245795726776,
      "step": 6290
    },
    {
      "epoch": 0.8130081300813008,
      "grad_norm": 3.248595153466242,
      "learning_rate": 1.007905261357939e-06,
      "log_odds_chosen": 1.513085961341858,
      "log_odds_ratio": -0.3797363340854645,
      "logits/chosen": -1.0828125476837158,
      "logits/rejected": -0.9544922113418579,
      "logps/chosen": -0.68017578125,
      "logps/rejected": -1.7449219226837158,
      "loss": 0.7202,
      "nll_loss": 0.726757824420929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06802978366613388,
      "rewards/margins": 0.10655517876148224,
      "rewards/rejected": -0.17446288466453552,
      "step": 6300
    },
    {
      "epoch": 0.8142986191766679,
      "grad_norm": 3.4301028144856294,
      "learning_rate": 1.0071062875808811e-06,
      "log_odds_chosen": 1.5554687976837158,
      "log_odds_ratio": -0.3907226622104645,
      "logits/chosen": -1.104882836341858,
      "logits/rejected": -0.9857422113418579,
      "logps/chosen": -0.6585937738418579,
      "logps/rejected": -1.7199218273162842,
      "loss": 0.6967,
      "nll_loss": 0.688281238079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0657958984375,
      "rewards/margins": 0.10616455227136612,
      "rewards/rejected": -0.1719970703125,
      "step": 6310
    },
    {
      "epoch": 0.8155891082720351,
      "grad_norm": 3.059708363591424,
      "learning_rate": 1.0063092108532552e-06,
      "log_odds_chosen": 1.568359375,
      "log_odds_ratio": -0.36474609375,
      "logits/chosen": -1.075585961341858,
      "logits/rejected": -0.9878906011581421,
      "logps/chosen": -0.634765625,
      "logps/rejected": -1.740625023841858,
      "loss": 0.6859,
      "nll_loss": 0.634082019329071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06351318210363388,
      "rewards/margins": 0.110595703125,
      "rewards/rejected": -0.1741943359375,
      "step": 6320
    },
    {
      "epoch": 0.8168795973674022,
      "grad_norm": 3.6224086693313717,
      "learning_rate": 1.005514023679802e-06,
      "log_odds_chosen": 1.844335913658142,
      "log_odds_ratio": -0.32475584745407104,
      "logits/chosen": -1.067968726158142,
      "logits/rejected": -0.9583984613418579,
      "logps/chosen": -0.59521484375,
      "logps/rejected": -1.8761718273162842,
      "loss": 0.6932,
      "nll_loss": 0.642871081829071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05954589694738388,
      "rewards/margins": 0.12816162407398224,
      "rewards/rejected": -0.18769530951976776,
      "step": 6330
    },
    {
      "epoch": 0.8181700864627693,
      "grad_norm": 3.0964185183999806,
      "learning_rate": 1.0047207186066567e-06,
      "log_odds_chosen": 1.767968773841858,
      "log_odds_ratio": -0.331787109375,
      "logits/chosen": -1.1142578125,
      "logits/rejected": -0.9609375,
      "logps/chosen": -0.633105456829071,
      "logps/rejected": -1.908203125,
      "loss": 0.7203,
      "nll_loss": 0.739062488079071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06333007663488388,
      "rewards/margins": 0.12745361030101776,
      "rewards/rejected": -0.19074706733226776,
      "step": 6340
    },
    {
      "epoch": 0.8194605755581366,
      "grad_norm": 3.3406638472894703,
      "learning_rate": 1.0039292882210538e-06,
      "log_odds_chosen": 1.5343749523162842,
      "log_odds_ratio": -0.3846679627895355,
      "logits/chosen": -1.0984375476837158,
      "logits/rejected": -0.984570324420929,
      "logps/chosen": -0.64501953125,
      "logps/rejected": -1.6857421398162842,
      "loss": 0.6902,
      "nll_loss": 0.62890625,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06444092094898224,
      "rewards/margins": 0.10402221977710724,
      "rewards/rejected": -0.16855469346046448,
      "step": 6350
    },
    {
      "epoch": 0.8207510646535037,
      "grad_norm": 2.945283943558219,
      "learning_rate": 1.0031397251510382e-06,
      "log_odds_chosen": 1.8195312023162842,
      "log_odds_ratio": -0.323974609375,
      "logits/chosen": -1.1134765148162842,
      "logits/rejected": -0.951953113079071,
      "logps/chosen": -0.6358398199081421,
      "logps/rejected": -1.895898461341858,
      "loss": 0.7178,
      "nll_loss": 0.6390625238418579,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06362304836511612,
      "rewards/margins": 0.12596435844898224,
      "rewards/rejected": -0.1895751953125,
      "step": 6360
    },
    {
      "epoch": 0.8220415537488708,
      "grad_norm": 3.0422547985021304,
      "learning_rate": 1.0023520220651762e-06,
      "log_odds_chosen": 2.025390625,
      "log_odds_ratio": -0.27366942167282104,
      "logits/chosen": -1.0871093273162842,
      "logits/rejected": -0.9742187261581421,
      "logps/chosen": -0.644824206829071,
      "logps/rejected": -2.094531297683716,
      "loss": 0.7243,
      "nll_loss": 0.6615234613418579,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.06448974460363388,
      "rewards/margins": 0.14472655951976776,
      "rewards/rejected": -0.20927734673023224,
      "step": 6370
    },
    {
      "epoch": 0.823332042844238,
      "grad_norm": 2.922618524439534,
      "learning_rate": 1.0015661716722687e-06,
      "log_odds_chosen": 1.9119141101837158,
      "log_odds_ratio": -0.3208984434604645,
      "logits/chosen": -1.0906250476837158,
      "logits/rejected": -0.939257800579071,
      "logps/chosen": -0.636914074420929,
      "logps/rejected": -2.021484375,
      "loss": 0.7028,
      "nll_loss": 0.683300793170929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06368408352136612,
      "rewards/margins": 0.13837890326976776,
      "rewards/rejected": -0.20217284560203552,
      "step": 6380
    },
    {
      "epoch": 0.8246225319396051,
      "grad_norm": 3.7690996841703197,
      "learning_rate": 1.0007821667210687e-06,
      "log_odds_chosen": 2.048046827316284,
      "log_odds_ratio": -0.2989746034145355,
      "logits/chosen": -1.136132836341858,
      "logits/rejected": -0.958789050579071,
      "logps/chosen": -0.6373046636581421,
      "logps/rejected": -2.08984375,
      "loss": 0.7121,
      "nll_loss": 0.665820300579071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.063720703125,
      "rewards/margins": 0.1453857421875,
      "rewards/rejected": -0.20903320610523224,
      "step": 6390
    },
    {
      "epoch": 0.8259130210349722,
      "grad_norm": 3.2137280442450478,
      "learning_rate": 1e-06,
      "log_odds_chosen": 1.802734375,
      "log_odds_ratio": -0.36444091796875,
      "logits/chosen": -1.029296875,
      "logits/rejected": -0.8941406011581421,
      "logps/chosen": -0.704394519329071,
      "logps/rejected": -2.0335936546325684,
      "loss": 0.7449,
      "nll_loss": 0.7713867425918579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07041015475988388,
      "rewards/margins": 0.13297119736671448,
      "rewards/rejected": -0.20339354872703552,
      "step": 6400
    },
    {
      "epoch": 0.8272035101303394,
      "grad_norm": 4.482973438900458,
      "learning_rate": 9.992196643368784e-07,
      "log_odds_chosen": 1.792578101158142,
      "log_odds_ratio": -0.3377929627895355,
      "logits/chosen": -1.083593726158142,
      "logits/rejected": -0.993359386920929,
      "logps/chosen": -0.6548827886581421,
      "logps/rejected": -1.863671898841858,
      "loss": 0.691,
      "nll_loss": 0.6415039300918579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.065521240234375,
      "rewards/margins": 0.12075195461511612,
      "rewards/rejected": -0.18632812798023224,
      "step": 6410
    },
    {
      "epoch": 0.8284939992257065,
      "grad_norm": 3.2346183647343745,
      "learning_rate": 9.984411525986355e-07,
      "log_odds_chosen": 1.460058569908142,
      "log_odds_ratio": -0.396240234375,
      "logits/chosen": -1.089453101158142,
      "logits/rejected": -0.984179675579071,
      "logps/chosen": -0.67431640625,
      "logps/rejected": -1.6863281726837158,
      "loss": 0.7083,
      "nll_loss": 0.6703125238418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0673828125,
      "rewards/margins": 0.10109863430261612,
      "rewards/rejected": -0.16860350966453552,
      "step": 6420
    },
    {
      "epoch": 0.8297844883210737,
      "grad_norm": 3.144796418316069,
      "learning_rate": 9.97664457691046e-07,
      "log_odds_chosen": 1.7244141101837158,
      "log_odds_ratio": -0.33305662870407104,
      "logits/chosen": -1.0789062976837158,
      "logits/rejected": -0.9488281011581421,
      "logps/chosen": -0.66015625,
      "logps/rejected": -1.836328148841858,
      "loss": 0.6985,
      "nll_loss": 0.6893554925918579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06601562350988388,
      "rewards/margins": 0.11771240085363388,
      "rewards/rejected": -0.1837158203125,
      "step": 6430
    },
    {
      "epoch": 0.8310749774164409,
      "grad_norm": 3.5346079791516485,
      "learning_rate": 9.968895725584535e-07,
      "log_odds_chosen": 1.494726538658142,
      "log_odds_ratio": -0.403564453125,
      "logits/chosen": -1.0867187976837158,
      "logits/rejected": -0.9966796636581421,
      "logps/chosen": -0.6988281011581421,
      "logps/rejected": -1.6964843273162842,
      "loss": 0.7264,
      "nll_loss": 0.7381836175918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06987304985523224,
      "rewards/margins": 0.09990844875574112,
      "rewards/rejected": -0.16970214247703552,
      "step": 6440
    },
    {
      "epoch": 0.832365466511808,
      "grad_norm": 3.3732281188456072,
      "learning_rate": 9.961164901835046e-07,
      "log_odds_chosen": 1.517968773841858,
      "log_odds_ratio": -0.39960938692092896,
      "logits/chosen": -1.06640625,
      "logits/rejected": -0.947265625,
      "logps/chosen": -0.721875011920929,
      "logps/rejected": -1.821679711341858,
      "loss": 0.7349,
      "nll_loss": 0.6981445550918579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07219238579273224,
      "rewards/margins": 0.11008300632238388,
      "rewards/rejected": -0.18220214545726776,
      "step": 6450
    },
    {
      "epoch": 0.8336559556071751,
      "grad_norm": 3.166494885793705,
      "learning_rate": 9.95345203586879e-07,
      "log_odds_chosen": 1.7844727039337158,
      "log_odds_ratio": -0.32587891817092896,
      "logits/chosen": -1.1232421398162842,
      "logits/rejected": -0.978710949420929,
      "logps/chosen": -0.654492199420929,
      "logps/rejected": -1.898828148841858,
      "loss": 0.7184,
      "nll_loss": 0.675097644329071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06546630710363388,
      "rewards/margins": 0.12455444037914276,
      "rewards/rejected": -0.19013671576976776,
      "step": 6460
    },
    {
      "epoch": 0.8349464447025423,
      "grad_norm": 3.278505032623662,
      "learning_rate": 9.94575705827027e-07,
      "log_odds_chosen": 1.627832055091858,
      "log_odds_ratio": -0.37089842557907104,
      "logits/chosen": -1.10546875,
      "logits/rejected": -0.915234386920929,
      "logps/chosen": -0.666210949420929,
      "logps/rejected": -1.8468749523162842,
      "loss": 0.7167,
      "nll_loss": 0.682812511920929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.066650390625,
      "rewards/margins": 0.11793823540210724,
      "rewards/rejected": -0.18454590439796448,
      "step": 6470
    },
    {
      "epoch": 0.8362369337979094,
      "grad_norm": 3.202787126161141,
      "learning_rate": 9.938079899999065e-07,
      "log_odds_chosen": 1.778906226158142,
      "log_odds_ratio": -0.3633789122104645,
      "logits/chosen": -1.105078101158142,
      "logits/rejected": -0.9984375238418579,
      "logps/chosen": -0.633593738079071,
      "logps/rejected": -1.9191405773162842,
      "loss": 0.6991,
      "nll_loss": 0.679492175579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0633544921875,
      "rewards/margins": 0.12861327826976776,
      "rewards/rejected": -0.19206543266773224,
      "step": 6480
    },
    {
      "epoch": 0.8375274228932765,
      "grad_norm": 3.4827600809470125,
      "learning_rate": 9.930420492387219e-07,
      "log_odds_chosen": 1.7576172351837158,
      "log_odds_ratio": -0.3367675840854645,
      "logits/chosen": -1.1232421398162842,
      "logits/rejected": -0.990429699420929,
      "logps/chosen": -0.683789074420929,
      "logps/rejected": -1.9265625476837158,
      "loss": 0.7124,
      "nll_loss": 0.6563476324081421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06838379055261612,
      "rewards/margins": 0.12409667670726776,
      "rewards/rejected": -0.19255371391773224,
      "step": 6490
    },
    {
      "epoch": 0.8388179119886437,
      "grad_norm": 3.1163611457522036,
      "learning_rate": 9.922778767136676e-07,
      "log_odds_chosen": 1.5587890148162842,
      "log_odds_ratio": -0.39677733182907104,
      "logits/chosen": -1.119140625,
      "logits/rejected": -0.9886718988418579,
      "logps/chosen": -0.71630859375,
      "logps/rejected": -1.853906273841858,
      "loss": 0.6959,
      "nll_loss": 0.633105456829071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07158203423023224,
      "rewards/margins": 0.11376953125,
      "rewards/rejected": -0.18532714247703552,
      "step": 6500
    },
    {
      "epoch": 0.8401084010840109,
      "grad_norm": 2.8651917521750474,
      "learning_rate": 9.915154656316713e-07,
      "log_odds_chosen": 1.8230469226837158,
      "log_odds_ratio": -0.3362060487270355,
      "logits/chosen": -1.091406226158142,
      "logits/rejected": -0.946484386920929,
      "logps/chosen": -0.65087890625,
      "logps/rejected": -1.955468773841858,
      "loss": 0.6985,
      "nll_loss": 0.7323242425918579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0650634765625,
      "rewards/margins": 0.13055419921875,
      "rewards/rejected": -0.19560547173023224,
      "step": 6510
    },
    {
      "epoch": 0.841398890179378,
      "grad_norm": 2.8607861687320373,
      "learning_rate": 9.907548092361398e-07,
      "log_odds_chosen": 1.835546851158142,
      "log_odds_ratio": -0.33076173067092896,
      "logits/chosen": -1.0574219226837158,
      "logits/rejected": -0.927539050579071,
      "logps/chosen": -0.626171886920929,
      "logps/rejected": -1.9148437976837158,
      "loss": 0.6908,
      "nll_loss": 0.6937500238418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06251220405101776,
      "rewards/margins": 0.1290283203125,
      "rewards/rejected": -0.19167479872703552,
      "step": 6520
    },
    {
      "epoch": 0.8426893792747451,
      "grad_norm": 2.9361729716527165,
      "learning_rate": 9.899959008067097e-07,
      "log_odds_chosen": 1.8859374523162842,
      "log_odds_ratio": -0.33491212129592896,
      "logits/chosen": -1.0740234851837158,
      "logits/rejected": -0.9486328363418579,
      "logps/chosen": -0.603515625,
      "logps/rejected": -1.9187500476837158,
      "loss": 0.6869,
      "nll_loss": 0.6800781488418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06038818508386612,
      "rewards/margins": 0.13154296576976776,
      "rewards/rejected": -0.1920166015625,
      "step": 6530
    },
    {
      "epoch": 0.8439798683701123,
      "grad_norm": 3.2249144335377933,
      "learning_rate": 9.892387336589959e-07,
      "log_odds_chosen": 1.864843726158142,
      "log_odds_ratio": -0.3332275450229645,
      "logits/chosen": -1.1427733898162842,
      "logits/rejected": -1.0027344226837158,
      "logps/chosen": -0.6689453125,
      "logps/rejected": -2.033203125,
      "loss": 0.6889,
      "nll_loss": 0.6668945550918579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06689453125,
      "rewards/margins": 0.13640137016773224,
      "rewards/rejected": -0.20322266221046448,
      "step": 6540
    },
    {
      "epoch": 0.8452703574654794,
      "grad_norm": 2.988712388322602,
      "learning_rate": 9.884833011443446e-07,
      "log_odds_chosen": 1.743749976158142,
      "log_odds_ratio": -0.35468751192092896,
      "logits/chosen": -1.075585961341858,
      "logits/rejected": -0.948437511920929,
      "logps/chosen": -0.63037109375,
      "logps/rejected": -1.8058593273162842,
      "loss": 0.6751,
      "nll_loss": 0.6083008050918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06308593600988388,
      "rewards/margins": 0.1175537109375,
      "rewards/rejected": -0.1805419921875,
      "step": 6550
    },
    {
      "epoch": 0.8465608465608465,
      "grad_norm": 4.191432928558559,
      "learning_rate": 9.877295966495897e-07,
      "log_odds_chosen": 1.805273413658142,
      "log_odds_ratio": -0.33820801973342896,
      "logits/chosen": -1.068750023841858,
      "logits/rejected": -0.9585937261581421,
      "logps/chosen": -0.6727539300918579,
      "logps/rejected": -1.941796898841858,
      "loss": 0.7181,
      "nll_loss": 0.661914050579071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06722412258386612,
      "rewards/margins": 0.12689208984375,
      "rewards/rejected": -0.19414062798023224,
      "step": 6560
    },
    {
      "epoch": 0.8478513356562137,
      "grad_norm": 2.924953030851781,
      "learning_rate": 9.86977613596807e-07,
      "log_odds_chosen": 1.818359375,
      "log_odds_ratio": -0.3187011778354645,
      "logits/chosen": -1.140039086341858,
      "logits/rejected": -1.0060546398162842,
      "logps/chosen": -0.621289074420929,
      "logps/rejected": -1.9226562976837158,
      "loss": 0.6918,
      "nll_loss": 0.64404296875,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0621337890625,
      "rewards/margins": 0.1302490234375,
      "rewards/rejected": -0.1925048828125,
      "step": 6570
    },
    {
      "epoch": 0.8491418247515808,
      "grad_norm": 3.044595452182508,
      "learning_rate": 9.862273454430757e-07,
      "log_odds_chosen": 1.991796851158142,
      "log_odds_ratio": -0.306396484375,
      "logits/chosen": -1.0291016101837158,
      "logits/rejected": -0.910937488079071,
      "logps/chosen": -0.615429699420929,
      "logps/rejected": -2.0179686546325684,
      "loss": 0.6965,
      "nll_loss": 0.7300781011581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06151122972369194,
      "rewards/margins": 0.14033202826976776,
      "rewards/rejected": -0.201904296875,
      "step": 6580
    },
    {
      "epoch": 0.850432313846948,
      "grad_norm": 3.193749564949259,
      "learning_rate": 9.85478785680238e-07,
      "log_odds_chosen": 1.889062523841858,
      "log_odds_ratio": -0.331787109375,
      "logits/chosen": -1.0798828601837158,
      "logits/rejected": -0.9681640863418579,
      "logps/chosen": -0.676464855670929,
      "logps/rejected": -2.076953172683716,
      "loss": 0.6902,
      "nll_loss": 0.688281238079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06761474907398224,
      "rewards/margins": 0.14002685248851776,
      "rewards/rejected": -0.20781250298023224,
      "step": 6590
    },
    {
      "epoch": 0.8517228029423152,
      "grad_norm": 3.17137056590019,
      "learning_rate": 9.847319278346618e-07,
      "log_odds_chosen": 1.525488257408142,
      "log_odds_ratio": -0.432373046875,
      "logits/chosen": -1.0578124523162842,
      "logits/rejected": -0.949023425579071,
      "logps/chosen": -0.7041991949081421,
      "logps/rejected": -1.8240234851837158,
      "loss": 0.7032,
      "nll_loss": 0.7076171636581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07038573920726776,
      "rewards/margins": 0.11200714111328125,
      "rewards/rejected": -0.18242187798023224,
      "step": 6600
    },
    {
      "epoch": 0.8530132920376823,
      "grad_norm": 3.5937024804765936,
      "learning_rate": 9.839867654670063e-07,
      "log_odds_chosen": 1.5539062023162842,
      "log_odds_ratio": -0.3813232481479645,
      "logits/chosen": -1.109960913658142,
      "logits/rejected": -0.9554687738418579,
      "logps/chosen": -0.65771484375,
      "logps/rejected": -1.748437523841858,
      "loss": 0.6937,
      "nll_loss": 0.64599609375,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06578369438648224,
      "rewards/margins": 0.10902099311351776,
      "rewards/rejected": -0.17473144829273224,
      "step": 6610
    },
    {
      "epoch": 0.8543037811330494,
      "grad_norm": 2.9116458286848865,
      "learning_rate": 9.832432921719876e-07,
      "log_odds_chosen": 1.798828125,
      "log_odds_ratio": -0.32133787870407104,
      "logits/chosen": -1.135156273841858,
      "logits/rejected": -0.983593761920929,
      "logps/chosen": -0.64599609375,
      "logps/rejected": -1.923828125,
      "loss": 0.705,
      "nll_loss": 0.6373046636581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06458739936351776,
      "rewards/margins": 0.12752684950828552,
      "rewards/rejected": -0.19233398139476776,
      "step": 6620
    },
    {
      "epoch": 0.8555942702284166,
      "grad_norm": 2.7361396173146844,
      "learning_rate": 9.825015015781493e-07,
      "log_odds_chosen": 1.3816406726837158,
      "log_odds_ratio": -0.41547852754592896,
      "logits/chosen": -1.0955078601837158,
      "logits/rejected": -0.9505859613418579,
      "logps/chosen": -0.7447265386581421,
      "logps/rejected": -1.710546851158142,
      "loss": 0.7212,
      "nll_loss": 0.700390636920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07445068657398224,
      "rewards/margins": 0.0966796875,
      "rewards/rejected": -0.17121581733226776,
      "step": 6630
    },
    {
      "epoch": 0.8568847593237837,
      "grad_norm": 3.039943284406918,
      "learning_rate": 9.81761387347632e-07,
      "log_odds_chosen": 1.865234375,
      "log_odds_ratio": -0.3152099549770355,
      "logits/chosen": -1.1181640625,
      "logits/rejected": -0.968554675579071,
      "logps/chosen": -0.625683605670929,
      "logps/rejected": -1.899999976158142,
      "loss": 0.6944,
      "nll_loss": 0.648730456829071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06260986626148224,
      "rewards/margins": 0.12745361030101776,
      "rewards/rejected": -0.1900634765625,
      "step": 6640
    },
    {
      "epoch": 0.8581752484191508,
      "grad_norm": 3.853068381669492,
      "learning_rate": 9.810229431759452e-07,
      "log_odds_chosen": 1.7744140625,
      "log_odds_ratio": -0.3348632752895355,
      "logits/chosen": -1.08984375,
      "logits/rejected": -0.953125,
      "logps/chosen": -0.666796863079071,
      "logps/rejected": -1.933203101158142,
      "loss": 0.6993,
      "nll_loss": 0.680859386920929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06672362983226776,
      "rewards/margins": 0.12666015326976776,
      "rewards/rejected": -0.19331054389476776,
      "step": 6650
    },
    {
      "epoch": 0.859465737514518,
      "grad_norm": 2.9418379092696427,
      "learning_rate": 9.802861627917437e-07,
      "log_odds_chosen": 1.7921874523162842,
      "log_odds_ratio": -0.3359375,
      "logits/chosen": -1.059179663658142,
      "logits/rejected": -0.9814453125,
      "logps/chosen": -0.6444336175918579,
      "logps/rejected": -1.9363281726837158,
      "loss": 0.6772,
      "nll_loss": 0.7061523199081421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06442870944738388,
      "rewards/margins": 0.12926025688648224,
      "rewards/rejected": -0.19370117783546448,
      "step": 6660
    },
    {
      "epoch": 0.8607562266098852,
      "grad_norm": 3.823182656097496,
      "learning_rate": 9.795510399566016e-07,
      "log_odds_chosen": 1.763085961341858,
      "log_odds_ratio": -0.349609375,
      "logits/chosen": -1.131445288658142,
      "logits/rejected": -0.9638671875,
      "logps/chosen": -0.632128894329071,
      "logps/rejected": -1.869140625,
      "loss": 0.6791,
      "nll_loss": 0.63720703125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06328125298023224,
      "rewards/margins": 0.12359619140625,
      "rewards/rejected": -0.18681640923023224,
      "step": 6670
    },
    {
      "epoch": 0.8620467157052523,
      "grad_norm": 3.219376172628775,
      "learning_rate": 9.788175684647926e-07,
      "log_odds_chosen": 1.74609375,
      "log_odds_ratio": -0.3231201171875,
      "logits/chosen": -1.11328125,
      "logits/rejected": -0.976367175579071,
      "logps/chosen": -0.6396484375,
      "logps/rejected": -1.87890625,
      "loss": 0.6737,
      "nll_loss": 0.6583007574081421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06400146335363388,
      "rewards/margins": 0.12395019829273224,
      "rewards/rejected": -0.18798828125,
      "step": 6680
    },
    {
      "epoch": 0.8633372048006195,
      "grad_norm": 3.1590140954613255,
      "learning_rate": 9.780857421430687e-07,
      "log_odds_chosen": 1.639062523841858,
      "log_odds_ratio": -0.3788818418979645,
      "logits/chosen": -1.091796875,
      "logits/rejected": -0.9798828363418579,
      "logps/chosen": -0.6963866949081421,
      "logps/rejected": -1.8917968273162842,
      "loss": 0.6839,
      "nll_loss": 0.6856445074081421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06965331733226776,
      "rewards/margins": 0.11944580078125,
      "rewards/rejected": -0.18903808295726776,
      "step": 6690
    },
    {
      "epoch": 0.8646276938959866,
      "grad_norm": 3.019463409981712,
      "learning_rate": 9.773555548504417e-07,
      "log_odds_chosen": 1.74755859375,
      "log_odds_ratio": -0.34331053495407104,
      "logits/chosen": -1.108789086341858,
      "logits/rejected": -1.006445288658142,
      "logps/chosen": -0.6148437261581421,
      "logps/rejected": -1.817968726158142,
      "loss": 0.7051,
      "nll_loss": 0.6207031011581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06151122972369194,
      "rewards/margins": 0.12043456733226776,
      "rewards/rejected": -0.181884765625,
      "step": 6700
    },
    {
      "epoch": 0.8659181829913537,
      "grad_norm": 2.932716962775448,
      "learning_rate": 9.76627000477968e-07,
      "log_odds_chosen": 1.5925781726837158,
      "log_odds_ratio": -0.4010009765625,
      "logits/chosen": -1.0714843273162842,
      "logits/rejected": -0.9632812738418579,
      "logps/chosen": -0.676562488079071,
      "logps/rejected": -1.830078125,
      "loss": 0.6887,
      "nll_loss": 0.6865234375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06765136867761612,
      "rewards/margins": 0.11536864936351776,
      "rewards/rejected": -0.18305663764476776,
      "step": 6710
    },
    {
      "epoch": 0.8672086720867209,
      "grad_norm": 3.242329345957608,
      "learning_rate": 9.75900072948533e-07,
      "log_odds_chosen": 1.5958983898162842,
      "log_odds_ratio": -0.38837891817092896,
      "logits/chosen": -1.0964844226837158,
      "logits/rejected": -0.991015613079071,
      "logps/chosen": -0.62353515625,
      "logps/rejected": -1.719140648841858,
      "loss": 0.7042,
      "nll_loss": 0.6791015863418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06235351413488388,
      "rewards/margins": 0.109619140625,
      "rewards/rejected": -0.17194823920726776,
      "step": 6720
    },
    {
      "epoch": 0.868499161182088,
      "grad_norm": 2.960159020551024,
      "learning_rate": 9.751747662166388e-07,
      "log_odds_chosen": 1.608984351158142,
      "log_odds_ratio": -0.35346680879592896,
      "logits/chosen": -1.115820288658142,
      "logits/rejected": -1.002343773841858,
      "logps/chosen": -0.617382824420929,
      "logps/rejected": -1.717187523841858,
      "loss": 0.7035,
      "nll_loss": 0.6170898675918579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06173095852136612,
      "rewards/margins": 0.10987548530101776,
      "rewards/rejected": -0.17153319716453552,
      "step": 6730
    },
    {
      "epoch": 0.8697896502774551,
      "grad_norm": 2.9843179655268517,
      "learning_rate": 9.744510742681917e-07,
      "log_odds_chosen": 1.88671875,
      "log_odds_ratio": -0.3248046934604645,
      "logits/chosen": -1.089257836341858,
      "logits/rejected": -0.9720703363418579,
      "logps/chosen": -0.620898425579071,
      "logps/rejected": -1.928125023841858,
      "loss": 0.6932,
      "nll_loss": 0.6610351800918579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06207275390625,
      "rewards/margins": 0.13063964247703552,
      "rewards/rejected": -0.19272461533546448,
      "step": 6740
    },
    {
      "epoch": 0.8710801393728222,
      "grad_norm": 2.642528963818851,
      "learning_rate": 9.737289911202953e-07,
      "log_odds_chosen": 1.5650513172149658,
      "log_odds_ratio": -0.4195800721645355,
      "logits/chosen": -1.049218773841858,
      "logits/rejected": -0.9468749761581421,
      "logps/chosen": -0.6766601800918579,
      "logps/rejected": -1.814453125,
      "loss": 0.6889,
      "nll_loss": 0.639941394329071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06765136867761612,
      "rewards/margins": 0.113983154296875,
      "rewards/rejected": -0.18159179389476776,
      "step": 6750
    },
    {
      "epoch": 0.8723706284681895,
      "grad_norm": 3.5378084273609587,
      "learning_rate": 9.730085108210398e-07,
      "log_odds_chosen": 1.6076171398162842,
      "log_odds_ratio": -0.3917480409145355,
      "logits/chosen": -1.0759766101837158,
      "logits/rejected": -0.9957031011581421,
      "logps/chosen": -0.6625000238418579,
      "logps/rejected": -1.779687523841858,
      "loss": 0.661,
      "nll_loss": 0.672070324420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06623534858226776,
      "rewards/margins": 0.11181640625,
      "rewards/rejected": -0.17800292372703552,
      "step": 6760
    },
    {
      "epoch": 0.8736611175635566,
      "grad_norm": 3.046842536608086,
      "learning_rate": 9.72289627449298e-07,
      "log_odds_chosen": 1.9285156726837158,
      "log_odds_ratio": -0.297607421875,
      "logits/chosen": -1.077539086341858,
      "logits/rejected": -0.958203136920929,
      "logps/chosen": -0.641406238079071,
      "logps/rejected": -2.062695264816284,
      "loss": 0.7231,
      "nll_loss": 0.708789050579071,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06417236477136612,
      "rewards/margins": 0.14204101264476776,
      "rewards/rejected": -0.20624999701976776,
      "step": 6770
    },
    {
      "epoch": 0.8749516066589237,
      "grad_norm": 3.711751585425996,
      "learning_rate": 9.715723351145206e-07,
      "log_odds_chosen": 1.824804663658142,
      "log_odds_ratio": -0.3541503846645355,
      "logits/chosen": -1.122656226158142,
      "logits/rejected": -0.982226550579071,
      "logps/chosen": -0.6336914300918579,
      "logps/rejected": -1.9509766101837158,
      "loss": 0.6897,
      "nll_loss": 0.6543945074081421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06337890774011612,
      "rewards/margins": 0.13159790635108948,
      "rewards/rejected": -0.19492188096046448,
      "step": 6780
    },
    {
      "epoch": 0.8762420957542909,
      "grad_norm": 3.0592406007690087,
      "learning_rate": 9.70856627956532e-07,
      "log_odds_chosen": 1.614843726158142,
      "log_odds_ratio": -0.3643554747104645,
      "logits/chosen": -1.106054663658142,
      "logits/rejected": -0.989062488079071,
      "logps/chosen": -0.686328113079071,
      "logps/rejected": -1.8425781726837158,
      "loss": 0.7118,
      "nll_loss": 0.7334960699081421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.068603515625,
      "rewards/margins": 0.11566162109375,
      "rewards/rejected": -0.18425293266773224,
      "step": 6790
    },
    {
      "epoch": 0.877532584849658,
      "grad_norm": 3.3145837133137186,
      "learning_rate": 9.701425001453318e-07,
      "log_odds_chosen": 1.927343726158142,
      "log_odds_ratio": -0.3062500059604645,
      "logits/chosen": -1.0988280773162842,
      "logits/rejected": -0.9703124761581421,
      "logps/chosen": -0.6298828125,
      "logps/rejected": -2.0042967796325684,
      "loss": 0.6956,
      "nll_loss": 0.69189453125,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06298828125,
      "rewards/margins": 0.13751220703125,
      "rewards/rejected": -0.20041504502296448,
      "step": 6800
    },
    {
      "epoch": 0.8788230739450251,
      "grad_norm": 3.0706724197005304,
      "learning_rate": 9.694299458808932e-07,
      "log_odds_chosen": 1.5978515148162842,
      "log_odds_ratio": -0.3787597715854645,
      "logits/chosen": -1.1396484375,
      "logits/rejected": -1.033203125,
      "logps/chosen": -0.690625011920929,
      "logps/rejected": -1.843164086341858,
      "loss": 0.6894,
      "nll_loss": 0.68408203125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06905517727136612,
      "rewards/margins": 0.11541748046875,
      "rewards/rejected": -0.18454590439796448,
      "step": 6810
    },
    {
      "epoch": 0.8801135630403923,
      "grad_norm": 2.8671290706605834,
      "learning_rate": 9.687189593929655e-07,
      "log_odds_chosen": 1.662695288658142,
      "log_odds_ratio": -0.3827148377895355,
      "logits/chosen": -1.1142578125,
      "logits/rejected": -0.9888671636581421,
      "logps/chosen": -0.669238269329071,
      "logps/rejected": -1.882421851158142,
      "loss": 0.6955,
      "nll_loss": 0.666210949420929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06695556640625,
      "rewards/margins": 0.12130127102136612,
      "rewards/rejected": -0.18825682997703552,
      "step": 6820
    },
    {
      "epoch": 0.8814040521357595,
      "grad_norm": 2.9162282785967353,
      "learning_rate": 9.680095349408789e-07,
      "log_odds_chosen": 1.8400390148162842,
      "log_odds_ratio": -0.3070312440395355,
      "logits/chosen": -1.09765625,
      "logits/rejected": -0.9619140625,
      "logps/chosen": -0.6553710699081421,
      "logps/rejected": -1.954687476158142,
      "loss": 0.6829,
      "nll_loss": 0.6541992425918579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06552734225988388,
      "rewards/margins": 0.12989501655101776,
      "rewards/rejected": -0.19541016221046448,
      "step": 6830
    },
    {
      "epoch": 0.8826945412311266,
      "grad_norm": 2.6181055774468485,
      "learning_rate": 9.673016668133487e-07,
      "log_odds_chosen": 1.778906226158142,
      "log_odds_ratio": -0.35371094942092896,
      "logits/chosen": -1.1318359375,
      "logits/rejected": -0.98046875,
      "logps/chosen": -0.699414074420929,
      "logps/rejected": -1.990625023841858,
      "loss": 0.6719,
      "nll_loss": 0.645703136920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06988525390625,
      "rewards/margins": 0.12900391221046448,
      "rewards/rejected": -0.19892577826976776,
      "step": 6840
    },
    {
      "epoch": 0.8839850303264938,
      "grad_norm": 3.5655466285650825,
      "learning_rate": 9.66595349328283e-07,
      "log_odds_chosen": 1.5359375476837158,
      "log_odds_ratio": -0.3519531190395355,
      "logits/chosen": -1.0871093273162842,
      "logits/rejected": -0.977343738079071,
      "logps/chosen": -0.634960949420929,
      "logps/rejected": -1.6748046875,
      "loss": 0.6678,
      "nll_loss": 0.642773449420929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06348876655101776,
      "rewards/margins": 0.10394897311925888,
      "rewards/rejected": -0.16755370795726776,
      "step": 6850
    },
    {
      "epoch": 0.8852755194218609,
      "grad_norm": 3.228024669643171,
      "learning_rate": 9.658905768325902e-07,
      "log_odds_chosen": 2.0751953125,
      "log_odds_ratio": -0.2889648377895355,
      "logits/chosen": -1.0769531726837158,
      "logits/rejected": -0.9417968988418579,
      "logps/chosen": -0.611621081829071,
      "logps/rejected": -2.0699219703674316,
      "loss": 0.6917,
      "nll_loss": 0.694042980670929,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06116943433880806,
      "rewards/margins": 0.14571532607078552,
      "rewards/rejected": -0.20693358778953552,
      "step": 6860
    },
    {
      "epoch": 0.886566008517228,
      "grad_norm": 2.997213902242432,
      "learning_rate": 9.651873437019902e-07,
      "log_odds_chosen": 1.7843749523162842,
      "log_odds_ratio": -0.3716064393520355,
      "logits/chosen": -1.116796851158142,
      "logits/rejected": -0.9722656011581421,
      "logps/chosen": -0.64501953125,
      "logps/rejected": -1.9425780773162842,
      "loss": 0.6887,
      "nll_loss": 0.64111328125,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06447754055261612,
      "rewards/margins": 0.12971191108226776,
      "rewards/rejected": -0.194091796875,
      "step": 6870
    },
    {
      "epoch": 0.8878564976125952,
      "grad_norm": 10.435282646236105,
      "learning_rate": 9.644856443408243e-07,
      "log_odds_chosen": 1.9033203125,
      "log_odds_ratio": -0.35624998807907104,
      "logits/chosen": -1.0958983898162842,
      "logits/rejected": -0.980664074420929,
      "logps/chosen": -0.630566418170929,
      "logps/rejected": -1.986718773841858,
      "loss": 0.7016,
      "nll_loss": 0.654980480670929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06307373195886612,
      "rewards/margins": 0.13555908203125,
      "rewards/rejected": -0.19875487685203552,
      "step": 6880
    },
    {
      "epoch": 0.8891469867079623,
      "grad_norm": 3.0867015916143665,
      "learning_rate": 9.637854731818697e-07,
      "log_odds_chosen": 1.8679687976837158,
      "log_odds_ratio": -0.3505859375,
      "logits/chosen": -1.0496094226837158,
      "logits/rejected": -0.937304675579071,
      "logps/chosen": -0.6182616949081421,
      "logps/rejected": -1.908593773841858,
      "loss": 0.6794,
      "nll_loss": 0.6488281488418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06184082105755806,
      "rewards/margins": 0.12891845405101776,
      "rewards/rejected": -0.19084472954273224,
      "step": 6890
    },
    {
      "epoch": 0.8904374758033294,
      "grad_norm": 4.065069619751776,
      "learning_rate": 9.630868246861536e-07,
      "log_odds_chosen": 1.7234375476837158,
      "log_odds_ratio": -0.3628906309604645,
      "logits/chosen": -1.0769531726837158,
      "logits/rejected": -0.9537109136581421,
      "logps/chosen": -0.632519543170929,
      "logps/rejected": -1.832421898841858,
      "loss": 0.6925,
      "nll_loss": 0.640820324420929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06328125298023224,
      "rewards/margins": 0.11988525092601776,
      "rewards/rejected": -0.183349609375,
      "step": 6900
    },
    {
      "epoch": 0.8917279648986967,
      "grad_norm": 3.5078271054455916,
      "learning_rate": 9.623896933427685e-07,
      "log_odds_chosen": 1.668701171875,
      "log_odds_ratio": -0.3796630799770355,
      "logits/chosen": -1.117773413658142,
      "logits/rejected": -0.9789062738418579,
      "logps/chosen": -0.5855468511581421,
      "logps/rejected": -1.7470703125,
      "loss": 0.6891,
      "nll_loss": 0.6117187738418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05855713039636612,
      "rewards/margins": 0.11613769829273224,
      "rewards/rejected": -0.17478027939796448,
      "step": 6910
    },
    {
      "epoch": 0.8930184539940638,
      "grad_norm": 2.946212784673711,
      "learning_rate": 9.6169407366869e-07,
      "log_odds_chosen": 1.6780273914337158,
      "log_odds_ratio": -0.3855957090854645,
      "logits/chosen": -1.096093773841858,
      "logits/rejected": -0.976757824420929,
      "logps/chosen": -0.637499988079071,
      "logps/rejected": -1.7843749523162842,
      "loss": 0.6816,
      "nll_loss": 0.6815429925918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06369628757238388,
      "rewards/margins": 0.11483459174633026,
      "rewards/rejected": -0.17841796576976776,
      "step": 6920
    },
    {
      "epoch": 0.8943089430894309,
      "grad_norm": 3.663684173283029,
      "learning_rate": 9.609999602085963e-07,
      "log_odds_chosen": 1.6642577648162842,
      "log_odds_ratio": -0.3636718690395355,
      "logits/chosen": -1.0857422351837158,
      "logits/rejected": -0.9964843988418579,
      "logps/chosen": -0.6806640625,
      "logps/rejected": -1.834375023841858,
      "loss": 0.6785,
      "nll_loss": 0.6421874761581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06802978366613388,
      "rewards/margins": 0.11530761420726776,
      "rewards/rejected": -0.1834716796875,
      "step": 6930
    },
    {
      "epoch": 0.895599432184798,
      "grad_norm": 3.3021104763096196,
      "learning_rate": 9.603073475346872e-07,
      "log_odds_chosen": 1.723046898841858,
      "log_odds_ratio": -0.35627442598342896,
      "logits/chosen": -1.0548827648162842,
      "logits/rejected": -0.928515613079071,
      "logps/chosen": -0.6495116949081421,
      "logps/rejected": -1.874414086341858,
      "loss": 0.6632,
      "nll_loss": 0.670214831829071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06500244140625,
      "rewards/margins": 0.12242431938648224,
      "rewards/rejected": -0.18735352158546448,
      "step": 6940
    },
    {
      "epoch": 0.8968899212801652,
      "grad_norm": 3.706745442862787,
      "learning_rate": 9.596162302465074e-07,
      "log_odds_chosen": 1.5703125,
      "log_odds_ratio": -0.37309569120407104,
      "logits/chosen": -1.144921898841858,
      "logits/rejected": -0.993359386920929,
      "logps/chosen": -0.6834961175918579,
      "logps/rejected": -1.744531273841858,
      "loss": 0.6897,
      "nll_loss": 0.6675781011581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.068359375,
      "rewards/margins": 0.1060791015625,
      "rewards/rejected": -0.17453613877296448,
      "step": 6950
    },
    {
      "epoch": 0.8981804103755323,
      "grad_norm": 3.2046950311480917,
      "learning_rate": 9.589266029707683e-07,
      "log_odds_chosen": 1.611718773841858,
      "log_odds_ratio": -0.3689941465854645,
      "logits/chosen": -1.1318359375,
      "logits/rejected": -1.003515601158142,
      "logps/chosen": -0.712109386920929,
      "logps/rejected": -1.832421898841858,
      "loss": 0.6831,
      "nll_loss": 0.640820324420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07124023139476776,
      "rewards/margins": 0.11204834282398224,
      "rewards/rejected": -0.183349609375,
      "step": 6960
    },
    {
      "epoch": 0.8994708994708994,
      "grad_norm": 3.028234172219317,
      "learning_rate": 9.582384603611731e-07,
      "log_odds_chosen": 1.7678711414337158,
      "log_odds_ratio": -0.34797364473342896,
      "logits/chosen": -1.116601586341858,
      "logits/rejected": -1.001953125,
      "logps/chosen": -0.648144543170929,
      "logps/rejected": -1.9089844226837158,
      "loss": 0.6912,
      "nll_loss": 0.645800769329071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06484375149011612,
      "rewards/margins": 0.12621459364891052,
      "rewards/rejected": -0.19106444716453552,
      "step": 6970
    },
    {
      "epoch": 0.9007613885662666,
      "grad_norm": 3.2497028340924374,
      "learning_rate": 9.575517970982428e-07,
      "log_odds_chosen": 2.187695264816284,
      "log_odds_ratio": -0.2735595703125,
      "logits/chosen": -1.0958983898162842,
      "logits/rejected": -0.969531238079071,
      "logps/chosen": -0.659375011920929,
      "logps/rejected": -2.2601561546325684,
      "loss": 0.7063,
      "nll_loss": 0.6924804449081421,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.06591796875,
      "rewards/margins": 0.16011962294578552,
      "rewards/rejected": -0.22609862685203552,
      "step": 6980
    },
    {
      "epoch": 0.9020518776616337,
      "grad_norm": 2.8839727607010617,
      "learning_rate": 9.568666078891436e-07,
      "log_odds_chosen": 1.689453125,
      "log_odds_ratio": -0.33979493379592896,
      "logits/chosen": -1.066015601158142,
      "logits/rejected": -0.956835925579071,
      "logps/chosen": -0.611328125,
      "logps/rejected": -1.763281226158142,
      "loss": 0.6919,
      "nll_loss": 0.6279296875,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0611572265625,
      "rewards/margins": 0.11520995944738388,
      "rewards/rejected": -0.17619629204273224,
      "step": 6990
    },
    {
      "epoch": 0.9033423667570009,
      "grad_norm": 3.5554965272103853,
      "learning_rate": 9.561828874675149e-07,
      "log_odds_chosen": 1.8904297351837158,
      "log_odds_ratio": -0.3439697325229645,
      "logits/chosen": -1.074804663658142,
      "logits/rejected": -0.966796875,
      "logps/chosen": -0.634765625,
      "logps/rejected": -1.9167969226837158,
      "loss": 0.6686,
      "nll_loss": 0.6058593988418579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06346435844898224,
      "rewards/margins": 0.12816162407398224,
      "rewards/rejected": -0.19160155951976776,
      "step": 7000
    },
    {
      "epoch": 0.9046328558523681,
      "grad_norm": 3.999118812832631,
      "learning_rate": 9.555006305933e-07,
      "log_odds_chosen": 1.893945336341858,
      "log_odds_ratio": -0.32487791776657104,
      "logits/chosen": -1.095703125,
      "logits/rejected": -0.9271484613418579,
      "logps/chosen": -0.6548827886581421,
      "logps/rejected": -2.010937452316284,
      "loss": 0.6715,
      "nll_loss": 0.64208984375,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06549072265625,
      "rewards/margins": 0.1356201171875,
      "rewards/rejected": -0.20112304389476776,
      "step": 7010
    },
    {
      "epoch": 0.9059233449477352,
      "grad_norm": 2.9717003321924,
      "learning_rate": 9.548198320525771e-07,
      "log_odds_chosen": 1.9328124523162842,
      "log_odds_ratio": -0.3331542909145355,
      "logits/chosen": -1.070703148841858,
      "logits/rejected": -0.957226574420929,
      "logps/chosen": -0.6512695550918579,
      "logps/rejected": -2.025390625,
      "loss": 0.6936,
      "nll_loss": 0.6332031488418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06511230766773224,
      "rewards/margins": 0.13730469346046448,
      "rewards/rejected": -0.20244140923023224,
      "step": 7020
    },
    {
      "epoch": 0.9072138340431023,
      "grad_norm": 3.365829133088305,
      "learning_rate": 9.54140486657392e-07,
      "log_odds_chosen": 1.859960913658142,
      "log_odds_ratio": -0.3392089903354645,
      "logits/chosen": -1.069921851158142,
      "logits/rejected": -0.9437500238418579,
      "logps/chosen": -0.6639648675918579,
      "logps/rejected": -1.9949219226837158,
      "loss": 0.6758,
      "nll_loss": 0.6309570074081421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06644286960363388,
      "rewards/margins": 0.13299560546875,
      "rewards/rejected": -0.19916991889476776,
      "step": 7030
    },
    {
      "epoch": 0.9085043231384695,
      "grad_norm": 2.763686246566196,
      "learning_rate": 9.534625892455922e-07,
      "log_odds_chosen": 1.6560547351837158,
      "log_odds_ratio": -0.37431639432907104,
      "logits/chosen": -1.0685546398162842,
      "logits/rejected": -0.9320312738418579,
      "logps/chosen": -0.681445300579071,
      "logps/rejected": -1.8660156726837158,
      "loss": 0.6774,
      "nll_loss": 0.669628918170929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06813964992761612,
      "rewards/margins": 0.11857757717370987,
      "rewards/rejected": -0.18671874701976776,
      "step": 7040
    },
    {
      "epoch": 0.9097948122338366,
      "grad_norm": 3.1834944525545104,
      "learning_rate": 9.527861346806618e-07,
      "log_odds_chosen": 1.925390601158142,
      "log_odds_ratio": -0.32172852754592896,
      "logits/chosen": -1.066992163658142,
      "logits/rejected": -0.942578136920929,
      "logps/chosen": -0.6463867425918579,
      "logps/rejected": -1.986914038658142,
      "loss": 0.6872,
      "nll_loss": 0.668749988079071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06464843451976776,
      "rewards/margins": 0.13410644233226776,
      "rewards/rejected": -0.19865722954273224,
      "step": 7050
    },
    {
      "epoch": 0.9110853013292037,
      "grad_norm": 3.655455775713598,
      "learning_rate": 9.521111178515582e-07,
      "log_odds_chosen": 1.779882788658142,
      "log_odds_ratio": -0.3077148497104645,
      "logits/chosen": -1.099218726158142,
      "logits/rejected": -0.976367175579071,
      "logps/chosen": -0.597363293170929,
      "logps/rejected": -1.8250000476837158,
      "loss": 0.6731,
      "nll_loss": 0.5400390625,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.05972900241613388,
      "rewards/margins": 0.12248535454273224,
      "rewards/rejected": -0.1822509765625,
      "step": 7060
    },
    {
      "epoch": 0.912375790424571,
      "grad_norm": 2.8159580653405065,
      "learning_rate": 9.514375336725502e-07,
      "log_odds_chosen": 1.835351586341858,
      "log_odds_ratio": -0.3180175721645355,
      "logits/chosen": -1.100195288658142,
      "logits/rejected": -0.97265625,
      "logps/chosen": -0.573046863079071,
      "logps/rejected": -1.821874976158142,
      "loss": 0.6621,
      "nll_loss": 0.6451171636581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05733642727136612,
      "rewards/margins": 0.12486572563648224,
      "rewards/rejected": -0.18215331435203552,
      "step": 7070
    },
    {
      "epoch": 0.9136662795199381,
      "grad_norm": 3.494557482376523,
      "learning_rate": 9.507653770830566e-07,
      "log_odds_chosen": 1.7978515625,
      "log_odds_ratio": -0.3416503965854645,
      "logits/chosen": -1.103515625,
      "logits/rejected": -0.9791015386581421,
      "logps/chosen": -0.6898437738418579,
      "logps/rejected": -2.0082030296325684,
      "loss": 0.6831,
      "nll_loss": 0.6722656488418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06904296576976776,
      "rewards/margins": 0.13176269829273224,
      "rewards/rejected": -0.20073242485523224,
      "step": 7080
    },
    {
      "epoch": 0.9149567686153052,
      "grad_norm": 3.3710633297132135,
      "learning_rate": 9.500946430474869e-07,
      "log_odds_chosen": 1.955468773841858,
      "log_odds_ratio": -0.2971435487270355,
      "logits/chosen": -1.065039038658142,
      "logits/rejected": -0.944140613079071,
      "logps/chosen": -0.6612304449081421,
      "logps/rejected": -2.083203077316284,
      "loss": 0.6889,
      "nll_loss": 0.6373046636581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06610107421875,
      "rewards/margins": 0.14202880859375,
      "rewards/rejected": -0.20810547471046448,
      "step": 7090
    },
    {
      "epoch": 0.9162472577106724,
      "grad_norm": 3.076142652416141,
      "learning_rate": 9.494253265550825e-07,
      "log_odds_chosen": 1.7998535633087158,
      "log_odds_ratio": -0.3482910096645355,
      "logits/chosen": -1.0652344226837158,
      "logits/rejected": -0.943554699420929,
      "logps/chosen": -0.617382824420929,
      "logps/rejected": -1.826171875,
      "loss": 0.6779,
      "nll_loss": 0.7049804925918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06165771558880806,
      "rewards/margins": 0.12058715522289276,
      "rewards/rejected": -0.182373046875,
      "step": 7100
    },
    {
      "epoch": 0.9175377468060395,
      "grad_norm": 2.9243043651526017,
      "learning_rate": 9.4875742261976e-07,
      "log_odds_chosen": 1.9609375,
      "log_odds_ratio": -0.29841309785842896,
      "logits/chosen": -1.104882836341858,
      "logits/rejected": -0.970898449420929,
      "logps/chosen": -0.614941418170929,
      "logps/rejected": -1.9851562976837158,
      "loss": 0.6746,
      "nll_loss": 0.6128906011581421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06145019456744194,
      "rewards/margins": 0.13701172173023224,
      "rewards/rejected": -0.19853515923023224,
      "step": 7110
    },
    {
      "epoch": 0.9188282359014066,
      "grad_norm": 3.3210751233132583,
      "learning_rate": 9.480909262799544e-07,
      "log_odds_chosen": 1.5085937976837158,
      "log_odds_ratio": -0.3974609375,
      "logits/chosen": -1.1212890148162842,
      "logits/rejected": -1.0066406726837158,
      "logps/chosen": -0.630566418170929,
      "logps/rejected": -1.6144530773162842,
      "loss": 0.6773,
      "nll_loss": 0.625292956829071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06306152045726776,
      "rewards/margins": 0.09840087592601776,
      "rewards/rejected": -0.16140136122703552,
      "step": 7120
    },
    {
      "epoch": 0.9201187249967738,
      "grad_norm": 4.0286489790030515,
      "learning_rate": 9.47425832598465e-07,
      "log_odds_chosen": 2.0660157203674316,
      "log_odds_ratio": -0.3036132752895355,
      "logits/chosen": -1.1408202648162842,
      "logits/rejected": -0.9808593988418579,
      "logps/chosen": -0.6270507574081421,
      "logps/rejected": -2.0921874046325684,
      "loss": 0.676,
      "nll_loss": 0.597851574420929,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06271972507238388,
      "rewards/margins": 0.146484375,
      "rewards/rejected": -0.20917968451976776,
      "step": 7130
    },
    {
      "epoch": 0.9214092140921409,
      "grad_norm": 2.6331275243214147,
      "learning_rate": 9.467621366623017e-07,
      "log_odds_chosen": 1.8445312976837158,
      "log_odds_ratio": -0.3343261778354645,
      "logits/chosen": -1.124609351158142,
      "logits/rejected": -0.985156238079071,
      "logps/chosen": -0.604199230670929,
      "logps/rejected": -1.8992187976837158,
      "loss": 0.6656,
      "nll_loss": 0.5897461175918579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06038818508386612,
      "rewards/margins": 0.12941893935203552,
      "rewards/rejected": -0.18974609673023224,
      "step": 7140
    },
    {
      "epoch": 0.922699703187508,
      "grad_norm": 2.672664158091753,
      "learning_rate": 9.46099833582532e-07,
      "log_odds_chosen": 1.470458984375,
      "log_odds_ratio": -0.3795410096645355,
      "logits/chosen": -1.112695336341858,
      "logits/rejected": -1.0085937976837158,
      "logps/chosen": -0.6787109375,
      "logps/rejected": -1.704687476158142,
      "loss": 0.6771,
      "nll_loss": 0.7240234613418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06788329780101776,
      "rewards/margins": 0.10242919623851776,
      "rewards/rejected": -0.17033691704273224,
      "step": 7150
    },
    {
      "epoch": 0.9239901922828752,
      "grad_norm": 3.118457727383749,
      "learning_rate": 9.45438918494131e-07,
      "log_odds_chosen": 1.8875000476837158,
      "log_odds_ratio": -0.30473631620407104,
      "logits/chosen": -1.1201171875,
      "logits/rejected": -0.992382824420929,
      "logps/chosen": -0.6371093988418579,
      "logps/rejected": -1.9386718273162842,
      "loss": 0.663,
      "nll_loss": 0.604199230670929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.063720703125,
      "rewards/margins": 0.13007812201976776,
      "rewards/rejected": -0.19384765625,
      "step": 7160
    },
    {
      "epoch": 0.9252806813782424,
      "grad_norm": 3.8176703336183655,
      "learning_rate": 9.447793865558291e-07,
      "log_odds_chosen": 1.6287109851837158,
      "log_odds_ratio": -0.39189451932907104,
      "logits/chosen": -1.091796875,
      "logits/rejected": -0.9886718988418579,
      "logps/chosen": -0.671875,
      "logps/rejected": -1.7841796875,
      "loss": 0.6912,
      "nll_loss": 0.677929699420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06724853813648224,
      "rewards/margins": 0.11119689792394638,
      "rewards/rejected": -0.17839355766773224,
      "step": 7170
    },
    {
      "epoch": 0.9265711704736095,
      "grad_norm": 2.891306003864294,
      "learning_rate": 9.441212329499659e-07,
      "log_odds_chosen": 1.805273413658142,
      "log_odds_ratio": -0.35932618379592896,
      "logits/chosen": -1.0925781726837158,
      "logits/rejected": -1.001562476158142,
      "logps/chosen": -0.62890625,
      "logps/rejected": -1.8816406726837158,
      "loss": 0.6657,
      "nll_loss": 0.680957019329071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0628662109375,
      "rewards/margins": 0.12534180283546448,
      "rewards/rejected": -0.18820801377296448,
      "step": 7180
    },
    {
      "epoch": 0.9278616595689766,
      "grad_norm": 3.228635745899415,
      "learning_rate": 9.434644528823399e-07,
      "log_odds_chosen": 1.716796875,
      "log_odds_ratio": -0.3456054627895355,
      "logits/chosen": -1.127343773841858,
      "logits/rejected": -1.0056641101837158,
      "logps/chosen": -0.6650390625,
      "logps/rejected": -1.9226562976837158,
      "loss": 0.6823,
      "nll_loss": 0.7294921875,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.0665283203125,
      "rewards/margins": 0.12586669623851776,
      "rewards/rejected": -0.19243164360523224,
      "step": 7190
    },
    {
      "epoch": 0.9291521486643438,
      "grad_norm": 3.396076656928139,
      "learning_rate": 9.428090415820634e-07,
      "log_odds_chosen": 1.680078148841858,
      "log_odds_ratio": -0.39277344942092896,
      "logits/chosen": -1.123437523841858,
      "logits/rejected": -1.0089843273162842,
      "logps/chosen": -0.7074218988418579,
      "logps/rejected": -1.908789038658142,
      "loss": 0.6821,
      "nll_loss": 0.6304687261581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07076416164636612,
      "rewards/margins": 0.12007751315832138,
      "rewards/rejected": -0.19082030653953552,
      "step": 7200
    },
    {
      "epoch": 0.9304426377597109,
      "grad_norm": 2.9095886362595147,
      "learning_rate": 9.42154994301416e-07,
      "log_odds_chosen": 1.9386718273162842,
      "log_odds_ratio": -0.3110595643520355,
      "logits/chosen": -1.122460961341858,
      "logits/rejected": -0.9974609613418579,
      "logps/chosen": -0.64599609375,
      "logps/rejected": -2.0503907203674316,
      "loss": 0.6776,
      "nll_loss": 0.6436523199081421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06462402641773224,
      "rewards/margins": 0.14024658501148224,
      "rewards/rejected": -0.20468750596046448,
      "step": 7210
    },
    {
      "epoch": 0.931733126855078,
      "grad_norm": 3.66545307255962,
      "learning_rate": 9.415023063157008e-07,
      "log_odds_chosen": 1.759179711341858,
      "log_odds_ratio": -0.36333006620407104,
      "logits/chosen": -1.1062500476837158,
      "logits/rejected": -0.9644531011581421,
      "logps/chosen": -0.67138671875,
      "logps/rejected": -1.9347655773162842,
      "loss": 0.6762,
      "nll_loss": 0.668652355670929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06712646782398224,
      "rewards/margins": 0.12644653022289276,
      "rewards/rejected": -0.193603515625,
      "step": 7220
    },
    {
      "epoch": 0.9330236159504453,
      "grad_norm": 3.583691853035753,
      "learning_rate": 9.408509729231009e-07,
      "log_odds_chosen": 1.951562523841858,
      "log_odds_ratio": -0.3077148497104645,
      "logits/chosen": -1.138281226158142,
      "logits/rejected": -0.9927734136581421,
      "logps/chosen": -0.6822265386581421,
      "logps/rejected": -2.0894532203674316,
      "loss": 0.6777,
      "nll_loss": 0.5830078125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0682373046875,
      "rewards/margins": 0.14069823920726776,
      "rewards/rejected": -0.20893554389476776,
      "step": 7230
    },
    {
      "epoch": 0.9343141050458124,
      "grad_norm": 3.8881209363110005,
      "learning_rate": 9.402009894445369e-07,
      "log_odds_chosen": 1.7927734851837158,
      "log_odds_ratio": -0.34687501192092896,
      "logits/chosen": -1.1435546875,
      "logits/rejected": -1.0187499523162842,
      "logps/chosen": -0.6551758050918579,
      "logps/rejected": -1.982421875,
      "loss": 0.696,
      "nll_loss": 0.621874988079071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06545410305261612,
      "rewards/margins": 0.13264159858226776,
      "rewards/rejected": -0.19821777939796448,
      "step": 7240
    },
    {
      "epoch": 0.9356045941411795,
      "grad_norm": 3.2091285668772556,
      "learning_rate": 9.395523512235255e-07,
      "log_odds_chosen": 2.01171875,
      "log_odds_ratio": -0.29216307401657104,
      "logits/chosen": -1.0769531726837158,
      "logits/rejected": -0.959179699420929,
      "logps/chosen": -0.639843761920929,
      "logps/rejected": -2.0703125,
      "loss": 0.6791,
      "nll_loss": 0.64453125,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06394042819738388,
      "rewards/margins": 0.1429443359375,
      "rewards/rejected": -0.20688477158546448,
      "step": 7250
    },
    {
      "epoch": 0.9368950832365467,
      "grad_norm": 2.6139412971263263,
      "learning_rate": 9.389050536260404e-07,
      "log_odds_chosen": 1.775781273841858,
      "log_odds_ratio": -0.33867186307907104,
      "logits/chosen": -1.0964844226837158,
      "logits/rejected": -0.9556640386581421,
      "logps/chosen": -0.616992175579071,
      "logps/rejected": -1.8669922351837158,
      "loss": 0.6739,
      "nll_loss": 0.6181640625,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06174316257238388,
      "rewards/margins": 0.12502440810203552,
      "rewards/rejected": -0.18681640923023224,
      "step": 7260
    },
    {
      "epoch": 0.9381855723319138,
      "grad_norm": 3.446859668027108,
      "learning_rate": 9.382590920403722e-07,
      "log_odds_chosen": 2.0914063453674316,
      "log_odds_ratio": -0.26896971464157104,
      "logits/chosen": -1.111328125,
      "logits/rejected": -0.9781249761581421,
      "logps/chosen": -0.615917980670929,
      "logps/rejected": -2.0923829078674316,
      "loss": 0.6571,
      "nll_loss": 0.6177734136581421,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06158447265625,
      "rewards/margins": 0.1475830078125,
      "rewards/rejected": -0.20913085341453552,
      "step": 7270
    },
    {
      "epoch": 0.9394760614272809,
      "grad_norm": 3.386572674493646,
      "learning_rate": 9.376144618769908e-07,
      "log_odds_chosen": 1.9484374523162842,
      "log_odds_ratio": -0.307861328125,
      "logits/chosen": -1.1150391101837158,
      "logits/rejected": -0.968554675579071,
      "logps/chosen": -0.640625,
      "logps/rejected": -2.057812452316284,
      "loss": 0.6595,
      "nll_loss": 0.6126953363418579,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06413574516773224,
      "rewards/margins": 0.14178466796875,
      "rewards/rejected": -0.20573730766773224,
      "step": 7280
    },
    {
      "epoch": 0.9407665505226481,
      "grad_norm": 2.94158776243296,
      "learning_rate": 9.369711585684086e-07,
      "log_odds_chosen": 1.796484351158142,
      "log_odds_ratio": -0.3426757752895355,
      "logits/chosen": -1.1232421398162842,
      "logits/rejected": -0.9732421636581421,
      "logps/chosen": -0.6346679925918579,
      "logps/rejected": -1.8917968273162842,
      "loss": 0.6907,
      "nll_loss": 0.65966796875,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.0634765625,
      "rewards/margins": 0.12575682997703552,
      "rewards/rejected": -0.18918457627296448,
      "step": 7290
    },
    {
      "epoch": 0.9420570396180152,
      "grad_norm": 4.076191013362035,
      "learning_rate": 9.363291775690445e-07,
      "log_odds_chosen": 1.967187523841858,
      "log_odds_ratio": -0.3042236268520355,
      "logits/chosen": -1.099609375,
      "logits/rejected": -0.963085949420929,
      "logps/chosen": -0.6094726324081421,
      "logps/rejected": -2.007031202316284,
      "loss": 0.6718,
      "nll_loss": 0.641796886920929,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06098632887005806,
      "rewards/margins": 0.1397705078125,
      "rewards/rejected": -0.2005615234375,
      "step": 7300
    },
    {
      "epoch": 0.9433475287133823,
      "grad_norm": 3.228606303186405,
      "learning_rate": 9.356885143550886e-07,
      "log_odds_chosen": 2.1556639671325684,
      "log_odds_ratio": -0.28082275390625,
      "logits/chosen": -1.163476586341858,
      "logits/rejected": -1.0041015148162842,
      "logps/chosen": -0.6498047113418579,
      "logps/rejected": -2.2203125953674316,
      "loss": 0.681,
      "nll_loss": 0.61328125,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06495361030101776,
      "rewards/margins": 0.15700682997703552,
      "rewards/rejected": -0.22202149033546448,
      "step": 7310
    },
    {
      "epoch": 0.9446380178087496,
      "grad_norm": 3.2341310574652136,
      "learning_rate": 9.350491644243688e-07,
      "log_odds_chosen": 1.869726538658142,
      "log_odds_ratio": -0.3248535096645355,
      "logits/chosen": -1.092187523841858,
      "logits/rejected": -0.991992175579071,
      "logps/chosen": -0.634472668170929,
      "logps/rejected": -1.9640624523162842,
      "loss": 0.6697,
      "nll_loss": 0.6348632574081421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06346435844898224,
      "rewards/margins": 0.13295897841453552,
      "rewards/rejected": -0.19633789360523224,
      "step": 7320
    },
    {
      "epoch": 0.9459285069041167,
      "grad_norm": 3.2375044137833093,
      "learning_rate": 9.344111232962179e-07,
      "log_odds_chosen": 1.5847656726837158,
      "log_odds_ratio": -0.39155274629592896,
      "logits/chosen": -1.0593750476837158,
      "logits/rejected": -0.960156261920929,
      "logps/chosen": -0.653515636920929,
      "logps/rejected": -1.7453124523162842,
      "loss": 0.6773,
      "nll_loss": 0.6753906011581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06536865234375,
      "rewards/margins": 0.109130859375,
      "rewards/rejected": -0.17460937798023224,
      "step": 7330
    },
    {
      "epoch": 0.9472189959994838,
      "grad_norm": 3.7895603264133437,
      "learning_rate": 9.337743865113415e-07,
      "log_odds_chosen": 1.7439453601837158,
      "log_odds_ratio": -0.3455810546875,
      "logits/chosen": -1.084375023841858,
      "logits/rejected": -0.941601574420929,
      "logps/chosen": -0.62841796875,
      "logps/rejected": -1.8312499523162842,
      "loss": 0.6653,
      "nll_loss": 0.5625976324081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06284179538488388,
      "rewards/margins": 0.1202392578125,
      "rewards/rejected": -0.18325194716453552,
      "step": 7340
    },
    {
      "epoch": 0.948509485094851,
      "grad_norm": 2.900579307988274,
      "learning_rate": 9.331389496316868e-07,
      "log_odds_chosen": 1.865625023841858,
      "log_odds_ratio": -0.33391112089157104,
      "logits/chosen": -1.11328125,
      "logits/rejected": -0.984375,
      "logps/chosen": -0.630175769329071,
      "logps/rejected": -1.9404296875,
      "loss": 0.6737,
      "nll_loss": 0.62744140625,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06301269680261612,
      "rewards/margins": 0.13103942573070526,
      "rewards/rejected": -0.19406738877296448,
      "step": 7350
    },
    {
      "epoch": 0.9497999741902181,
      "grad_norm": 3.9515361019035398,
      "learning_rate": 9.325048082403138e-07,
      "log_odds_chosen": 1.9402344226837158,
      "log_odds_ratio": -0.3038085997104645,
      "logits/chosen": -1.0634765625,
      "logits/rejected": -0.9419921636581421,
      "logps/chosen": -0.633105456829071,
      "logps/rejected": -1.986718773841858,
      "loss": 0.6532,
      "nll_loss": 0.6473633050918579,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06333007663488388,
      "rewards/margins": 0.13540038466453552,
      "rewards/rejected": -0.19887694716453552,
      "step": 7360
    },
    {
      "epoch": 0.9510904632855852,
      "grad_norm": 3.2965825407180356,
      "learning_rate": 9.318719579412648e-07,
      "log_odds_chosen": 1.9421875476837158,
      "log_odds_ratio": -0.3338867127895355,
      "logits/chosen": -1.090429663658142,
      "logits/rejected": -0.978710949420929,
      "logps/chosen": -0.676562488079071,
      "logps/rejected": -2.0679688453674316,
      "loss": 0.6859,
      "nll_loss": 0.6407226324081421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06768798828125,
      "rewards/margins": 0.13923339545726776,
      "rewards/rejected": -0.20688477158546448,
      "step": 7370
    },
    {
      "epoch": 0.9523809523809523,
      "grad_norm": 2.7099569849818193,
      "learning_rate": 9.312403943594374e-07,
      "log_odds_chosen": 1.645898461341858,
      "log_odds_ratio": -0.33305662870407104,
      "logits/chosen": -1.068359375,
      "logits/rejected": -0.957812488079071,
      "logps/chosen": -0.5927734375,
      "logps/rejected": -1.6789062023162842,
      "loss": 0.6685,
      "nll_loss": 0.66943359375,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.05928955227136612,
      "rewards/margins": 0.108642578125,
      "rewards/rejected": -0.16777344048023224,
      "step": 7380
    },
    {
      "epoch": 0.9536714414763195,
      "grad_norm": 3.7986675097598406,
      "learning_rate": 9.306101131404582e-07,
      "log_odds_chosen": 2.0582032203674316,
      "log_odds_ratio": -0.296630859375,
      "logits/chosen": -1.0958983898162842,
      "logits/rejected": -0.928906261920929,
      "logps/chosen": -0.622851550579071,
      "logps/rejected": -2.10546875,
      "loss": 0.6599,
      "nll_loss": 0.628125011920929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06231689453125,
      "rewards/margins": 0.14837646484375,
      "rewards/rejected": -0.21044921875,
      "step": 7390
    },
    {
      "epoch": 0.9549619305716867,
      "grad_norm": 3.539936337246152,
      "learning_rate": 9.299811099505542e-07,
      "log_odds_chosen": 1.8068358898162842,
      "log_odds_ratio": -0.35893553495407104,
      "logits/chosen": -1.069921851158142,
      "logits/rejected": -0.9488281011581421,
      "logps/chosen": -0.728710949420929,
      "logps/rejected": -2.034374952316284,
      "loss": 0.6768,
      "nll_loss": 0.68017578125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.07293701171875,
      "rewards/margins": 0.13056030869483948,
      "rewards/rejected": -0.2034912109375,
      "step": 7400
    },
    {
      "epoch": 0.9562524196670538,
      "grad_norm": 3.474381994321401,
      "learning_rate": 9.293533804764305e-07,
      "log_odds_chosen": 1.78515625,
      "log_odds_ratio": -0.3757690489292145,
      "logits/chosen": -1.048437476158142,
      "logits/rejected": -0.924023449420929,
      "logps/chosen": -0.66796875,
      "logps/rejected": -1.969140648841858,
      "loss": 0.6678,
      "nll_loss": 0.6651366949081421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06671142578125,
      "rewards/margins": 0.13009032607078552,
      "rewards/rejected": -0.19699707627296448,
      "step": 7410
    },
    {
      "epoch": 0.957542908762421,
      "grad_norm": 3.3700654773887146,
      "learning_rate": 9.28726920425144e-07,
      "log_odds_chosen": 1.970703125,
      "log_odds_ratio": -0.3067871034145355,
      "logits/chosen": -1.075585961341858,
      "logits/rejected": -0.981640636920929,
      "logps/chosen": -0.580371081829071,
      "logps/rejected": -1.9445312023162842,
      "loss": 0.6553,
      "nll_loss": 0.6078125238418579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05803222581744194,
      "rewards/margins": 0.13637694716453552,
      "rewards/rejected": -0.19440917670726776,
      "step": 7420
    },
    {
      "epoch": 0.9588333978577881,
      "grad_norm": 2.8352483371691948,
      "learning_rate": 9.281017255239815e-07,
      "log_odds_chosen": 1.685937523841858,
      "log_odds_ratio": -0.3646484315395355,
      "logits/chosen": -1.0849609375,
      "logits/rejected": -0.9322265386581421,
      "logps/chosen": -0.6908203363418579,
      "logps/rejected": -1.9375,
      "loss": 0.6527,
      "nll_loss": 0.6620117425918579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06912841647863388,
      "rewards/margins": 0.124755859375,
      "rewards/rejected": -0.19389648735523224,
      "step": 7430
    },
    {
      "epoch": 0.9601238869531552,
      "grad_norm": 3.8319398029906404,
      "learning_rate": 9.274777915203365e-07,
      "log_odds_chosen": 1.7096679210662842,
      "log_odds_ratio": -0.33574217557907104,
      "logits/chosen": -1.0662109851837158,
      "logits/rejected": -0.941601574420929,
      "logps/chosen": -0.6307617425918579,
      "logps/rejected": -1.8230469226837158,
      "loss": 0.6722,
      "nll_loss": 0.627148449420929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06313476711511612,
      "rewards/margins": 0.11927489936351776,
      "rewards/rejected": -0.18244628608226776,
      "step": 7440
    },
    {
      "epoch": 0.9614143760485224,
      "grad_norm": 3.208330522529873,
      "learning_rate": 9.268551141815875e-07,
      "log_odds_chosen": 1.638525366783142,
      "log_odds_ratio": -0.34819334745407104,
      "logits/chosen": -1.0398437976837158,
      "logits/rejected": -0.9154297113418579,
      "logps/chosen": -0.63037109375,
      "logps/rejected": -1.7658202648162842,
      "loss": 0.6642,
      "nll_loss": 0.68310546875,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06302490085363388,
      "rewards/margins": 0.11359862983226776,
      "rewards/rejected": -0.17659911513328552,
      "step": 7450
    },
    {
      "epoch": 0.9627048651438895,
      "grad_norm": 3.5322050288270725,
      "learning_rate": 9.262336892949784e-07,
      "log_odds_chosen": 1.9249999523162842,
      "log_odds_ratio": -0.3111328184604645,
      "logits/chosen": -1.099609375,
      "logits/rejected": -0.963085949420929,
      "logps/chosen": -0.6435546875,
      "logps/rejected": -2.0257811546325684,
      "loss": 0.6761,
      "nll_loss": 0.607128918170929,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.06428222358226776,
      "rewards/margins": 0.13818359375,
      "rewards/rejected": -0.20263671875,
      "step": 7460
    },
    {
      "epoch": 0.9639953542392566,
      "grad_norm": 3.9877814112711594,
      "learning_rate": 9.256135126674977e-07,
      "log_odds_chosen": 1.6843750476837158,
      "log_odds_ratio": -0.36420899629592896,
      "logits/chosen": -1.0724608898162842,
      "logits/rejected": -0.940625011920929,
      "logps/chosen": -0.681347668170929,
      "logps/rejected": -1.8996093273162842,
      "loss": 0.6796,
      "nll_loss": 0.6883789300918579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06820068508386612,
      "rewards/margins": 0.12176513671875,
      "rewards/rejected": -0.18989257514476776,
      "step": 7470
    },
    {
      "epoch": 0.9652858433346239,
      "grad_norm": 3.8830768313347517,
      "learning_rate": 9.249945801257605e-07,
      "log_odds_chosen": 2.0289063453674316,
      "log_odds_ratio": -0.30500489473342896,
      "logits/chosen": -1.047460913658142,
      "logits/rejected": -0.9378906488418579,
      "logps/chosen": -0.600390613079071,
      "logps/rejected": -2.0394530296325684,
      "loss": 0.6481,
      "nll_loss": 0.6185547113418579,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06005859375,
      "rewards/margins": 0.1441650390625,
      "rewards/rejected": -0.20419922471046448,
      "step": 7480
    },
    {
      "epoch": 0.966576332429991,
      "grad_norm": 2.9931493063273416,
      "learning_rate": 9.243768875158902e-07,
      "log_odds_chosen": 1.837890625,
      "log_odds_ratio": -0.32207030057907104,
      "logits/chosen": -1.0978515148162842,
      "logits/rejected": -0.9593750238418579,
      "logps/chosen": -0.5936523675918579,
      "logps/rejected": -1.845703125,
      "loss": 0.6688,
      "nll_loss": 0.5948241949081421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05933837965130806,
      "rewards/margins": 0.12526854872703552,
      "rewards/rejected": -0.184814453125,
      "step": 7490
    },
    {
      "epoch": 0.9678668215253581,
      "grad_norm": 2.5992405702799433,
      "learning_rate": 9.23760430703401e-07,
      "log_odds_chosen": 1.624902367591858,
      "log_odds_ratio": -0.38959962129592896,
      "logits/chosen": -1.058984398841858,
      "logits/rejected": -0.962695300579071,
      "logps/chosen": -0.6460937261581421,
      "logps/rejected": -1.746484398841858,
      "loss": 0.6468,
      "nll_loss": 0.5899413824081421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06463623046875,
      "rewards/margins": 0.11016235500574112,
      "rewards/rejected": -0.17473144829273224,
      "step": 7500
    },
    {
      "epoch": 0.9691573106207253,
      "grad_norm": 3.3326634870383267,
      "learning_rate": 9.231452055730832e-07,
      "log_odds_chosen": 1.650390625,
      "log_odds_ratio": -0.3848632872104645,
      "logits/chosen": -1.0673828125,
      "logits/rejected": -0.990039050579071,
      "logps/chosen": -0.663867175579071,
      "logps/rejected": -1.810937523841858,
      "loss": 0.6535,
      "nll_loss": 0.6435546875,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06641845405101776,
      "rewards/margins": 0.11478271335363388,
      "rewards/rejected": -0.18122558295726776,
      "step": 7510
    },
    {
      "epoch": 0.9704477997160924,
      "grad_norm": 3.096809480089861,
      "learning_rate": 9.225312080288851e-07,
      "log_odds_chosen": 1.922460913658142,
      "log_odds_ratio": -0.3490234315395355,
      "logits/chosen": -1.1257812976837158,
      "logits/rejected": -0.998242199420929,
      "logps/chosen": -0.69091796875,
      "logps/rejected": -2.088671922683716,
      "loss": 0.6369,
      "nll_loss": 0.5625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06907959282398224,
      "rewards/margins": 0.13979491591453552,
      "rewards/rejected": -0.2088623046875,
      "step": 7520
    },
    {
      "epoch": 0.9717382888114595,
      "grad_norm": 3.222948057528756,
      "learning_rate": 9.219184339938013e-07,
      "log_odds_chosen": 2.0628905296325684,
      "log_odds_ratio": -0.29931640625,
      "logits/chosen": -1.0925781726837158,
      "logits/rejected": -0.938281238079071,
      "logps/chosen": -0.6539062261581421,
      "logps/rejected": -2.1703124046325684,
      "loss": 0.6675,
      "nll_loss": 0.6473633050918579,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06538085639476776,
      "rewards/margins": 0.15172728896141052,
      "rewards/rejected": -0.21706542372703552,
      "step": 7530
    },
    {
      "epoch": 0.9730287779068267,
      "grad_norm": 3.0720853199299287,
      "learning_rate": 9.213068794097574e-07,
      "log_odds_chosen": 2.1483397483825684,
      "log_odds_ratio": -0.3097167909145355,
      "logits/chosen": -1.069921851158142,
      "logits/rejected": -0.94921875,
      "logps/chosen": -0.602734386920929,
      "logps/rejected": -2.161328077316284,
      "loss": 0.673,
      "nll_loss": 0.592968761920929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06033935397863388,
      "rewards/margins": 0.15576782822608948,
      "rewards/rejected": -0.2161865234375,
      "step": 7540
    },
    {
      "epoch": 0.9743192670021938,
      "grad_norm": 3.1881919858392234,
      "learning_rate": 9.206965402374975e-07,
      "log_odds_chosen": 2.0655274391174316,
      "log_odds_ratio": -0.3031249940395355,
      "logits/chosen": -1.07421875,
      "logits/rejected": -0.9273437261581421,
      "logps/chosen": -0.57958984375,
      "logps/rejected": -2.0453124046325684,
      "loss": 0.6599,
      "nll_loss": 0.6083984375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05795898288488388,
      "rewards/margins": 0.14653320610523224,
      "rewards/rejected": -0.20454101264476776,
      "step": 7550
    },
    {
      "epoch": 0.975609756097561,
      "grad_norm": 3.420538228064402,
      "learning_rate": 9.200874124564723e-07,
      "log_odds_chosen": 1.870996117591858,
      "log_odds_ratio": -0.28752440214157104,
      "logits/chosen": -1.051367163658142,
      "logits/rejected": -0.886914074420929,
      "logps/chosen": -0.644726574420929,
      "logps/rejected": -1.9083983898162842,
      "loss": 0.6541,
      "nll_loss": 0.649121105670929,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06447754055261612,
      "rewards/margins": 0.126495361328125,
      "rewards/rejected": -0.19086913764476776,
      "step": 7560
    },
    {
      "epoch": 0.9769002451929282,
      "grad_norm": 3.055725883730373,
      "learning_rate": 9.194794920647274e-07,
      "log_odds_chosen": 1.769921898841858,
      "log_odds_ratio": -0.3478027284145355,
      "logits/chosen": -1.083593726158142,
      "logits/rejected": -0.950390636920929,
      "logps/chosen": -0.650683581829071,
      "logps/rejected": -1.8484375476837158,
      "loss": 0.6629,
      "nll_loss": 0.656542956829071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06510009616613388,
      "rewards/margins": 0.11982421576976776,
      "rewards/rejected": -0.18483886122703552,
      "step": 7570
    },
    {
      "epoch": 0.9781907342882953,
      "grad_norm": 3.090318523241275,
      "learning_rate": 9.188727750787932e-07,
      "log_odds_chosen": 1.7620728015899658,
      "log_odds_ratio": -0.3774658143520355,
      "logits/chosen": -1.0642578601837158,
      "logits/rejected": -0.957812488079071,
      "logps/chosen": -0.708300769329071,
      "logps/rejected": -1.976953148841858,
      "loss": 0.6372,
      "nll_loss": 0.6421874761581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07076416164636612,
      "rewards/margins": 0.12715911865234375,
      "rewards/rejected": -0.197998046875,
      "step": 7580
    },
    {
      "epoch": 0.9794812233836624,
      "grad_norm": 3.085302623380204,
      "learning_rate": 9.182672575335757e-07,
      "log_odds_chosen": 1.6784179210662842,
      "log_odds_ratio": -0.3583984375,
      "logits/chosen": -1.1085937023162842,
      "logits/rejected": -0.982226550579071,
      "logps/chosen": -0.6294921636581421,
      "logps/rejected": -1.8447265625,
      "loss": 0.6894,
      "nll_loss": 0.6473633050918579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06297607719898224,
      "rewards/margins": 0.12142334133386612,
      "rewards/rejected": -0.1844482421875,
      "step": 7590
    },
    {
      "epoch": 0.9807717124790295,
      "grad_norm": 3.4587171011828244,
      "learning_rate": 9.176629354822469e-07,
      "log_odds_chosen": 1.8134765625,
      "log_odds_ratio": -0.33613282442092896,
      "logits/chosen": -1.056640625,
      "logits/rejected": -0.9234374761581421,
      "logps/chosen": -0.65087890625,
      "logps/rejected": -1.9226562976837158,
      "loss": 0.6668,
      "nll_loss": 0.6270507574081421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.0650634765625,
      "rewards/margins": 0.127166748046875,
      "rewards/rejected": -0.192138671875,
      "step": 7600
    },
    {
      "epoch": 0.9820622015743967,
      "grad_norm": 3.943827933052706,
      "learning_rate": 9.170598049961371e-07,
      "log_odds_chosen": 1.6630859375,
      "log_odds_ratio": -0.352294921875,
      "logits/chosen": -1.0837891101837158,
      "logits/rejected": -0.966601550579071,
      "logps/chosen": -0.605664074420929,
      "logps/rejected": -1.758203148841858,
      "loss": 0.6527,
      "nll_loss": 0.65185546875,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06058349460363388,
      "rewards/margins": 0.11524657905101776,
      "rewards/rejected": -0.17580565810203552,
      "step": 7610
    },
    {
      "epoch": 0.9833526906697638,
      "grad_norm": 3.6091274436542324,
      "learning_rate": 9.164578621646276e-07,
      "log_odds_chosen": 1.6228516101837158,
      "log_odds_ratio": -0.36176759004592896,
      "logits/chosen": -1.070703148841858,
      "logits/rejected": -0.966601550579071,
      "logps/chosen": -0.6456054449081421,
      "logps/rejected": -1.769921898841858,
      "loss": 0.637,
      "nll_loss": 0.666699230670929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0645751953125,
      "rewards/margins": 0.11247558891773224,
      "rewards/rejected": -0.17709961533546448,
      "step": 7620
    },
    {
      "epoch": 0.9846431797651309,
      "grad_norm": 2.9090043635434,
      "learning_rate": 9.15857103095044e-07,
      "log_odds_chosen": 1.7051270008087158,
      "log_odds_ratio": -0.35478514432907104,
      "logits/chosen": -1.077734351158142,
      "logits/rejected": -0.966601550579071,
      "logps/chosen": -0.6302734613418579,
      "logps/rejected": -1.799218773841858,
      "loss": 0.6629,
      "nll_loss": 0.658203125,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06302490085363388,
      "rewards/margins": 0.11686553806066513,
      "rewards/rejected": -0.17988280951976776,
      "step": 7630
    },
    {
      "epoch": 0.9859336688604982,
      "grad_norm": 4.560849220459965,
      "learning_rate": 9.15257523912551e-07,
      "log_odds_chosen": 1.779296875,
      "log_odds_ratio": -0.34062498807907104,
      "logits/chosen": -1.072265625,
      "logits/rejected": -0.9361327886581421,
      "logps/chosen": -0.6338866949081421,
      "logps/rejected": -1.855859398841858,
      "loss": 0.6598,
      "nll_loss": 0.6480468511581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06341552734375,
      "rewards/margins": 0.1221923828125,
      "rewards/rejected": -0.18564453721046448,
      "step": 7640
    },
    {
      "epoch": 0.9872241579558653,
      "grad_norm": 2.8263133888371765,
      "learning_rate": 9.146591207600472e-07,
      "log_odds_chosen": 2.0326170921325684,
      "log_odds_ratio": -0.291748046875,
      "logits/chosen": -1.127539038658142,
      "logits/rejected": -0.989453136920929,
      "logps/chosen": -0.5960937738418579,
      "logps/rejected": -1.994140625,
      "loss": 0.6629,
      "nll_loss": 0.625292956829071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05959472805261612,
      "rewards/margins": 0.13997802138328552,
      "rewards/rejected": -0.1995849609375,
      "step": 7650
    },
    {
      "epoch": 0.9885146470512324,
      "grad_norm": 3.606204720094968,
      "learning_rate": 9.140618897980601e-07,
      "log_odds_chosen": 1.658300757408142,
      "log_odds_ratio": -0.4019531309604645,
      "logits/chosen": -1.1042969226837158,
      "logits/rejected": -1.0310547351837158,
      "logps/chosen": -0.639843761920929,
      "logps/rejected": -1.8240234851837158,
      "loss": 0.63,
      "nll_loss": 0.5833984613418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06405029445886612,
      "rewards/margins": 0.1182861328125,
      "rewards/rejected": -0.18244628608226776,
      "step": 7660
    },
    {
      "epoch": 0.9898051361465996,
      "grad_norm": 3.790625217689734,
      "learning_rate": 9.134658272046442e-07,
      "log_odds_chosen": 1.935156226158142,
      "log_odds_ratio": -0.3365234434604645,
      "logits/chosen": -1.1457030773162842,
      "logits/rejected": -0.9908202886581421,
      "logps/chosen": -0.623242199420929,
      "logps/rejected": -2.026171922683716,
      "loss": 0.6518,
      "nll_loss": 0.613085925579071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06234131008386612,
      "rewards/margins": 0.14042969048023224,
      "rewards/rejected": -0.20266112685203552,
      "step": 7670
    },
    {
      "epoch": 0.9910956252419667,
      "grad_norm": 4.310214818077457,
      "learning_rate": 9.128709291752768e-07,
      "log_odds_chosen": 1.858007788658142,
      "log_odds_ratio": -0.327392578125,
      "logits/chosen": -1.105078101158142,
      "logits/rejected": -0.9791015386581421,
      "logps/chosen": -0.6180664300918579,
      "logps/rejected": -1.9460937976837158,
      "loss": 0.6561,
      "nll_loss": 0.6343749761581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06177978590130806,
      "rewards/margins": 0.13266602158546448,
      "rewards/rejected": -0.194580078125,
      "step": 7680
    },
    {
      "epoch": 0.9923861143373338,
      "grad_norm": 3.4952305371069294,
      "learning_rate": 9.122771919227568e-07,
      "log_odds_chosen": 1.8798828125,
      "log_odds_ratio": -0.3134765625,
      "logits/chosen": -1.1427733898162842,
      "logits/rejected": -0.9931640625,
      "logps/chosen": -0.606738269329071,
      "logps/rejected": -1.9249999523162842,
      "loss": 0.6539,
      "nll_loss": 0.5835937261581421,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06072998046875,
      "rewards/margins": 0.13166503608226776,
      "rewards/rejected": -0.19245605170726776,
      "step": 7690
    },
    {
      "epoch": 0.993676603432701,
      "grad_norm": 3.0173376398159015,
      "learning_rate": 9.116846116771035e-07,
      "log_odds_chosen": 1.850000023841858,
      "log_odds_ratio": -0.3314453065395355,
      "logits/chosen": -1.111718773841858,
      "logits/rejected": -0.981249988079071,
      "logps/chosen": -0.6822265386581421,
      "logps/rejected": -1.986718773841858,
      "loss": 0.6544,
      "nll_loss": 0.6654297113418579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06821288913488388,
      "rewards/margins": 0.13044433295726776,
      "rewards/rejected": -0.19873046875,
      "step": 7700
    },
    {
      "epoch": 0.9949670925280681,
      "grad_norm": 2.9708126759122044,
      "learning_rate": 9.110931846854553e-07,
      "log_odds_chosen": 1.8654663562774658,
      "log_odds_ratio": -0.32329100370407104,
      "logits/chosen": -1.104882836341858,
      "logits/rejected": -1.0095703601837158,
      "logps/chosen": -0.631054699420929,
      "logps/rejected": -1.912109375,
      "loss": 0.6458,
      "nll_loss": 0.5770508050918579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06300048530101776,
      "rewards/margins": 0.12813416123390198,
      "rewards/rejected": -0.19111327826976776,
      "step": 7710
    },
    {
      "epoch": 0.9962575816234353,
      "grad_norm": 3.209392275968329,
      "learning_rate": 9.105029072119708e-07,
      "log_odds_chosen": 1.895410180091858,
      "log_odds_ratio": -0.3289550840854645,
      "logits/chosen": -1.130273461341858,
      "logits/rejected": -0.9703124761581421,
      "logps/chosen": -0.608691394329071,
      "logps/rejected": -1.9140625,
      "loss": 0.6609,
      "nll_loss": 0.612597644329071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06088867038488388,
      "rewards/margins": 0.13044433295726776,
      "rewards/rejected": -0.19150391221046448,
      "step": 7720
    },
    {
      "epoch": 0.9975480707188025,
      "grad_norm": 3.2211524608055786,
      "learning_rate": 9.099137755377291e-07,
      "log_odds_chosen": 1.773828148841858,
      "log_odds_ratio": -0.365478515625,
      "logits/chosen": -1.101171851158142,
      "logits/rejected": -0.974804699420929,
      "logps/chosen": -0.632617175579071,
      "logps/rejected": -1.881250023841858,
      "loss": 0.6647,
      "nll_loss": 0.574902355670929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06324462592601776,
      "rewards/margins": 0.12482909858226776,
      "rewards/rejected": -0.18793945014476776,
      "step": 7730
    },
    {
      "epoch": 0.9988385598141696,
      "grad_norm": 3.5050265764396373,
      "learning_rate": 9.093257859606311e-07,
      "log_odds_chosen": 1.765625,
      "log_odds_ratio": -0.332275390625,
      "logits/chosen": -1.0896484851837158,
      "logits/rejected": -1.011132836341858,
      "logps/chosen": -0.623242199420929,
      "logps/rejected": -1.8132812976837158,
      "loss": 0.6394,
      "nll_loss": 0.6415039300918579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06226806715130806,
      "rewards/margins": 0.11907958984375,
      "rewards/rejected": -0.1812744140625,
      "step": 7740
    },
    {
      "epoch": 1.0,
      "step": 7749,
      "total_flos": 0.0,
      "train_loss": 0.8908563570324598,
      "train_runtime": 50080.4212,
      "train_samples_per_second": 19.805,
      "train_steps_per_second": 0.155
    }
  ],
  "logging_steps": 10,
  "max_steps": 7749,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 5000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}