{
  "best_metric": 1.046362042427063,
  "best_model_checkpoint": "models/llama3.2-3b-orpo-finegrained/checkpoint-10000",
  "epoch": 0.9999635741084762,
  "eval_steps": 10000,
  "global_step": 13726,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0007285178304739008,
      "grad_norm": 15.9533663959914,
      "learning_rate": 5.000000000000001e-07,
      "log_odds_chosen": 0.2954650819301605,
      "log_odds_ratio": -0.6751953363418579,
      "logits/chosen": -0.976367175579071,
      "logits/rejected": -0.9677734375,
      "logps/chosen": -1.923437476158142,
      "logps/rejected": -2.193359375,
      "loss": 2.467,
      "nll_loss": 2.473437547683716,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.09619140625,
      "rewards/margins": 0.013394164852797985,
      "rewards/rejected": -0.10952148586511612,
      "step": 10
    },
    {
      "epoch": 0.0014570356609478017,
      "grad_norm": 9.146989672403897,
      "learning_rate": 1.0000000000000002e-06,
      "log_odds_chosen": 0.42283934354782104,
      "log_odds_ratio": -0.636523425579071,
      "logits/chosen": -1.0166015625,
      "logits/rejected": -1.0060546398162842,
      "logps/chosen": -1.7492187023162842,
      "logps/rejected": -2.1128907203674316,
      "loss": 2.3041,
      "nll_loss": 2.2210936546325684,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.08741454780101776,
      "rewards/margins": 0.01822357252240181,
      "rewards/rejected": -0.10561523586511612,
      "step": 20
    },
    {
      "epoch": 0.0021855534914217026,
      "grad_norm": 8.431720247071095,
      "learning_rate": 1.5e-06,
      "log_odds_chosen": 0.32763671875,
      "log_odds_ratio": -0.615429699420929,
      "logits/chosen": -1.1181640625,
      "logits/rejected": -1.0593750476837158,
      "logps/chosen": -1.4753906726837158,
      "logps/rejected": -1.745703101158142,
      "loss": 2.0254,
      "nll_loss": 2.003124952316284,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07374267280101776,
      "rewards/margins": 0.01353607140481472,
      "rewards/rejected": -0.08720703423023224,
      "step": 30
    },
    {
      "epoch": 0.0029140713218956033,
      "grad_norm": 3.070871995535844,
      "learning_rate": 2.0000000000000003e-06,
      "log_odds_chosen": 0.26588743925094604,
      "log_odds_ratio": -0.638378918170929,
      "logits/chosen": -1.220312476158142,
      "logits/rejected": -1.187890648841858,
      "logps/chosen": -1.2716796398162842,
      "logps/rejected": -1.4753906726837158,
      "loss": 1.6893,
      "nll_loss": 1.564453125,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06356201320886612,
      "rewards/margins": 0.01025543175637722,
      "rewards/rejected": -0.07381591945886612,
      "step": 40
    },
    {
      "epoch": 0.003642589152369504,
      "grad_norm": 2.50519684835902,
      "learning_rate": 2.5e-06,
      "log_odds_chosen": 0.2961181700229645,
      "log_odds_ratio": -0.6336914300918579,
      "logits/chosen": -1.087304711341858,
      "logits/rejected": -1.026757836341858,
      "logps/chosen": -1.1375000476837158,
      "logps/rejected": -1.370703101158142,
      "loss": 1.5608,
      "nll_loss": 1.4910156726837158,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.05686035007238388,
      "rewards/margins": 0.01169738732278347,
      "rewards/rejected": -0.06850586086511612,
      "step": 50
    },
    {
      "epoch": 0.004371106982843405,
      "grad_norm": 2.4890297217545614,
      "learning_rate": 3e-06,
      "log_odds_chosen": 0.23422852158546448,
      "log_odds_ratio": -0.654101550579071,
      "logits/chosen": -0.986132800579071,
      "logits/rejected": -0.9429687261581421,
      "logps/chosen": -1.1218750476837158,
      "logps/rejected": -1.3017578125,
      "loss": 1.5064,
      "nll_loss": 1.439453125,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.05606689304113388,
      "rewards/margins": 0.009020996280014515,
      "rewards/rejected": -0.06510009616613388,
      "step": 60
    },
    {
      "epoch": 0.0050996248133173055,
      "grad_norm": 2.421992764968879,
      "learning_rate": 3.5e-06,
      "log_odds_chosen": 0.15443114936351776,
      "log_odds_ratio": -0.6845703125,
      "logits/chosen": -0.953906238079071,
      "logits/rejected": -0.9349609613418579,
      "logps/chosen": -1.076757788658142,
      "logps/rejected": -1.2019531726837158,
      "loss": 1.4832,
      "nll_loss": 1.478124976158142,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.05384521558880806,
      "rewards/margins": 0.006236267276108265,
      "rewards/rejected": -0.06009521335363388,
      "step": 70
    },
    {
      "epoch": 0.005828142643791207,
      "grad_norm": 2.190954187789956,
      "learning_rate": 4.000000000000001e-06,
      "log_odds_chosen": 0.35310059785842896,
      "log_odds_ratio": -0.6044921875,
      "logits/chosen": -1.127539038658142,
      "logits/rejected": -1.0302734375,
      "logps/chosen": -1.012304663658142,
      "logps/rejected": -1.274804711341858,
      "loss": 1.4286,
      "nll_loss": 1.3318359851837158,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05062256008386612,
      "rewards/margins": 0.01313095074146986,
      "rewards/rejected": -0.063720703125,
      "step": 80
    },
    {
      "epoch": 0.006556660474265108,
      "grad_norm": 2.3217044861060985,
      "learning_rate": 4.5e-06,
      "log_odds_chosen": 0.2669677734375,
      "log_odds_ratio": -0.6327148675918579,
      "logits/chosen": -1.176171898841858,
      "logits/rejected": -1.102148413658142,
      "logps/chosen": -0.986132800579071,
      "logps/rejected": -1.180078148841858,
      "loss": 1.4013,
      "nll_loss": 1.3683593273162842,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04930419847369194,
      "rewards/margins": 0.00968170166015625,
      "rewards/rejected": -0.05898437649011612,
      "step": 90
    },
    {
      "epoch": 0.007285178304739008,
      "grad_norm": 2.1534708677073717,
      "learning_rate": 5e-06,
      "log_odds_chosen": 0.2549194395542145,
      "log_odds_ratio": -0.6446288824081421,
      "logits/chosen": -1.109960913658142,
      "logits/rejected": -1.0662109851837158,
      "logps/chosen": -0.976757824420929,
      "logps/rejected": -1.1707031726837158,
      "loss": 1.3713,
      "nll_loss": 1.378515601158142,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.04885254055261612,
      "rewards/margins": 0.00968856830149889,
      "rewards/rejected": -0.05852050706744194,
      "step": 100
    },
    {
      "epoch": 0.00801369613521291,
      "grad_norm": 2.2876112609998844,
      "learning_rate": 4.767312946227961e-06,
      "log_odds_chosen": 0.19966430962085724,
      "log_odds_ratio": -0.647265613079071,
      "logits/chosen": -1.0193359851837158,
      "logits/rejected": -0.9755859375,
      "logps/chosen": -0.9624999761581421,
      "logps/rejected": -1.107812523841858,
      "loss": 1.3473,
      "nll_loss": 1.3552734851837158,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.04812011867761612,
      "rewards/margins": 0.0072647095657885075,
      "rewards/rejected": -0.05539550632238388,
      "step": 110
    },
    {
      "epoch": 0.00874221396568681,
      "grad_norm": 2.314651639607303,
      "learning_rate": 4.564354645876385e-06,
      "log_odds_chosen": 0.29967039823532104,
      "log_odds_ratio": -0.6214843988418579,
      "logits/chosen": -1.014062523841858,
      "logits/rejected": -0.935742199420929,
      "logps/chosen": -0.901171863079071,
      "logps/rejected": -1.1179687976837158,
      "loss": 1.3222,
      "nll_loss": 1.2853515148162842,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.04508056491613388,
      "rewards/margins": 0.01084747351706028,
      "rewards/rejected": -0.055908203125,
      "step": 120
    },
    {
      "epoch": 0.009470731796160712,
      "grad_norm": 1.887787158393894,
      "learning_rate": 4.385290096535147e-06,
      "log_odds_chosen": 0.17949219048023224,
      "log_odds_ratio": -0.686328113079071,
      "logits/chosen": -1.0271484851837158,
      "logits/rejected": -0.978710949420929,
      "logps/chosen": -0.8705078363418579,
      "logps/rejected": -1.015039086341858,
      "loss": 1.316,
      "nll_loss": 1.248437523841858,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04349365085363388,
      "rewards/margins": 0.0072456360794603825,
      "rewards/rejected": -0.05074463039636612,
      "step": 130
    },
    {
      "epoch": 0.010199249626634611,
      "grad_norm": 1.7734017639775328,
      "learning_rate": 4.2257712736425835e-06,
      "log_odds_chosen": 0.17213134467601776,
      "log_odds_ratio": -0.685351550579071,
      "logits/chosen": -1.0166015625,
      "logits/rejected": -0.9496093988418579,
      "logps/chosen": -0.8832031488418579,
      "logps/rejected": -1.0109374523162842,
      "loss": 1.2749,
      "nll_loss": 1.3035156726837158,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.04417724534869194,
      "rewards/margins": 0.00638656597584486,
      "rewards/rejected": -0.050537109375,
      "step": 140
    },
    {
      "epoch": 0.010927767457108512,
      "grad_norm": 1.6134667428770773,
      "learning_rate": 4.082482904638631e-06,
      "log_odds_chosen": 0.20954589545726776,
      "log_odds_ratio": -0.6512695550918579,
      "logits/chosen": -1.0134766101837158,
      "logits/rejected": -0.9515625238418579,
      "logps/chosen": -0.832812488079071,
      "logps/rejected": -0.982617199420929,
      "loss": 1.2442,
      "nll_loss": 1.21484375,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.04163818433880806,
      "rewards/margins": 0.0074897767044603825,
      "rewards/rejected": -0.04915771633386612,
      "step": 150
    },
    {
      "epoch": 0.011656285287582413,
      "grad_norm": 1.4461584051638796,
      "learning_rate": 3.952847075210474e-06,
      "log_odds_chosen": 0.22348633408546448,
      "log_odds_ratio": -0.6573241949081421,
      "logits/chosen": -1.082421898841858,
      "logits/rejected": -1.0076172351837158,
      "logps/chosen": -0.8267577886581421,
      "logps/rejected": -0.9984375238418579,
      "loss": 1.2417,
      "nll_loss": 1.227929711341858,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.04135742038488388,
      "rewards/margins": 0.00859680213034153,
      "rewards/rejected": -0.04993896558880806,
      "step": 160
    },
    {
      "epoch": 0.012384803118056315,
      "grad_norm": 1.5056033737559291,
      "learning_rate": 3.834824944236852e-06,
      "log_odds_chosen": 0.2266845703125,
      "log_odds_ratio": -0.683300793170929,
      "logits/chosen": -1.0603516101837158,
      "logits/rejected": -1.0134766101837158,
      "logps/chosen": -0.841601550579071,
      "logps/rejected": -1.0148437023162842,
      "loss": 1.2321,
      "nll_loss": 1.1408202648162842,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.04204712063074112,
      "rewards/margins": 0.008634185418486595,
      "rewards/rejected": -0.05073242262005806,
      "step": 170
    },
    {
      "epoch": 0.013113320948530216,
      "grad_norm": 1.4451374897190143,
      "learning_rate": 3.72677996249965e-06,
      "log_odds_chosen": 0.19724121689796448,
      "log_odds_ratio": -0.6943359375,
      "logits/chosen": -1.0080077648162842,
      "logits/rejected": -0.935742199420929,
      "logps/chosen": -0.8080078363418579,
      "logps/rejected": -0.9677734375,
      "loss": 1.2049,
      "nll_loss": 1.152734398841858,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0404052734375,
      "rewards/margins": 0.00798721332103014,
      "rewards/rejected": -0.04840087890625,
      "step": 180
    },
    {
      "epoch": 0.013841838779004117,
      "grad_norm": 1.3906807004536919,
      "learning_rate": 3.6273812505500587e-06,
      "log_odds_chosen": 0.3205322325229645,
      "log_odds_ratio": -0.6629883050918579,
      "logits/chosen": -0.980273425579071,
      "logits/rejected": -0.9029296636581421,
      "logps/chosen": -0.822070300579071,
      "logps/rejected": -1.078710913658142,
      "loss": 1.2027,
      "nll_loss": 1.2169921398162842,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.04110107570886612,
      "rewards/margins": 0.012827301397919655,
      "rewards/rejected": -0.05390625074505806,
      "step": 190
    },
    {
      "epoch": 0.014570356609478016,
      "grad_norm": 1.3505200453009385,
      "learning_rate": 3.5355339059327378e-06,
      "log_odds_chosen": 0.33245849609375,
      "log_odds_ratio": -0.6133788824081421,
      "logits/chosen": -0.952343761920929,
      "logits/rejected": -0.9437500238418579,
      "logps/chosen": -0.7802734375,
      "logps/rejected": -0.995312511920929,
      "loss": 1.1788,
      "nll_loss": 1.1466796398162842,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.03899536281824112,
      "rewards/margins": 0.0107421875,
      "rewards/rejected": -0.04971923679113388,
      "step": 200
    },
    {
      "epoch": 0.015298874439951917,
      "grad_norm": 1.3769145316727318,
      "learning_rate": 3.450327796711771e-06,
      "log_odds_chosen": 0.3091674745082855,
      "log_odds_ratio": -0.63671875,
      "logits/chosen": -1.0177733898162842,
      "logits/rejected": -0.934765636920929,
      "logps/chosen": -0.775195300579071,
      "logps/rejected": -0.9927734136581421,
      "loss": 1.1995,
      "nll_loss": 1.137304663658142,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.03872070461511612,
      "rewards/margins": 0.01094207726418972,
      "rewards/rejected": -0.0496826171875,
      "step": 210
    },
    {
      "epoch": 0.01602739227042582,
      "grad_norm": 1.543013555956433,
      "learning_rate": 3.3709993123162106e-06,
      "log_odds_chosen": 0.40642088651657104,
      "log_odds_ratio": -0.6234375238418579,
      "logits/chosen": -0.9691406488418579,
      "logits/rejected": -0.911914050579071,
      "logps/chosen": -0.8099609613418579,
      "logps/rejected": -1.095703125,
      "loss": 1.2125,
      "nll_loss": 1.215234398841858,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.04050292819738388,
      "rewards/margins": 0.01429672259837389,
      "rewards/rejected": -0.05484618991613388,
      "step": 220
    },
    {
      "epoch": 0.016755910100899718,
      "grad_norm": 1.8438357705892283,
      "learning_rate": 3.296902366978936e-06,
      "log_odds_chosen": 0.3792968690395355,
      "log_odds_ratio": -0.60498046875,
      "logits/chosen": -0.9798828363418579,
      "logits/rejected": -0.9068359136581421,
      "logps/chosen": -0.8228515386581421,
      "logps/rejected": -1.0859375,
      "loss": 1.2124,
      "nll_loss": 1.1603515148162842,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.04111327975988388,
      "rewards/margins": 0.013135528191924095,
      "rewards/rejected": -0.05428466945886612,
      "step": 230
    },
    {
      "epoch": 0.01748442793137362,
      "grad_norm": 1.502639535957259,
      "learning_rate": 3.2274861218395142e-06,
      "log_odds_chosen": 0.36455076932907104,
      "log_odds_ratio": -0.6126953363418579,
      "logits/chosen": -0.9634765386581421,
      "logits/rejected": -0.921875,
      "logps/chosen": -0.788867175579071,
      "logps/rejected": -1.038476586341858,
      "loss": 1.1361,
      "nll_loss": 1.080078125,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.03945312649011612,
      "rewards/margins": 0.012447357177734375,
      "rewards/rejected": -0.05192871019244194,
      "step": 240
    },
    {
      "epoch": 0.01821294576184752,
      "grad_norm": 1.543004101903392,
      "learning_rate": 3.1622776601683796e-06,
      "log_odds_chosen": 0.38410645723342896,
      "log_odds_ratio": -0.622753918170929,
      "logits/chosen": -0.890429675579071,
      "logits/rejected": -0.822460949420929,
      "logps/chosen": -0.748242199420929,
      "logps/rejected": -1.016210913658142,
      "loss": 1.172,
      "nll_loss": 1.110937476158142,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.03739624097943306,
      "rewards/margins": 0.013417053036391735,
      "rewards/rejected": -0.05081786960363388,
      "step": 250
    },
    {
      "epoch": 0.018941463592321423,
      "grad_norm": 1.5510517402518604,
      "learning_rate": 3.1008683647302113e-06,
      "log_odds_chosen": 0.33720701932907104,
      "log_odds_ratio": -0.6124023199081421,
      "logits/chosen": -0.851757824420929,
      "logits/rejected": -0.8179687261581421,
      "logps/chosen": -0.7197265625,
      "logps/rejected": -0.9339843988418579,
      "loss": 1.145,
      "nll_loss": 1.122656226158142,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.03603515774011612,
      "rewards/margins": 0.010726546868681908,
      "rewards/rejected": -0.04672851413488388,
      "step": 260
    },
    {
      "epoch": 0.019669981422795323,
      "grad_norm": 1.5208767512479846,
      "learning_rate": 3.0429030972509227e-06,
      "log_odds_chosen": 0.28759765625,
      "log_odds_ratio": -0.6490234136581421,
      "logits/chosen": -0.8404296636581421,
      "logits/rejected": -0.780078113079071,
      "logps/chosen": -0.7542968988418579,
      "logps/rejected": -0.9654296636581421,
      "loss": 1.1325,
      "nll_loss": 1.1052734851837158,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.03769531100988388,
      "rewards/margins": 0.01059799175709486,
      "rewards/rejected": -0.04829101637005806,
      "step": 270
    },
    {
      "epoch": 0.020398499253269222,
      "grad_norm": 1.3965265108613503,
      "learning_rate": 2.988071523335984e-06,
      "log_odds_chosen": 0.39448243379592896,
      "log_odds_ratio": -0.6200195550918579,
      "logits/chosen": -0.8578125238418579,
      "logits/rejected": -0.80859375,
      "logps/chosen": -0.738476574420929,
      "logps/rejected": -1.029882788658142,
      "loss": 1.1289,
      "nll_loss": 1.0906250476837158,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.03693847730755806,
      "rewards/margins": 0.014559554867446423,
      "rewards/rejected": -0.05147094652056694,
      "step": 280
    },
    {
      "epoch": 0.021127017083743125,
      "grad_norm": 1.3739087170811393,
      "learning_rate": 2.9361010975735177e-06,
      "log_odds_chosen": 0.29168701171875,
      "log_odds_ratio": -0.6351562738418579,
      "logits/chosen": -0.898632824420929,
      "logits/rejected": -0.8382812738418579,
      "logps/chosen": -0.7421875,
      "logps/rejected": -0.9369140863418579,
      "loss": 1.1082,
      "nll_loss": 1.070703148841858,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.037109375,
      "rewards/margins": 0.009739684872329235,
      "rewards/rejected": -0.04686279222369194,
      "step": 290
    },
    {
      "epoch": 0.021855534914217024,
      "grad_norm": 1.3201088890045447,
      "learning_rate": 2.8867513459481293e-06,
      "log_odds_chosen": 0.36787110567092896,
      "log_odds_ratio": -0.604785144329071,
      "logits/chosen": -0.857421875,
      "logits/rejected": -0.8080078363418579,
      "logps/chosen": -0.683789074420929,
      "logps/rejected": -0.9228515625,
      "loss": 1.1064,
      "nll_loss": 1.1583983898162842,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03424072265625,
      "rewards/margins": 0.011936950497329235,
      "rewards/rejected": -0.046142578125,
      "step": 300
    },
    {
      "epoch": 0.022584052744690927,
      "grad_norm": 1.3403248552953595,
      "learning_rate": 2.839809171235324e-06,
      "log_odds_chosen": 0.25688475370407104,
      "log_odds_ratio": -0.6685546636581421,
      "logits/chosen": -0.958984375,
      "logits/rejected": -0.896679699420929,
      "logps/chosen": -0.7525390386581421,
      "logps/rejected": -0.913867175579071,
      "loss": 1.1092,
      "nll_loss": 1.046289086341858,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.03764648362994194,
      "rewards/margins": 0.008044433780014515,
      "rewards/rejected": -0.04570312425494194,
      "step": 310
    },
    {
      "epoch": 0.023312570575164827,
      "grad_norm": 1.47469653888456,
      "learning_rate": 2.7950849718747376e-06,
      "log_odds_chosen": 0.21955566108226776,
      "log_odds_ratio": -0.6854492425918579,
      "logits/chosen": -0.9771484136581421,
      "logits/rejected": -0.9058593511581421,
      "logps/chosen": -0.7289062738418579,
      "logps/rejected": -0.8779296875,
      "loss": 1.1085,
      "nll_loss": 1.06640625,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.03643188625574112,
      "rewards/margins": 0.0074668885208666325,
      "rewards/rejected": -0.04390258714556694,
      "step": 320
    },
    {
      "epoch": 0.02404108840563873,
      "grad_norm": 1.3979630722657226,
      "learning_rate": 2.752409412815902e-06,
      "log_odds_chosen": 0.3363403379917145,
      "log_odds_ratio": -0.6353515386581421,
      "logits/chosen": -0.9603515863418579,
      "logits/rejected": -0.90625,
      "logps/chosen": -0.76171875,
      "logps/rejected": -0.973828136920929,
      "loss": 1.1122,
      "nll_loss": 1.0750000476837158,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.03810424730181694,
      "rewards/margins": 0.01054534874856472,
      "rewards/rejected": -0.04862060397863388,
      "step": 330
    },
    {
      "epoch": 0.02476960623611263,
      "grad_norm": 1.3492442452036264,
      "learning_rate": 2.711630722733202e-06,
      "log_odds_chosen": 0.3592773377895355,
      "log_odds_ratio": -0.622363269329071,
      "logits/chosen": -0.9677734375,
      "logits/rejected": -0.894726574420929,
      "logps/chosen": -0.701953113079071,
      "logps/rejected": -0.931835949420929,
      "loss": 1.1117,
      "nll_loss": 1.092187523841858,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.03508300706744194,
      "rewards/margins": 0.011486816219985485,
      "rewards/rejected": -0.04661254957318306,
      "step": 340
    },
    {
      "epoch": 0.02549812406658653,
      "grad_norm": 1.4993641207316597,
      "learning_rate": 2.6726124191242444e-06,
      "log_odds_chosen": 0.3841308653354645,
      "log_odds_ratio": -0.6231445074081421,
      "logits/chosen": -0.982226550579071,
      "logits/rejected": -0.935351550579071,
      "logps/chosen": -0.6690429449081421,
      "logps/rejected": -0.9007812738418579,
      "loss": 1.0516,
      "nll_loss": 0.934374988079071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.03345947340130806,
      "rewards/margins": 0.01158828753978014,
      "rewards/rejected": -0.04506836086511612,
      "step": 350
    },
    {
      "epoch": 0.02622664189706043,
      "grad_norm": 1.3391568428716587,
      "learning_rate": 2.6352313834736496e-06,
      "log_odds_chosen": 0.3044067323207855,
      "log_odds_ratio": -0.628125011920929,
      "logits/chosen": -0.9341796636581421,
      "logits/rejected": -0.875195324420929,
      "logps/chosen": -0.688671886920929,
      "logps/rejected": -0.877734363079071,
      "loss": 1.0774,
      "nll_loss": 1.0439453125,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03446044772863388,
      "rewards/margins": 0.00945129431784153,
      "rewards/rejected": -0.04388427734375,
      "step": 360
    },
    {
      "epoch": 0.02695515972753433,
      "grad_norm": 1.3340248414473945,
      "learning_rate": 2.599376224550182e-06,
      "log_odds_chosen": 0.24779053032398224,
      "log_odds_ratio": -0.706835925579071,
      "logits/chosen": -0.9351562261581421,
      "logits/rejected": -0.8746093511581421,
      "logps/chosen": -0.6943359375,
      "logps/rejected": -0.830273449420929,
      "loss": 1.0856,
      "nll_loss": 1.149804711341858,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.034698486328125,
      "rewards/margins": 0.0068038939498364925,
      "rewards/rejected": -0.04151000827550888,
      "step": 370
    },
    {
      "epoch": 0.027683677558008234,
      "grad_norm": 1.268560690934275,
      "learning_rate": 2.564945880212886e-06,
      "log_odds_chosen": 0.3297363221645355,
      "log_odds_ratio": -0.6346679925918579,
      "logits/chosen": -0.881640613079071,
      "logits/rejected": -0.8541015386581421,
      "logps/chosen": -0.6539062261581421,
      "logps/rejected": -0.834765613079071,
      "loss": 1.0982,
      "nll_loss": 1.055078148841858,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.03270263597369194,
      "rewards/margins": 0.009056473150849342,
      "rewards/rejected": -0.041748046875,
      "step": 380
    },
    {
      "epoch": 0.028412195388482133,
      "grad_norm": 1.301127740283187,
      "learning_rate": 2.5318484177091667e-06,
      "log_odds_chosen": 0.21931152045726776,
      "log_odds_ratio": -0.696093738079071,
      "logits/chosen": -0.940625011920929,
      "logits/rejected": -0.8714843988418579,
      "logps/chosen": -0.6600586175918579,
      "logps/rejected": -0.7972656488418579,
      "loss": 1.0435,
      "nll_loss": 1.0343749523162842,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.032989501953125,
      "rewards/margins": 0.0068763731978833675,
      "rewards/rejected": -0.039886474609375,
      "step": 390
    },
    {
      "epoch": 0.029140713218956033,
      "grad_norm": 1.197208167024546,
      "learning_rate": 2.5e-06,
      "log_odds_chosen": 0.40928953886032104,
      "log_odds_ratio": -0.602832019329071,
      "logits/chosen": -0.8607422113418579,
      "logits/rejected": -0.8184570074081421,
      "logps/chosen": -0.63720703125,
      "logps/rejected": -0.8802734613418579,
      "loss": 1.0667,
      "nll_loss": 1.038476586341858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03187866136431694,
      "rewards/margins": 0.01215286273509264,
      "rewards/rejected": -0.043975830078125,
      "step": 400
    },
    {
      "epoch": 0.029869231049429935,
      "grad_norm": 1.387387351733591,
      "learning_rate": 2.4693239916239746e-06,
      "log_odds_chosen": 0.3548950254917145,
      "log_odds_ratio": -0.6070312261581421,
      "logits/chosen": -0.9105468988418579,
      "logits/rejected": -0.828906238079071,
      "logps/chosen": -0.66552734375,
      "logps/rejected": -0.875195324420929,
      "loss": 1.0384,
      "nll_loss": 1.0236327648162842,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.033294677734375,
      "rewards/margins": 0.010447311215102673,
      "rewards/rejected": -0.043731689453125,
      "step": 410
    },
    {
      "epoch": 0.030597748879903835,
      "grad_norm": 1.1630171778797551,
      "learning_rate": 2.4397501823713327e-06,
      "log_odds_chosen": 0.42451173067092896,
      "log_odds_ratio": -0.5966796875,
      "logits/chosen": -0.9609375,
      "logits/rejected": -0.8785156011581421,
      "logps/chosen": -0.6585937738418579,
      "logps/rejected": -0.9126952886581421,
      "loss": 1.0221,
      "nll_loss": 1.020898461341858,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03291625902056694,
      "rewards/margins": 0.01271896343678236,
      "rewards/rejected": -0.04562988132238388,
      "step": 420
    },
    {
      "epoch": 0.031326266710377734,
      "grad_norm": 1.3827011581075224,
      "learning_rate": 2.411214110852061e-06,
      "log_odds_chosen": 0.21364745497703552,
      "log_odds_ratio": -0.6751953363418579,
      "logits/chosen": -0.83984375,
      "logits/rejected": -0.773242175579071,
      "logps/chosen": -0.6494140625,
      "logps/rejected": -0.7650390863418579,
      "loss": 1.0467,
      "nll_loss": 1.0642578601837158,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.03246460109949112,
      "rewards/margins": 0.005752944853156805,
      "rewards/rejected": -0.0382080078125,
      "step": 430
    },
    {
      "epoch": 0.03205478454085164,
      "grad_norm": 1.3180867107278555,
      "learning_rate": 2.3836564731139807e-06,
      "log_odds_chosen": 0.21593017876148224,
      "log_odds_ratio": -0.686718761920929,
      "logits/chosen": -0.893750011920929,
      "logits/rejected": -0.833203136920929,
      "logps/chosen": -0.6954101324081421,
      "logps/rejected": -0.8447265625,
      "loss": 1.0651,
      "nll_loss": 1.028906226158142,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.03477783128619194,
      "rewards/margins": 0.00748367328196764,
      "rewards/rejected": -0.04226074367761612,
      "step": 440
    },
    {
      "epoch": 0.03278330237132554,
      "grad_norm": 1.265756059653912,
      "learning_rate": 2.357022603955159e-06,
      "log_odds_chosen": 0.289794921875,
      "log_odds_ratio": -0.6695312261581421,
      "logits/chosen": -0.970507800579071,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.6343749761581421,
      "logps/rejected": -0.8138672113418579,
      "loss": 1.0324,
      "nll_loss": 0.9560546875,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.03172607347369194,
      "rewards/margins": 0.008957291021943092,
      "rewards/rejected": -0.04069824144244194,
      "step": 450
    },
    {
      "epoch": 0.033511820201799436,
      "grad_norm": 1.1721807886073912,
      "learning_rate": 2.3312620206007847e-06,
      "log_odds_chosen": 0.4729858338832855,
      "log_odds_ratio": -0.5615234375,
      "logits/chosen": -0.873828113079071,
      "logits/rejected": -0.824414074420929,
      "logps/chosen": -0.6048828363418579,
      "logps/rejected": -0.864062488079071,
      "loss": 1.0435,
      "nll_loss": 0.9820312261581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.03023071214556694,
      "rewards/margins": 0.012997055426239967,
      "rewards/rejected": -0.04321899265050888,
      "step": 460
    },
    {
      "epoch": 0.03424033803227334,
      "grad_norm": 1.313478833950966,
      "learning_rate": 2.3063280200722128e-06,
      "log_odds_chosen": 0.4706664979457855,
      "log_odds_ratio": -0.568554699420929,
      "logits/chosen": -0.847460925579071,
      "logits/rejected": -0.7757812738418579,
      "logps/chosen": -0.583691418170929,
      "logps/rejected": -0.8316406011581421,
      "loss": 1.0555,
      "nll_loss": 0.9671875238418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.02919921837747097,
      "rewards/margins": 0.012379837222397327,
      "rewards/rejected": -0.04160766676068306,
      "step": 470
    },
    {
      "epoch": 0.03496885586274724,
      "grad_norm": 1.2348450657636334,
      "learning_rate": 2.2821773229381924e-06,
      "log_odds_chosen": 0.43181151151657104,
      "log_odds_ratio": -0.61279296875,
      "logits/chosen": -0.8802734613418579,
      "logits/rejected": -0.8076171875,
      "logps/chosen": -0.6407226324081421,
      "logps/rejected": -0.9009765386581421,
      "loss": 1.0452,
      "nll_loss": 1.006250023841858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03204955905675888,
      "rewards/margins": 0.013011932373046875,
      "rewards/rejected": -0.04505004733800888,
      "step": 480
    },
    {
      "epoch": 0.035697373693221145,
      "grad_norm": 1.1863626864617351,
      "learning_rate": 2.2587697572631284e-06,
      "log_odds_chosen": 0.4403930604457855,
      "log_odds_ratio": -0.607714831829071,
      "logits/chosen": -0.9449218511581421,
      "logits/rejected": -0.885937511920929,
      "logps/chosen": -0.6298828125,
      "logps/rejected": -0.8828125,
      "loss": 1.0427,
      "nll_loss": 0.975781261920929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03147583082318306,
      "rewards/margins": 0.01268081646412611,
      "rewards/rejected": -0.04415283352136612,
      "step": 490
    },
    {
      "epoch": 0.03642589152369504,
      "grad_norm": 1.2402609256449026,
      "learning_rate": 2.23606797749979e-06,
      "log_odds_chosen": 0.24985352158546448,
      "log_odds_ratio": -0.670214831829071,
      "logits/chosen": -0.955273449420929,
      "logits/rejected": -0.900390625,
      "logps/chosen": -0.653613269329071,
      "logps/rejected": -0.797070324420929,
      "loss": 1.0709,
      "nll_loss": 1.009179711341858,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.03267822414636612,
      "rewards/margins": 0.0072158812545239925,
      "rewards/rejected": -0.03987426683306694,
      "step": 500
    },
    {
      "epoch": 0.037154409354168944,
      "grad_norm": 1.2130439008477287,
      "learning_rate": 2.2140372138502386e-06,
      "log_odds_chosen": 0.3489623963832855,
      "log_odds_ratio": -0.643261730670929,
      "logits/chosen": -0.9111328125,
      "logits/rejected": -0.861328125,
      "logps/chosen": -0.6636718511581421,
      "logps/rejected": -0.863476574420929,
      "loss": 1.045,
      "nll_loss": 1.0128905773162842,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.03316650539636612,
      "rewards/margins": 0.00996551476418972,
      "rewards/rejected": -0.04315795749425888,
      "step": 510
    },
    {
      "epoch": 0.037882927184642846,
      "grad_norm": 1.2682492599096589,
      "learning_rate": 2.1926450482675734e-06,
      "log_odds_chosen": 0.35509032011032104,
      "log_odds_ratio": -0.6309570074081421,
      "logits/chosen": -0.907421886920929,
      "logits/rejected": -0.8353515863418579,
      "logps/chosen": -0.62841796875,
      "logps/rejected": -0.8119140863418579,
      "loss": 1.054,
      "nll_loss": 1.068945288658142,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.031402587890625,
      "rewards/margins": 0.009177779778838158,
      "rewards/rejected": -0.04057617112994194,
      "step": 520
    },
    {
      "epoch": 0.03861144501511674,
      "grad_norm": 1.2865582024074884,
      "learning_rate": 2.1718612138153473e-06,
      "log_odds_chosen": 0.4597412049770355,
      "log_odds_ratio": -0.613574206829071,
      "logits/chosen": -0.9609375,
      "logits/rejected": -0.872851550579071,
      "logps/chosen": -0.6480468511581421,
      "logps/rejected": -0.922656238079071,
      "loss": 1.0473,
      "nll_loss": 0.980664074420929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03239746019244194,
      "rewards/margins": 0.01371078472584486,
      "rewards/rejected": -0.04609375074505806,
      "step": 530
    },
    {
      "epoch": 0.039339962845590645,
      "grad_norm": 1.2365779595828863,
      "learning_rate": 2.151657414559676e-06,
      "log_odds_chosen": 0.41510009765625,
      "log_odds_ratio": -0.612988293170929,
      "logits/chosen": -0.9632812738418579,
      "logits/rejected": -0.9150390625,
      "logps/chosen": -0.616894543170929,
      "logps/rejected": -0.851757824420929,
      "loss": 1.039,
      "nll_loss": 0.962109386920929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.03086547926068306,
      "rewards/margins": 0.01172485388815403,
      "rewards/rejected": -0.04257812350988388,
      "step": 540
    },
    {
      "epoch": 0.04006848067606455,
      "grad_norm": 1.199439111420552,
      "learning_rate": 2.132007163556104e-06,
      "log_odds_chosen": 0.42926025390625,
      "log_odds_ratio": -0.6141601800918579,
      "logits/chosen": -0.9193359613418579,
      "logits/rejected": -0.8560546636581421,
      "logps/chosen": -0.6175781488418579,
      "logps/rejected": -0.859375,
      "loss": 1.0504,
      "nll_loss": 0.962109386920929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0308837890625,
      "rewards/margins": 0.012096405029296875,
      "rewards/rejected": -0.04298095777630806,
      "step": 550
    },
    {
      "epoch": 0.040796998506538444,
      "grad_norm": 1.2963448782213514,
      "learning_rate": 2.1128856368212917e-06,
      "log_odds_chosen": 0.28815919160842896,
      "log_odds_ratio": -0.6717773675918579,
      "logits/chosen": -0.942578136920929,
      "logits/rejected": -0.882031261920929,
      "logps/chosen": -0.678906261920929,
      "logps/rejected": -0.839648425579071,
      "loss": 1.0328,
      "nll_loss": 1.0431640148162842,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.03392333909869194,
      "rewards/margins": 0.008064651861786842,
      "rewards/rejected": -0.04201049730181694,
      "step": 560
    },
    {
      "epoch": 0.04152551633701235,
      "grad_norm": 1.4991176630143255,
      "learning_rate": 2.0942695414584777e-06,
      "log_odds_chosen": 0.27735596895217896,
      "log_odds_ratio": -0.6739257574081421,
      "logits/chosen": -0.908007800579071,
      "logits/rejected": -0.853320300579071,
      "logps/chosen": -0.6747070550918579,
      "logps/rejected": -0.855664074420929,
      "loss": 1.041,
      "nll_loss": 1.0556640625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03376464918255806,
      "rewards/margins": 0.009012985043227673,
      "rewards/rejected": -0.042755126953125,
      "step": 570
    },
    {
      "epoch": 0.04225403416748625,
      "grad_norm": 1.1694573886259314,
      "learning_rate": 2.0761369963434992e-06,
      "log_odds_chosen": 0.45500487089157104,
      "log_odds_ratio": -0.5956054925918579,
      "logits/chosen": -0.949023425579071,
      "logits/rejected": -0.877148449420929,
      "logps/chosen": -0.612500011920929,
      "logps/rejected": -0.869335949420929,
      "loss": 1.0051,
      "nll_loss": 0.9388672113418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.03059082105755806,
      "rewards/margins": 0.01287078857421875,
      "rewards/rejected": -0.04348144680261612,
      "step": 580
    },
    {
      "epoch": 0.04298255199796015,
      "grad_norm": 1.2162624775952562,
      "learning_rate": 2.058467423981546e-06,
      "log_odds_chosen": 0.34337157011032104,
      "log_odds_ratio": -0.6548827886581421,
      "logits/chosen": -0.914257824420929,
      "logits/rejected": -0.85546875,
      "logps/chosen": -0.605273425579071,
      "logps/rejected": -0.8138672113418579,
      "loss": 1.0214,
      "nll_loss": 0.9873046875,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03025512769818306,
      "rewards/margins": 0.01042861957103014,
      "rewards/rejected": -0.04069213941693306,
      "step": 590
    },
    {
      "epoch": 0.04371106982843405,
      "grad_norm": 1.166617338590926,
      "learning_rate": 2.0412414523193154e-06,
      "log_odds_chosen": 0.48973387479782104,
      "log_odds_ratio": -0.5732421875,
      "logits/chosen": -0.9166015386581421,
      "logits/rejected": -0.838574230670929,
      "logps/chosen": -0.616992175579071,
      "logps/rejected": -0.909375011920929,
      "loss": 1.0412,
      "nll_loss": 1.016992211341858,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0308837890625,
      "rewards/margins": 0.014613723382353783,
      "rewards/rejected": -0.04545898362994194,
      "step": 600
    },
    {
      "epoch": 0.04443958765890795,
      "grad_norm": 1.2102841578643917,
      "learning_rate": 2.0244408254472904e-06,
      "log_odds_chosen": 0.5351806879043579,
      "log_odds_ratio": -0.573925793170929,
      "logits/chosen": -0.955078125,
      "logits/rejected": -0.8798828125,
      "logps/chosen": -0.609570324420929,
      "logps/rejected": -0.9107421636581421,
      "loss": 1.0153,
      "nll_loss": 0.966992199420929,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.03048095665872097,
      "rewards/margins": 0.015047455206513405,
      "rewards/rejected": -0.04552001878619194,
      "step": 610
    },
    {
      "epoch": 0.045168105489381855,
      "grad_norm": 1.3063126264389826,
      "learning_rate": 2.0080483222562476e-06,
      "log_odds_chosen": 0.39887696504592896,
      "log_odds_ratio": -0.610644519329071,
      "logits/chosen": -0.8919922113418579,
      "logits/rejected": -0.817187488079071,
      "logps/chosen": -0.6131836175918579,
      "logps/rejected": -0.8482421636581421,
      "loss": 1.0107,
      "nll_loss": 1.0076172351837158,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.03065795823931694,
      "rewards/margins": 0.011753082275390625,
      "rewards/rejected": -0.04240722581744194,
      "step": 620
    },
    {
      "epoch": 0.04589662331985575,
      "grad_norm": 1.255082051993984,
      "learning_rate": 1.9920476822239895e-06,
      "log_odds_chosen": 0.4494872987270355,
      "log_odds_ratio": -0.591796875,
      "logits/chosen": -0.873828113079071,
      "logits/rejected": -0.8160156011581421,
      "logps/chosen": -0.63720703125,
      "logps/rejected": -0.8779296875,
      "loss": 1.0089,
      "nll_loss": 1.009374976158142,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03188476711511612,
      "rewards/margins": 0.012020873837172985,
      "rewards/rejected": -0.04386596754193306,
      "step": 630
    },
    {
      "epoch": 0.04662514115032965,
      "grad_norm": 1.1864333149703672,
      "learning_rate": 1.976423537605237e-06,
      "log_odds_chosen": 0.5431884527206421,
      "log_odds_ratio": -0.580078125,
      "logits/chosen": -0.932421863079071,
      "logits/rejected": -0.8642578125,
      "logps/chosen": -0.6045898199081421,
      "logps/rejected": -0.9222656488418579,
      "loss": 1.001,
      "nll_loss": 0.9839843511581421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.03026122972369194,
      "rewards/margins": 0.015789031982421875,
      "rewards/rejected": -0.04605712741613388,
      "step": 640
    },
    {
      "epoch": 0.047353658980803556,
      "grad_norm": 1.2645534553519004,
      "learning_rate": 1.961161351381841e-06,
      "log_odds_chosen": 0.5426269769668579,
      "log_odds_ratio": -0.5684570074081421,
      "logits/chosen": -0.892382800579071,
      "logits/rejected": -0.8160156011581421,
      "logps/chosen": -0.59521484375,
      "logps/rejected": -0.89794921875,
      "loss": 1.0043,
      "nll_loss": 0.9439452886581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.02973632887005806,
      "rewards/margins": 0.015146255493164062,
      "rewards/rejected": -0.044891357421875,
      "step": 650
    },
    {
      "epoch": 0.04808217681127746,
      "grad_norm": 1.1747686785198292,
      "learning_rate": 1.9462473604038077e-06,
      "log_odds_chosen": 0.593188464641571,
      "log_odds_ratio": -0.610644519329071,
      "logits/chosen": -0.887890636920929,
      "logits/rejected": -0.8291015625,
      "logps/chosen": -0.584765613079071,
      "logps/rejected": -0.9830077886581421,
      "loss": 1.025,
      "nll_loss": 1.0007812976837158,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.02923583984375,
      "rewards/margins": 0.019895171746611595,
      "rewards/rejected": -0.049102783203125,
      "step": 660
    },
    {
      "epoch": 0.048810694641751355,
      "grad_norm": 1.18776648316632,
      "learning_rate": 1.9316685232156397e-06,
      "log_odds_chosen": 0.5345214605331421,
      "log_odds_ratio": -0.5699218511581421,
      "logits/chosen": -0.9253906011581421,
      "logits/rejected": -0.834765613079071,
      "logps/chosen": -0.6329101324081421,
      "logps/rejected": -0.920214831829071,
      "loss": 1.0303,
      "nll_loss": 0.998046875,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.03162841871380806,
      "rewards/margins": 0.014369201846420765,
      "rewards/rejected": -0.045989990234375,
      "step": 670
    },
    {
      "epoch": 0.04953921247222526,
      "grad_norm": 1.1617647189588096,
      "learning_rate": 1.917412472118426e-06,
      "log_odds_chosen": 0.541333019733429,
      "log_odds_ratio": -0.5972656011581421,
      "logits/chosen": -0.922656238079071,
      "logits/rejected": -0.835156261920929,
      "logps/chosen": -0.643261730670929,
      "logps/rejected": -0.955078125,
      "loss": 1.0169,
      "nll_loss": 1.0710937976837158,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.03217162936925888,
      "rewards/margins": 0.015583801083266735,
      "rewards/rejected": -0.04775390774011612,
      "step": 680
    },
    {
      "epoch": 0.05026773030269916,
      "grad_norm": 1.1871172213874464,
      "learning_rate": 1.9034674690672024e-06,
      "log_odds_chosen": 0.601000964641571,
      "log_odds_ratio": -0.576464831829071,
      "logits/chosen": -0.8931640386581421,
      "logits/rejected": -0.8388671875,
      "logps/chosen": -0.62109375,
      "logps/rejected": -0.968554675579071,
      "loss": 1.0189,
      "nll_loss": 1.022851586341858,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03106079064309597,
      "rewards/margins": 0.017378997057676315,
      "rewards/rejected": -0.04843749850988388,
      "step": 690
    },
    {
      "epoch": 0.05099624813317306,
      "grad_norm": 1.196320241884165,
      "learning_rate": 1.8898223650461362e-06,
      "log_odds_chosen": 0.4243408143520355,
      "log_odds_ratio": -0.6001952886581421,
      "logits/chosen": -0.9224609136581421,
      "logits/rejected": -0.848828136920929,
      "logps/chosen": -0.63525390625,
      "logps/rejected": -0.866992175579071,
      "loss": 1.0197,
      "nll_loss": 1.05859375,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.03176269680261612,
      "rewards/margins": 0.011584472842514515,
      "rewards/rejected": -0.04335937649011612,
      "step": 700
    },
    {
      "epoch": 0.05172476596364696,
      "grad_norm": 1.212168578567161,
      "learning_rate": 1.876466562602004e-06,
      "log_odds_chosen": 0.573901355266571,
      "log_odds_ratio": -0.578417956829071,
      "logits/chosen": -0.9214843511581421,
      "logits/rejected": -0.8431640863418579,
      "logps/chosen": -0.6041015386581421,
      "logps/rejected": -0.9359375238418579,
      "loss": 1.0076,
      "nll_loss": 0.9751952886581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03020019456744194,
      "rewards/margins": 0.016600798815488815,
      "rewards/rejected": -0.04681396484375,
      "step": 710
    },
    {
      "epoch": 0.05245328379412086,
      "grad_norm": 1.2324794932906062,
      "learning_rate": 1.863389981249825e-06,
      "log_odds_chosen": 0.47761231660842896,
      "log_odds_ratio": -0.584667980670929,
      "logits/chosen": -0.896289050579071,
      "logits/rejected": -0.8304687738418579,
      "logps/chosen": -0.6337890625,
      "logps/rejected": -0.885937511920929,
      "loss": 1.0053,
      "nll_loss": 0.9984375238418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03168334811925888,
      "rewards/margins": 0.01262664794921875,
      "rewards/rejected": -0.04432983323931694,
      "step": 720
    },
    {
      "epoch": 0.05318180162459476,
      "grad_norm": 1.1832994201166138,
      "learning_rate": 1.8505830254940132e-06,
      "log_odds_chosen": 0.4923339784145355,
      "log_odds_ratio": -0.6045898199081421,
      "logits/chosen": -0.8511718511581421,
      "logits/rejected": -0.8011718988418579,
      "logps/chosen": -0.5801757574081421,
      "logps/rejected": -0.8646484613418579,
      "loss": 1.0148,
      "nll_loss": 0.982226550579071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02901611290872097,
      "rewards/margins": 0.014204788021743298,
      "rewards/rejected": -0.04323730617761612,
      "step": 730
    },
    {
      "epoch": 0.05391031945506866,
      "grad_norm": 1.2818038845415187,
      "learning_rate": 1.8380365552345197e-06,
      "log_odds_chosen": 0.34840089082717896,
      "log_odds_ratio": -0.6689453125,
      "logits/chosen": -0.8515625,
      "logits/rejected": -0.8121093511581421,
      "logps/chosen": -0.626953125,
      "logps/rejected": -0.843554675579071,
      "loss": 1.0229,
      "nll_loss": 1.0212891101837158,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03134765475988388,
      "rewards/margins": 0.01082534808665514,
      "rewards/rejected": -0.04216919094324112,
      "step": 740
    },
    {
      "epoch": 0.054638837285542564,
      "grad_norm": 1.3523683581463872,
      "learning_rate": 1.8257418583505536e-06,
      "log_odds_chosen": 0.4212280213832855,
      "log_odds_ratio": -0.616894543170929,
      "logits/chosen": -0.9447265863418579,
      "logits/rejected": -0.846484363079071,
      "logps/chosen": -0.616992175579071,
      "logps/rejected": -0.8482421636581421,
      "loss": 1.0158,
      "nll_loss": 0.953125,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.03082275390625,
      "rewards/margins": 0.011573409661650658,
      "rewards/rejected": -0.04240722581744194,
      "step": 750
    },
    {
      "epoch": 0.05536735511601647,
      "grad_norm": 1.2019656393107072,
      "learning_rate": 1.8136906252750293e-06,
      "log_odds_chosen": 0.547119140625,
      "log_odds_ratio": -0.57373046875,
      "logits/chosen": -0.908203125,
      "logits/rejected": -0.841992199420929,
      "logps/chosen": -0.61083984375,
      "logps/rejected": -0.9085937738418579,
      "loss": 1.0222,
      "nll_loss": 0.9388672113418579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03054199181497097,
      "rewards/margins": 0.01490936242043972,
      "rewards/rejected": -0.04546508938074112,
      "step": 760
    },
    {
      "epoch": 0.05609587294649036,
      "grad_norm": 1.1900537849273092,
      "learning_rate": 1.801874925391118e-06,
      "log_odds_chosen": 0.42314451932907104,
      "log_odds_ratio": -0.6175781488418579,
      "logits/chosen": -0.924609363079071,
      "logits/rejected": -0.8539062738418579,
      "logps/chosen": -0.648632824420929,
      "logps/rejected": -0.8863281011581421,
      "loss": 1.0024,
      "nll_loss": 0.9615234136581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.03241576999425888,
      "rewards/margins": 0.011906432919204235,
      "rewards/rejected": -0.04429931566119194,
      "step": 770
    },
    {
      "epoch": 0.056824390776964266,
      "grad_norm": 1.2380064438904679,
      "learning_rate": 1.7902871850985824e-06,
      "log_odds_chosen": 0.4501098692417145,
      "log_odds_ratio": -0.603710949420929,
      "logits/chosen": -0.903515636920929,
      "logits/rejected": -0.8238281011581421,
      "logps/chosen": -0.62109375,
      "logps/rejected": -0.885546863079071,
      "loss": 0.9939,
      "nll_loss": 0.984179675579071,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03103027306497097,
      "rewards/margins": 0.01325073279440403,
      "rewards/rejected": -0.04429321363568306,
      "step": 780
    },
    {
      "epoch": 0.05755290860743817,
      "grad_norm": 1.2523169955450049,
      "learning_rate": 1.7789201674120502e-06,
      "log_odds_chosen": 0.32695311307907104,
      "log_odds_ratio": -0.673632800579071,
      "logits/chosen": -0.9457031488418579,
      "logits/rejected": -0.8765624761581421,
      "logps/chosen": -0.663281261920929,
      "logps/rejected": -0.8642578125,
      "loss": 1.0127,
      "nll_loss": 0.966992199420929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.03321533277630806,
      "rewards/margins": 0.009995651431381702,
      "rewards/rejected": -0.04316406324505806,
      "step": 790
    },
    {
      "epoch": 0.058281426437912065,
      "grad_norm": 1.2614851188857854,
      "learning_rate": 1.7677669529663689e-06,
      "log_odds_chosen": 0.4099487364292145,
      "log_odds_ratio": -0.621386706829071,
      "logits/chosen": -0.951953113079071,
      "logits/rejected": -0.864062488079071,
      "logps/chosen": -0.6631835699081421,
      "logps/rejected": -0.9117187261581421,
      "loss": 0.9948,
      "nll_loss": 0.9964843988418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.03314819186925888,
      "rewards/margins": 0.012448882684111595,
      "rewards/rejected": -0.04559936374425888,
      "step": 800
    },
    {
      "epoch": 0.05900994426838597,
      "grad_norm": 1.1914213968436156,
      "learning_rate": 1.7568209223157664e-06,
      "log_odds_chosen": 0.48750001192092896,
      "log_odds_ratio": -0.602734386920929,
      "logits/chosen": -0.9039062261581421,
      "logits/rejected": -0.843554675579071,
      "logps/chosen": -0.611621081829071,
      "logps/rejected": -0.8638671636581421,
      "loss": 1.0138,
      "nll_loss": 0.94921875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03056030347943306,
      "rewards/margins": 0.012659454718232155,
      "rewards/rejected": -0.04319458082318306,
      "step": 810
    },
    {
      "epoch": 0.05973846209885987,
      "grad_norm": 1.2391285362784021,
      "learning_rate": 1.7460757394239458e-06,
      "log_odds_chosen": 0.3900390565395355,
      "log_odds_ratio": -0.659960925579071,
      "logits/chosen": -0.9222656488418579,
      "logits/rejected": -0.869921863079071,
      "logps/chosen": -0.627734363079071,
      "logps/rejected": -0.8662109375,
      "loss": 1.0047,
      "nll_loss": 0.9017578363418579,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.03139648586511612,
      "rewards/margins": 0.01190109271556139,
      "rewards/rejected": -0.043304443359375,
      "step": 820
    },
    {
      "epoch": 0.060466979929333774,
      "grad_norm": 1.2606811726043967,
      "learning_rate": 1.7355253362515584e-06,
      "log_odds_chosen": 0.2829833924770355,
      "log_odds_ratio": -0.685839831829071,
      "logits/chosen": -0.9136718511581421,
      "logits/rejected": -0.8746093511581421,
      "logps/chosen": -0.625292956829071,
      "logps/rejected": -0.7779296636581421,
      "loss": 0.9863,
      "nll_loss": 0.962890625,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.031280517578125,
      "rewards/margins": 0.007628440856933594,
      "rewards/rejected": -0.03891601413488388,
      "step": 830
    },
    {
      "epoch": 0.06119549775980767,
      "grad_norm": 1.2111459772366093,
      "learning_rate": 1.7251638983558855e-06,
      "log_odds_chosen": 0.4200439453125,
      "log_odds_ratio": -0.605175793170929,
      "logits/chosen": -0.9166015386581421,
      "logits/rejected": -0.865429699420929,
      "logps/chosen": -0.630175769329071,
      "logps/rejected": -0.8773437738418579,
      "loss": 0.9756,
      "nll_loss": 0.9585937261581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.03151855617761612,
      "rewards/margins": 0.012355422601103783,
      "rewards/rejected": -0.04387206956744194,
      "step": 840
    },
    {
      "epoch": 0.06192401559028157,
      "grad_norm": 1.2490191400244082,
      "learning_rate": 1.7149858514250883e-06,
      "log_odds_chosen": 0.3470458984375,
      "log_odds_ratio": -0.6597656011581421,
      "logits/chosen": -0.872265636920929,
      "logits/rejected": -0.797070324420929,
      "logps/chosen": -0.63427734375,
      "logps/rejected": -0.859375,
      "loss": 1.0103,
      "nll_loss": 1.045507788658142,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.03173828125,
      "rewards/margins": 0.011228179559111595,
      "rewards/rejected": -0.04298095777630806,
      "step": 850
    },
    {
      "epoch": 0.06265253342075547,
      "grad_norm": 1.245271815294314,
      "learning_rate": 1.704985848676184e-06,
      "log_odds_chosen": 0.6527343988418579,
      "log_odds_ratio": -0.53955078125,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.866406261920929,
      "logps/chosen": -0.57958984375,
      "logps/rejected": -0.952343761920929,
      "loss": 0.9969,
      "nll_loss": 0.942187488079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02896728552877903,
      "rewards/margins": 0.01864166185259819,
      "rewards/rejected": -0.04758300632238388,
      "step": 860
    },
    {
      "epoch": 0.06338105125122938,
      "grad_norm": 1.3118253618162397,
      "learning_rate": 1.6951587590520263e-06,
      "log_odds_chosen": 0.532177746295929,
      "log_odds_ratio": -0.591601550579071,
      "logits/chosen": -0.8892577886581421,
      "logits/rejected": -0.795703113079071,
      "logps/chosen": -0.5986328125,
      "logps/rejected": -0.904492199420929,
      "loss": 1.0228,
      "nll_loss": 1.0048828125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.02996215783059597,
      "rewards/margins": 0.01529083214700222,
      "rewards/rejected": -0.04519043117761612,
      "step": 870
    },
    {
      "epoch": 0.06410956908170327,
      "grad_norm": 1.3074373422676748,
      "learning_rate": 1.6854996561581053e-06,
      "log_odds_chosen": 0.36079102754592896,
      "log_odds_ratio": -0.629687488079071,
      "logits/chosen": -0.9560546875,
      "logits/rejected": -0.89453125,
      "logps/chosen": -0.59912109375,
      "logps/rejected": -0.810253918170929,
      "loss": 1.0055,
      "nll_loss": 0.9427734613418579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.02996826171875,
      "rewards/margins": 0.010572051629424095,
      "rewards/rejected": -0.04055175930261612,
      "step": 880
    },
    {
      "epoch": 0.06483808691217717,
      "grad_norm": 1.3019689767097504,
      "learning_rate": 1.6760038078849776e-06,
      "log_odds_chosen": 0.6428467035293579,
      "log_odds_ratio": -0.5464843511581421,
      "logits/chosen": -0.949023425579071,
      "logits/rejected": -0.8671875,
      "logps/chosen": -0.6025390625,
      "logps/rejected": -0.9839843511581421,
      "loss": 0.9923,
      "nll_loss": 0.988085925579071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.030120849609375,
      "rewards/margins": 0.019062042236328125,
      "rewards/rejected": -0.04920654371380806,
      "step": 890
    },
    {
      "epoch": 0.06556660474265108,
      "grad_norm": 1.1918168397321973,
      "learning_rate": 1.6666666666666667e-06,
      "log_odds_chosen": 0.5846801996231079,
      "log_odds_ratio": -0.61328125,
      "logits/chosen": -0.9423828125,
      "logits/rejected": -0.897265613079071,
      "logps/chosen": -0.600878894329071,
      "logps/rejected": -0.952929675579071,
      "loss": 1.0031,
      "nll_loss": 0.9486328363418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.03003540076315403,
      "rewards/margins": 0.01764984056353569,
      "rewards/rejected": -0.04768066480755806,
      "step": 900
    },
    {
      "epoch": 0.06629512257312498,
      "grad_norm": 1.277910999072872,
      "learning_rate": 1.6574838603294898e-06,
      "log_odds_chosen": 0.5803588628768921,
      "log_odds_ratio": -0.56103515625,
      "logits/chosen": -0.9634765386581421,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.6265624761581421,
      "logps/rejected": -0.939257800579071,
      "loss": 1.0168,
      "nll_loss": 1.0205078125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03131103515625,
      "rewards/margins": 0.015634536743164062,
      "rewards/rejected": -0.04697265475988388,
      "step": 910
    },
    {
      "epoch": 0.06702364040359887,
      "grad_norm": 1.2673099400717343,
      "learning_rate": 1.648451183489468e-06,
      "log_odds_chosen": 0.6135009527206421,
      "log_odds_ratio": -0.5707031488418579,
      "logits/chosen": -0.954296886920929,
      "logits/rejected": -0.935742199420929,
      "logps/chosen": -0.5599609613418579,
      "logps/rejected": -0.899609386920929,
      "loss": 1.0083,
      "nll_loss": 0.9349609613418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02799072302877903,
      "rewards/margins": 0.017002105712890625,
      "rewards/rejected": -0.04499511793255806,
      "step": 920
    },
    {
      "epoch": 0.06775215823407278,
      "grad_norm": 1.2356515608460252,
      "learning_rate": 1.6395645894598825e-06,
      "log_odds_chosen": 0.500256359577179,
      "log_odds_ratio": -0.592480480670929,
      "logits/chosen": -0.933789074420929,
      "logits/rejected": -0.8695312738418579,
      "logps/chosen": -0.637499988079071,
      "logps/rejected": -0.8941406011581421,
      "loss": 0.9873,
      "nll_loss": 0.974804699420929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.03190307691693306,
      "rewards/margins": 0.01278762798756361,
      "rewards/rejected": -0.044677734375,
      "step": 930
    },
    {
      "epoch": 0.06848067606454668,
      "grad_norm": 1.2096399771828836,
      "learning_rate": 1.6308201826336057e-06,
      "log_odds_chosen": 0.39421385526657104,
      "log_odds_ratio": -0.650683581829071,
      "logits/chosen": -0.983593761920929,
      "logits/rejected": -0.907421886920929,
      "logps/chosen": -0.616406261920929,
      "logps/rejected": -0.857226550579071,
      "loss": 0.9707,
      "nll_loss": 0.9271484613418579,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.03080444410443306,
      "rewards/margins": 0.012049865908920765,
      "rewards/rejected": -0.04284057766199112,
      "step": 940
    },
    {
      "epoch": 0.06920919389502057,
      "grad_norm": 1.2597402754004785,
      "learning_rate": 1.6222142113076255e-06,
      "log_odds_chosen": 0.3767944276332855,
      "log_odds_ratio": -0.6581054925918579,
      "logits/chosen": -0.940625011920929,
      "logits/rejected": -0.883984386920929,
      "logps/chosen": -0.5845702886581421,
      "logps/rejected": -0.7889648675918579,
      "loss": 1.0141,
      "nll_loss": 1.013085961341858,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.029205322265625,
      "rewards/margins": 0.010249900631606579,
      "rewards/rejected": -0.03945312649011612,
      "step": 950
    },
    {
      "epoch": 0.06993771172549448,
      "grad_norm": 1.3036878123004485,
      "learning_rate": 1.6137430609197571e-06,
      "log_odds_chosen": 0.49174803495407104,
      "log_odds_ratio": -0.6055663824081421,
      "logits/chosen": -0.9371093511581421,
      "logits/rejected": -0.879101574420929,
      "logps/chosen": -0.6644531488418579,
      "logps/rejected": -0.9800781011581421,
      "loss": 0.9935,
      "nll_loss": 1.0011718273162842,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.033203125,
      "rewards/margins": 0.01576538011431694,
      "rewards/rejected": -0.04902954027056694,
      "step": 960
    },
    {
      "epoch": 0.07066622955596838,
      "grad_norm": 1.2220513550074152,
      "learning_rate": 1.605403247669839e-06,
      "log_odds_chosen": 0.522814929485321,
      "log_odds_ratio": -0.5663086175918579,
      "logits/chosen": -0.9564453363418579,
      "logits/rejected": -0.884765625,
      "logps/chosen": -0.5771484375,
      "logps/rejected": -0.847851574420929,
      "loss": 1.0145,
      "nll_loss": 0.954882800579071,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02884521521627903,
      "rewards/margins": 0.013541030697524548,
      "rewards/rejected": -0.04237060621380806,
      "step": 970
    },
    {
      "epoch": 0.07139474738644229,
      "grad_norm": 1.3202650289836475,
      "learning_rate": 1.59719141249985e-06,
      "log_odds_chosen": 0.47175294160842896,
      "log_odds_ratio": -0.616406261920929,
      "logits/chosen": -0.9166015386581421,
      "logits/rejected": -0.849609375,
      "logps/chosen": -0.60546875,
      "logps/rejected": -0.884570300579071,
      "loss": 0.996,
      "nll_loss": 0.997265636920929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.0302734375,
      "rewards/margins": 0.01393279992043972,
      "rewards/rejected": -0.04420776292681694,
      "step": 980
    },
    {
      "epoch": 0.07212326521691619,
      "grad_norm": 1.2205014546501503,
      "learning_rate": 1.5891043154093205e-06,
      "log_odds_chosen": 0.570507824420929,
      "log_odds_ratio": -0.5689452886581421,
      "logits/chosen": -0.971875011920929,
      "logits/rejected": -0.8951171636581421,
      "logps/chosen": -0.6182616949081421,
      "logps/rejected": -0.9521484375,
      "loss": 0.9823,
      "nll_loss": 0.9443359375,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.03093261644244194,
      "rewards/margins": 0.016645049676299095,
      "rewards/rejected": -0.04758911207318306,
      "step": 990
    },
    {
      "epoch": 0.07285178304739008,
      "grad_norm": 1.228229184265604,
      "learning_rate": 1.5811388300841898e-06,
      "log_odds_chosen": 0.5091797113418579,
      "log_odds_ratio": -0.6006835699081421,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.914843738079071,
      "logps/chosen": -0.628222644329071,
      "logps/rejected": -0.9193359613418579,
      "loss": 1.0003,
      "nll_loss": 0.936328113079071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03137817233800888,
      "rewards/margins": 0.014559173956513405,
      "rewards/rejected": -0.04597168043255806,
      "step": 1000
    },
    {
      "epoch": 0.07358030087786399,
      "grad_norm": 1.2476086300175937,
      "learning_rate": 1.5732919388188816e-06,
      "log_odds_chosen": 0.45361328125,
      "log_odds_ratio": -0.614941418170929,
      "logits/chosen": -0.945507824420929,
      "logits/rejected": -0.8822265863418579,
      "logps/chosen": -0.6416015625,
      "logps/rejected": -0.8929687738418579,
      "loss": 0.9879,
      "nll_loss": 1.025781273841858,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.03203124925494194,
      "rewards/margins": 0.012599563226103783,
      "rewards/rejected": -0.044677734375,
      "step": 1010
    },
    {
      "epoch": 0.07430881870833789,
      "grad_norm": 1.1841978538342508,
      "learning_rate": 1.565560727712874e-06,
      "log_odds_chosen": 0.438720703125,
      "log_odds_ratio": -0.629589855670929,
      "logits/chosen": -0.929492175579071,
      "logits/rejected": -0.8529297113418579,
      "logps/chosen": -0.613085925579071,
      "logps/rejected": -0.8841797113418579,
      "loss": 1.0124,
      "nll_loss": 0.978710949420929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.03068847581744194,
      "rewards/margins": 0.0135498046875,
      "rewards/rejected": -0.04419555515050888,
      "step": 1020
    },
    {
      "epoch": 0.07503733653881178,
      "grad_norm": 1.390028860561529,
      "learning_rate": 1.5579423821243897e-06,
      "log_odds_chosen": 0.651318371295929,
      "log_odds_ratio": -0.569140613079071,
      "logits/chosen": -0.9574218988418579,
      "logits/rejected": -0.915820300579071,
      "logps/chosen": -0.605273425579071,
      "logps/rejected": -0.978710949420929,
      "loss": 0.9701,
      "nll_loss": 0.898242175579071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.03026122972369194,
      "rewards/margins": 0.018727874383330345,
      "rewards/rejected": -0.0489501953125,
      "step": 1030
    },
    {
      "epoch": 0.07576585436928569,
      "grad_norm": 1.2317002855508061,
      "learning_rate": 1.5504341823651056e-06,
      "log_odds_chosen": 0.526904284954071,
      "log_odds_ratio": -0.583984375,
      "logits/chosen": -0.922656238079071,
      "logits/rejected": -0.8589843511581421,
      "logps/chosen": -0.585253894329071,
      "logps/rejected": -0.8609374761581421,
      "loss": 0.9787,
      "nll_loss": 0.9619140625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.02926025353372097,
      "rewards/margins": 0.013814544305205345,
      "rewards/rejected": -0.04304199293255806,
      "step": 1040
    },
    {
      "epoch": 0.07649437219975959,
      "grad_norm": 1.2889948712858512,
      "learning_rate": 1.5430334996209192e-06,
      "log_odds_chosen": 0.6490844488143921,
      "log_odds_ratio": -0.565136730670929,
      "logits/chosen": -0.9681640863418579,
      "logits/rejected": -0.8970702886581421,
      "logps/chosen": -0.5877929925918579,
      "logps/rejected": -0.986328125,
      "loss": 0.9956,
      "nll_loss": 0.924023449420929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02938232384622097,
      "rewards/margins": 0.01996917650103569,
      "rewards/rejected": -0.049346923828125,
      "step": 1050
    },
    {
      "epoch": 0.07722289003023348,
      "grad_norm": 1.2283989287486436,
      "learning_rate": 1.5357377920848783e-06,
      "log_odds_chosen": 0.559436023235321,
      "log_odds_ratio": -0.59765625,
      "logits/chosen": -0.931835949420929,
      "logits/rejected": -0.855664074420929,
      "logps/chosen": -0.622851550579071,
      "logps/rejected": -0.921875,
      "loss": 0.9938,
      "nll_loss": 0.9771484136581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.03114623948931694,
      "rewards/margins": 0.014951705932617188,
      "rewards/rejected": -0.04610595852136612,
      "step": 1060
    },
    {
      "epoch": 0.0779514078607074,
      "grad_norm": 1.2778047284704197,
      "learning_rate": 1.5285446012893579e-06,
      "log_odds_chosen": 0.6162475347518921,
      "log_odds_ratio": -0.563769519329071,
      "logits/chosen": -0.9482421875,
      "logits/rejected": -0.8666015863418579,
      "logps/chosen": -0.6053711175918579,
      "logps/rejected": -0.9546874761581421,
      "loss": 0.9797,
      "nll_loss": 0.9576171636581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.030303955078125,
      "rewards/margins": 0.017452239990234375,
      "rewards/rejected": -0.0477294921875,
      "step": 1070
    },
    {
      "epoch": 0.07867992569118129,
      "grad_norm": 1.2026898753454498,
      "learning_rate": 1.5214515486254614e-06,
      "log_odds_chosen": 0.551025390625,
      "log_odds_ratio": -0.589062511920929,
      "logits/chosen": -0.8880859613418579,
      "logits/rejected": -0.841796875,
      "logps/chosen": -0.5555664300918579,
      "logps/rejected": -0.8714843988418579,
      "loss": 0.9878,
      "nll_loss": 0.939648449420929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02775878831744194,
      "rewards/margins": 0.01577300950884819,
      "rewards/rejected": -0.04358520358800888,
      "step": 1080
    },
    {
      "epoch": 0.07940844352165519,
      "grad_norm": 1.3563494952949753,
      "learning_rate": 1.5144563320384566e-06,
      "log_odds_chosen": 0.6087402105331421,
      "log_odds_ratio": -0.573925793170929,
      "logits/chosen": -0.940234363079071,
      "logits/rejected": -0.877734363079071,
      "logps/chosen": -0.593066394329071,
      "logps/rejected": -0.961132824420929,
      "loss": 0.9941,
      "nll_loss": 0.919140636920929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.02965087816119194,
      "rewards/margins": 0.018425751477479935,
      "rewards/rejected": -0.04810180515050888,
      "step": 1090
    },
    {
      "epoch": 0.0801369613521291,
      "grad_norm": 1.370208442756465,
      "learning_rate": 1.5075567228888182e-06,
      "log_odds_chosen": 0.536914050579071,
      "log_odds_ratio": -0.601269543170929,
      "logits/chosen": -0.932421863079071,
      "logits/rejected": -0.8658202886581421,
      "logps/chosen": -0.6136718988418579,
      "logps/rejected": -0.9136718511581421,
      "loss": 0.9749,
      "nll_loss": 0.9248046875,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.03067626990377903,
      "rewards/margins": 0.0150299072265625,
      "rewards/rejected": -0.04573974758386612,
      "step": 1100
    },
    {
      "epoch": 0.08086547918260299,
      "grad_norm": 1.2826751636624225,
      "learning_rate": 1.5007505629691608e-06,
      "log_odds_chosen": 0.48491209745407104,
      "log_odds_ratio": -0.5933593511581421,
      "logits/chosen": -0.908007800579071,
      "logits/rejected": -0.8285156488418579,
      "logps/chosen": -0.58154296875,
      "logps/rejected": -0.861523449420929,
      "loss": 0.9804,
      "nll_loss": 0.945117175579071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02907104417681694,
      "rewards/margins": 0.01400146447122097,
      "rewards/rejected": -0.04309692233800888,
      "step": 1110
    },
    {
      "epoch": 0.08159399701307689,
      "grad_norm": 1.3630546667966033,
      "learning_rate": 1.494035761667992e-06,
      "log_odds_chosen": 0.4476684629917145,
      "log_odds_ratio": -0.645800769329071,
      "logits/chosen": -0.876953125,
      "logits/rejected": -0.827929675579071,
      "logps/chosen": -0.6166015863418579,
      "logps/rejected": -0.86279296875,
      "loss": 0.9858,
      "nll_loss": 1.023046851158142,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.03085937537252903,
      "rewards/margins": 0.01227645855396986,
      "rewards/rejected": -0.04309692233800888,
      "step": 1120
    },
    {
      "epoch": 0.0823225148435508,
      "grad_norm": 1.1912423877729335,
      "learning_rate": 1.487410293271824e-06,
      "log_odds_chosen": 0.4928344786167145,
      "log_odds_ratio": -0.595507800579071,
      "logits/chosen": -0.9234374761581421,
      "logits/rejected": -0.875195324420929,
      "logps/chosen": -0.5992187261581421,
      "logps/rejected": -0.86328125,
      "loss": 0.9717,
      "nll_loss": 0.92041015625,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02996215783059597,
      "rewards/margins": 0.01319808978587389,
      "rewards/rejected": -0.04317016527056694,
      "step": 1130
    },
    {
      "epoch": 0.0830510326740247,
      "grad_norm": 1.2922963312007965,
      "learning_rate": 1.480872194397731e-06,
      "log_odds_chosen": 0.718994140625,
      "log_odds_ratio": -0.5140625238418579,
      "logits/chosen": -0.9517577886581421,
      "logits/rejected": -0.8539062738418579,
      "logps/chosen": -0.6338866949081421,
      "logps/rejected": -1.065820336341858,
      "loss": 0.9632,
      "nll_loss": 0.944140613079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.03168945387005806,
      "rewards/margins": 0.02162017859518528,
      "rewards/rejected": -0.05333251878619194,
      "step": 1140
    },
    {
      "epoch": 0.0837795505044986,
      "grad_norm": 1.316922827729186,
      "learning_rate": 1.4744195615489715e-06,
      "log_odds_chosen": 0.47325438261032104,
      "log_odds_ratio": -0.5958007574081421,
      "logits/chosen": -0.915820300579071,
      "logits/rejected": -0.853515625,
      "logps/chosen": -0.611328125,
      "logps/rejected": -0.8716796636581421,
      "loss": 0.9822,
      "nll_loss": 0.971875011920929,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.03056030347943306,
      "rewards/margins": 0.013033675961196423,
      "rewards/rejected": -0.04356689378619194,
      "step": 1150
    },
    {
      "epoch": 0.0845080683349725,
      "grad_norm": 1.2332348308596637,
      "learning_rate": 1.4680505487867589e-06,
      "log_odds_chosen": 0.46037596464157104,
      "log_odds_ratio": -0.609179675579071,
      "logits/chosen": -0.9517577886581421,
      "logits/rejected": -0.887499988079071,
      "logps/chosen": -0.585253894329071,
      "logps/rejected": -0.855664074420929,
      "loss": 0.9753,
      "nll_loss": 0.9095703363418579,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.02927246131002903,
      "rewards/margins": 0.01351242046803236,
      "rewards/rejected": -0.04276122897863388,
      "step": 1160
    },
    {
      "epoch": 0.0852365861654464,
      "grad_norm": 1.2698426946921688,
      "learning_rate": 1.4617633655117156e-06,
      "log_odds_chosen": 0.4734252989292145,
      "log_odds_ratio": -0.6004883050918579,
      "logits/chosen": -0.9408203363418579,
      "logits/rejected": -0.869140625,
      "logps/chosen": -0.5997070074081421,
      "logps/rejected": -0.8658202886581421,
      "loss": 0.9678,
      "nll_loss": 0.9253906011581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.02996215783059597,
      "rewards/margins": 0.013317870907485485,
      "rewards/rejected": -0.04326782375574112,
      "step": 1170
    },
    {
      "epoch": 0.0859651039959203,
      "grad_norm": 1.2478593762989805,
      "learning_rate": 1.4555562743489552e-06,
      "log_odds_chosen": 0.5010010004043579,
      "log_odds_ratio": -0.563183605670929,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -0.602343738079071,
      "logps/rejected": -0.892382800579071,
      "loss": 0.9831,
      "nll_loss": 0.886523425579071,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.03012695349752903,
      "rewards/margins": 0.01454010047018528,
      "rewards/rejected": -0.04468994215130806,
      "step": 1180
    },
    {
      "epoch": 0.0866936218263942,
      "grad_norm": 1.2809444974932678,
      "learning_rate": 1.4494275891311214e-06,
      "log_odds_chosen": 0.4388061463832855,
      "log_odds_ratio": -0.6050781011581421,
      "logits/chosen": -0.944531261920929,
      "logits/rejected": -0.8589843511581421,
      "logps/chosen": -0.605273425579071,
      "logps/rejected": -0.843554675579071,
      "loss": 0.9717,
      "nll_loss": 0.9244140386581421,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.03027954138815403,
      "rewards/margins": 0.01188507117331028,
      "rewards/rejected": -0.04216308519244194,
      "step": 1190
    },
    {
      "epoch": 0.0874221396568681,
      "grad_norm": 1.2765219221531252,
      "learning_rate": 1.4433756729740647e-06,
      "log_odds_chosen": 0.5772949457168579,
      "log_odds_ratio": -0.57373046875,
      "logits/chosen": -0.916015625,
      "logits/rejected": -0.826953113079071,
      "logps/chosen": -0.598339855670929,
      "logps/rejected": -0.9546874761581421,
      "loss": 0.9834,
      "nll_loss": 0.9662109613418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02992553636431694,
      "rewards/margins": 0.01785736158490181,
      "rewards/rejected": -0.04776611179113388,
      "step": 1200
    },
    {
      "epoch": 0.08815065748734201,
      "grad_norm": 1.3386280499630672,
      "learning_rate": 1.4373989364401727e-06,
      "log_odds_chosen": 0.658886730670929,
      "log_odds_ratio": -0.55224609375,
      "logits/chosen": -0.9453125,
      "logits/rejected": -0.862500011920929,
      "logps/chosen": -0.57275390625,
      "logps/rejected": -0.9300781488418579,
      "loss": 0.9654,
      "nll_loss": 0.9595702886581421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.02863159216940403,
      "rewards/margins": 0.01785583421587944,
      "rewards/rejected": -0.0465087890625,
      "step": 1210
    },
    {
      "epoch": 0.0888791753178159,
      "grad_norm": 1.280772954151099,
      "learning_rate": 1.4314958357846706e-06,
      "log_odds_chosen": 0.51708984375,
      "log_odds_ratio": -0.603320300579071,
      "logits/chosen": -0.9111328125,
      "logits/rejected": -0.8701171875,
      "logps/chosen": -0.5889648199081421,
      "logps/rejected": -0.8797851800918579,
      "loss": 0.9985,
      "nll_loss": 0.955859363079071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.02944335900247097,
      "rewards/margins": 0.014523315243422985,
      "rewards/rejected": -0.04393921047449112,
      "step": 1220
    },
    {
      "epoch": 0.0896076931482898,
      "grad_norm": 1.183084517015655,
      "learning_rate": 1.4256648712805027e-06,
      "log_odds_chosen": 0.557812511920929,
      "log_odds_ratio": -0.581347644329071,
      "logits/chosen": -0.8744140863418579,
      "logits/rejected": -0.8134765625,
      "logps/chosen": -0.59912109375,
      "logps/rejected": -0.8955078125,
      "loss": 0.9943,
      "nll_loss": 0.9759765863418579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.029937744140625,
      "rewards/margins": 0.01481018029153347,
      "rewards/rejected": -0.044769287109375,
      "step": 1230
    },
    {
      "epoch": 0.09033621097876371,
      "grad_norm": 1.333731295694987,
      "learning_rate": 1.419904585617662e-06,
      "log_odds_chosen": 0.5431152582168579,
      "log_odds_ratio": -0.596484363079071,
      "logits/chosen": -0.941210925579071,
      "logits/rejected": -0.869921863079071,
      "logps/chosen": -0.6368163824081421,
      "logps/rejected": -0.9652343988418579,
      "loss": 0.9784,
      "nll_loss": 0.9609375,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03181762620806694,
      "rewards/margins": 0.016417503356933594,
      "rewards/rejected": -0.04829101637005806,
      "step": 1240
    },
    {
      "epoch": 0.0910647288092376,
      "grad_norm": 1.2318462293949892,
      "learning_rate": 1.4142135623730952e-06,
      "log_odds_chosen": 0.5527588129043579,
      "log_odds_ratio": -0.589062511920929,
      "logits/chosen": -0.9273437261581421,
      "logits/rejected": -0.8623046875,
      "logps/chosen": -0.6070312261581421,
      "logps/rejected": -0.93359375,
      "loss": 0.9867,
      "nll_loss": 0.961718738079071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03037719801068306,
      "rewards/margins": 0.016327286139130592,
      "rewards/rejected": -0.04669189453125,
      "step": 1250
    },
    {
      "epoch": 0.0917932466397115,
      "grad_norm": 1.2808873959562712,
      "learning_rate": 1.4085904245475275e-06,
      "log_odds_chosen": 0.6622070074081421,
      "log_odds_ratio": -0.525195300579071,
      "logits/chosen": -0.9134765863418579,
      "logits/rejected": -0.8291015625,
      "logps/chosen": -0.580761730670929,
      "logps/rejected": -0.9429687261581421,
      "loss": 0.9635,
      "nll_loss": 0.907421886920929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02899169921875,
      "rewards/margins": 0.018184661865234375,
      "rewards/rejected": -0.04718017578125,
      "step": 1260
    },
    {
      "epoch": 0.09252176447018541,
      "grad_norm": 1.2446545828701625,
      "learning_rate": 1.4030338331657844e-06,
      "log_odds_chosen": 0.36522215604782104,
      "log_odds_ratio": -0.644726574420929,
      "logits/chosen": -0.9515625238418579,
      "logits/rejected": -0.8734375238418579,
      "logps/chosen": -0.6380859613418579,
      "logps/rejected": -0.869335949420929,
      "loss": 0.9643,
      "nll_loss": 0.9638671875,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.03190917894244194,
      "rewards/margins": 0.011548995971679688,
      "rewards/rejected": -0.04345703125,
      "step": 1270
    },
    {
      "epoch": 0.0932502823006593,
      "grad_norm": 1.3284232417262638,
      "learning_rate": 1.3975424859373688e-06,
      "log_odds_chosen": 0.6805664300918579,
      "log_odds_ratio": -0.5265136957168579,
      "logits/chosen": -0.9296875,
      "logits/rejected": -0.83984375,
      "logps/chosen": -0.5595703125,
      "logps/rejected": -0.9173828363418579,
      "loss": 0.9656,
      "nll_loss": 0.876171886920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.027984619140625,
      "rewards/margins": 0.0178375244140625,
      "rewards/rejected": -0.04584961012005806,
      "step": 1280
    },
    {
      "epoch": 0.0939788001311332,
      "grad_norm": 1.240249628117818,
      "learning_rate": 1.3921151159742616e-06,
      "log_odds_chosen": 0.3670898377895355,
      "log_odds_ratio": -0.655468761920929,
      "logits/chosen": -0.919726550579071,
      "logits/rejected": -0.831250011920929,
      "logps/chosen": -0.646191418170929,
      "logps/rejected": -0.8695312738418579,
      "loss": 0.9839,
      "nll_loss": 1.0224609375,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.03232421725988388,
      "rewards/margins": 0.011131668463349342,
      "rewards/rejected": -0.04344482347369194,
      "step": 1290
    },
    {
      "epoch": 0.09470731796160711,
      "grad_norm": 1.254099164367804,
      "learning_rate": 1.386750490563073e-06,
      "log_odds_chosen": 0.588330090045929,
      "log_odds_ratio": -0.5826171636581421,
      "logits/chosen": -0.906054675579071,
      "logits/rejected": -0.8447265625,
      "logps/chosen": -0.6563476324081421,
      "logps/rejected": -0.9701172113418579,
      "loss": 0.9862,
      "nll_loss": 0.9931640625,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03281249850988388,
      "rewards/margins": 0.015676116570830345,
      "rewards/rejected": -0.048492431640625,
      "step": 1300
    },
    {
      "epoch": 0.09543583579208101,
      "grad_norm": 1.4086871764617632,
      "learning_rate": 1.3814474099888442e-06,
      "log_odds_chosen": 0.6194823980331421,
      "log_odds_ratio": -0.5419921875,
      "logits/chosen": -0.927734375,
      "logits/rejected": -0.884765625,
      "logps/chosen": -0.5572265386581421,
      "logps/rejected": -0.8779296875,
      "loss": 0.9615,
      "nll_loss": 0.9097656011581421,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02783203125,
      "rewards/margins": 0.016066741198301315,
      "rewards/rejected": -0.0439453125,
      "step": 1310
    },
    {
      "epoch": 0.09616435362255492,
      "grad_norm": 1.2758694128482435,
      "learning_rate": 1.376204706407951e-06,
      "log_odds_chosen": 0.45805662870407104,
      "log_odds_ratio": -0.6019531488418579,
      "logits/chosen": -0.911328136920929,
      "logits/rejected": -0.810546875,
      "logps/chosen": -0.5946289300918579,
      "logps/rejected": -0.841796875,
      "loss": 0.9637,
      "nll_loss": 0.948437511920929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.02973632887005806,
      "rewards/margins": 0.012365341186523438,
      "rewards/rejected": -0.0421142578125,
      "step": 1320
    },
    {
      "epoch": 0.09689287145302881,
      "grad_norm": 1.2461479224014989,
      "learning_rate": 1.3710212427677044e-06,
      "log_odds_chosen": 0.635449230670929,
      "log_odds_ratio": -0.555419921875,
      "logits/chosen": -0.9037109613418579,
      "logits/rejected": -0.8076171875,
      "logps/chosen": -0.608691394329071,
      "logps/rejected": -0.9869140386581421,
      "loss": 0.9718,
      "nll_loss": 0.999218761920929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03040161170065403,
      "rewards/margins": 0.01891479454934597,
      "rewards/rejected": -0.04932861402630806,
      "step": 1330
    },
    {
      "epoch": 0.09762138928350271,
      "grad_norm": 1.3214900968867742,
      "learning_rate": 1.3658959117703826e-06,
      "log_odds_chosen": 0.63494873046875,
      "log_odds_ratio": -0.542285144329071,
      "logits/chosen": -0.9365234375,
      "logits/rejected": -0.876757800579071,
      "logps/chosen": -0.5575195550918579,
      "logps/rejected": -0.882617175579071,
      "loss": 0.9687,
      "nll_loss": 0.9443359375,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02787475660443306,
      "rewards/margins": 0.01629943773150444,
      "rewards/rejected": -0.04413451999425888,
      "step": 1340
    },
    {
      "epoch": 0.09834990711397662,
      "grad_norm": 1.3326844933328759,
      "learning_rate": 1.3608276348795436e-06,
      "log_odds_chosen": 0.524829089641571,
      "log_odds_ratio": -0.5928710699081421,
      "logits/chosen": -0.921875,
      "logits/rejected": -0.863476574420929,
      "logps/chosen": -0.5997070074081421,
      "logps/rejected": -0.8990234136581421,
      "loss": 0.9826,
      "nll_loss": 0.9371093511581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.02998046949505806,
      "rewards/margins": 0.01499023474752903,
      "rewards/rejected": -0.04497070237994194,
      "step": 1350
    },
    {
      "epoch": 0.09907842494445052,
      "grad_norm": 1.1932033301882545,
      "learning_rate": 1.355815361366601e-06,
      "log_odds_chosen": 0.4991210997104645,
      "log_odds_ratio": -0.5849609375,
      "logits/chosen": -0.9193359613418579,
      "logits/rejected": -0.8232421875,
      "logps/chosen": -0.56298828125,
      "logps/rejected": -0.8343750238418579,
      "loss": 0.9649,
      "nll_loss": 0.9388672113418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02813720703125,
      "rewards/margins": 0.013580322265625,
      "rewards/rejected": -0.04172363132238388,
      "step": 1360
    },
    {
      "epoch": 0.09980694277492441,
      "grad_norm": 1.311074375627691,
      "learning_rate": 1.350858067395748e-06,
      "log_odds_chosen": 0.49232178926467896,
      "log_odds_ratio": -0.573925793170929,
      "logits/chosen": -0.9189453125,
      "logits/rejected": -0.8539062738418579,
      "logps/chosen": -0.5960937738418579,
      "logps/rejected": -0.8902343511581421,
      "loss": 0.9651,
      "nll_loss": 0.971875011920929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.02979736402630806,
      "rewards/margins": 0.014720725826919079,
      "rewards/rejected": -0.04450683668255806,
      "step": 1370
    },
    {
      "epoch": 0.10053546060539832,
      "grad_norm": 1.2477714157583342,
      "learning_rate": 1.345954755145414e-06,
      "log_odds_chosen": 0.5626465082168579,
      "log_odds_ratio": -0.5606445074081421,
      "logits/chosen": -0.9310547113418579,
      "logits/rejected": -0.864062488079071,
      "logps/chosen": -0.6024414300918579,
      "logps/rejected": -0.8939453363418579,
      "loss": 0.9682,
      "nll_loss": 0.879101574420929,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.03011474572122097,
      "rewards/margins": 0.014612197875976562,
      "rewards/rejected": -0.04475097730755806,
      "step": 1380
    },
    {
      "epoch": 0.10126397843587222,
      "grad_norm": 1.2124980773305476,
      "learning_rate": 1.3411044519645502e-06,
      "log_odds_chosen": 0.51727294921875,
      "log_odds_ratio": -0.59228515625,
      "logits/chosen": -0.9482421875,
      "logits/rejected": -0.8890625238418579,
      "logps/chosen": -0.6119140386581421,
      "logps/rejected": -0.8828125,
      "loss": 0.9577,
      "nll_loss": 0.911328136920929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03057861328125,
      "rewards/margins": 0.01357116736471653,
      "rewards/rejected": -0.04415283352136612,
      "step": 1390
    },
    {
      "epoch": 0.10199249626634611,
      "grad_norm": 1.2919449011840545,
      "learning_rate": 1.3363062095621222e-06,
      "log_odds_chosen": 0.6834716796875,
      "log_odds_ratio": -0.537890613079071,
      "logits/chosen": -0.931640625,
      "logits/rejected": -0.8656250238418579,
      "logps/chosen": -0.5894531011581421,
      "logps/rejected": -0.9886718988418579,
      "loss": 0.9707,
      "nll_loss": 0.9837890863418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02947998046875,
      "rewards/margins": 0.019939804449677467,
      "rewards/rejected": -0.04941406100988388,
      "step": 1400
    },
    {
      "epoch": 0.10272101409682002,
      "grad_norm": 4.941812681935647,
      "learning_rate": 1.3315591032282687e-06,
      "log_odds_chosen": 0.69580078125,
      "log_odds_ratio": -0.548535168170929,
      "logits/chosen": -0.9771484136581421,
      "logits/rejected": -0.895703136920929,
      "logps/chosen": -0.573535144329071,
      "logps/rejected": -0.9593750238418579,
      "loss": 0.9559,
      "nll_loss": 0.8851562738418579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02866821363568306,
      "rewards/margins": 0.019317626953125,
      "rewards/rejected": -0.047943115234375,
      "step": 1410
    },
    {
      "epoch": 0.10344953192729392,
      "grad_norm": 1.6833849951825468,
      "learning_rate": 1.3268622310856882e-06,
      "log_odds_chosen": 0.4738525450229645,
      "log_odds_ratio": -0.6070312261581421,
      "logits/chosen": -0.9527343511581421,
      "logits/rejected": -0.8658202886581421,
      "logps/chosen": -0.615429699420929,
      "logps/rejected": -0.9261718988418579,
      "loss": 0.9787,
      "nll_loss": 0.902539074420929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.03078002855181694,
      "rewards/margins": 0.015505981631577015,
      "rewards/rejected": -0.04630737379193306,
      "step": 1420
    },
    {
      "epoch": 0.10417804975776782,
      "grad_norm": 1.4003070140933773,
      "learning_rate": 1.3222147133698626e-06,
      "log_odds_chosen": 0.709033191204071,
      "log_odds_ratio": -0.529589831829071,
      "logits/chosen": -0.9613281488418579,
      "logits/rejected": -0.874218761920929,
      "logps/chosen": -0.641894519329071,
      "logps/rejected": -1.0369141101837158,
      "loss": 0.9469,
      "nll_loss": 1.002539038658142,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.03209228441119194,
      "rewards/margins": 0.01968078687787056,
      "rewards/rejected": -0.05178222805261612,
      "step": 1430
    },
    {
      "epoch": 0.10490656758824173,
      "grad_norm": 1.2888858618279913,
      "learning_rate": 1.3176156917368248e-06,
      "log_odds_chosen": 0.751708984375,
      "log_odds_ratio": -0.548779308795929,
      "logits/chosen": -0.9458984136581421,
      "logits/rejected": -0.8531249761581421,
      "logps/chosen": -0.5958007574081421,
      "logps/rejected": -1.0085937976837158,
      "loss": 0.9582,
      "nll_loss": 0.8910156488418579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02979126013815403,
      "rewards/margins": 0.02060699462890625,
      "rewards/rejected": -0.05042724683880806,
      "step": 1440
    },
    {
      "epoch": 0.10563508541871562,
      "grad_norm": 1.3185522151660192,
      "learning_rate": 1.3130643285972255e-06,
      "log_odds_chosen": 0.6639404296875,
      "log_odds_ratio": -0.5230468511581421,
      "logits/chosen": -0.9085937738418579,
      "logits/rejected": -0.8443359136581421,
      "logps/chosen": -0.5810546875,
      "logps/rejected": -0.956835925579071,
      "loss": 0.9658,
      "nll_loss": 0.948437511920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02906494215130806,
      "rewards/margins": 0.01880187913775444,
      "rewards/rejected": -0.047882080078125,
      "step": 1450
    },
    {
      "epoch": 0.10636360324918952,
      "grad_norm": 1.3138376992847098,
      "learning_rate": 1.3085598064755342e-06,
      "log_odds_chosen": 0.5897216796875,
      "log_odds_ratio": -0.558789074420929,
      "logits/chosen": -0.9457031488418579,
      "logits/rejected": -0.878125011920929,
      "logps/chosen": -0.593457043170929,
      "logps/rejected": -0.9263671636581421,
      "loss": 0.9591,
      "nll_loss": 0.9642578363418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.02964477613568306,
      "rewards/margins": 0.016693878918886185,
      "rewards/rejected": -0.04637451097369194,
      "step": 1460
    },
    {
      "epoch": 0.10709212107966343,
      "grad_norm": 1.4904218937802065,
      "learning_rate": 1.3041013273932528e-06,
      "log_odds_chosen": 0.595898449420929,
      "log_odds_ratio": -0.576953113079071,
      "logits/chosen": -0.9466797113418579,
      "logits/rejected": -0.875,
      "logps/chosen": -0.607128918170929,
      "logps/rejected": -0.959765613079071,
      "loss": 0.9732,
      "nll_loss": 0.8951171636581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03032836876809597,
      "rewards/margins": 0.01769104041159153,
      "rewards/rejected": -0.04802246019244194,
      "step": 1470
    },
    {
      "epoch": 0.10782063891013732,
      "grad_norm": 1.3419027085917665,
      "learning_rate": 1.299688112275091e-06,
      "log_odds_chosen": 0.5328369140625,
      "log_odds_ratio": -0.590136706829071,
      "logits/chosen": -0.935742199420929,
      "logits/rejected": -0.8480468988418579,
      "logps/chosen": -0.6407226324081421,
      "logps/rejected": -0.932421863079071,
      "loss": 0.9569,
      "nll_loss": 0.9326171875,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03204955905675888,
      "rewards/margins": 0.01458740234375,
      "rewards/rejected": -0.04664306715130806,
      "step": 1480
    },
    {
      "epoch": 0.10854915674061123,
      "grad_norm": 1.3389029756713513,
      "learning_rate": 1.2953194003770995e-06,
      "log_odds_chosen": 0.55108642578125,
      "log_odds_ratio": -0.5897461175918579,
      "logits/chosen": -0.949999988079071,
      "logits/rejected": -0.860156238079071,
      "logps/chosen": -0.6131836175918579,
      "logps/rejected": -0.9339843988418579,
      "loss": 0.9633,
      "nll_loss": 0.9380859136581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.030670166015625,
      "rewards/margins": 0.015996551141142845,
      "rewards/rejected": -0.04666747897863388,
      "step": 1490
    },
    {
      "epoch": 0.10927767457108513,
      "grad_norm": 1.1757457908946563,
      "learning_rate": 1.2909944487358056e-06,
      "log_odds_chosen": 0.5101073980331421,
      "log_odds_ratio": -0.581250011920929,
      "logits/chosen": -0.9417968988418579,
      "logits/rejected": -0.857421875,
      "logps/chosen": -0.5838867425918579,
      "logps/rejected": -0.860546886920929,
      "loss": 0.9303,
      "nll_loss": 0.8818359375,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.029205322265625,
      "rewards/margins": 0.013838577084243298,
      "rewards/rejected": -0.043060302734375,
      "step": 1500
    },
    {
      "epoch": 0.11000619240155902,
      "grad_norm": 1.2532459382437797,
      "learning_rate": 1.286712531637447e-06,
      "log_odds_chosen": 0.557568371295929,
      "log_odds_ratio": -0.568066418170929,
      "logits/chosen": -0.945507824420929,
      "logits/rejected": -0.8677734136581421,
      "logps/chosen": -0.6109374761581421,
      "logps/rejected": -0.9365234375,
      "loss": 0.9381,
      "nll_loss": 0.922656238079071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03050537034869194,
      "rewards/margins": 0.016326904296875,
      "rewards/rejected": -0.046875,
      "step": 1510
    },
    {
      "epoch": 0.11073471023203293,
      "grad_norm": 1.2921595878161103,
      "learning_rate": 1.282472940106443e-06,
      "log_odds_chosen": 0.49163818359375,
      "log_odds_ratio": -0.61181640625,
      "logits/chosen": -0.9205077886581421,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -0.6000000238418579,
      "logps/rejected": -0.8841797113418579,
      "loss": 0.9668,
      "nll_loss": 0.9996093511581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03000488318502903,
      "rewards/margins": 0.014185333624482155,
      "rewards/rejected": -0.04419555515050888,
      "step": 1520
    },
    {
      "epoch": 0.11146322806250683,
      "grad_norm": 1.2484947787377194,
      "learning_rate": 1.278274981412284e-06,
      "log_odds_chosen": 0.57080078125,
      "log_odds_ratio": -0.5625,
      "logits/chosen": -0.8935546875,
      "logits/rejected": -0.8285156488418579,
      "logps/chosen": -0.6073242425918579,
      "logps/rejected": -0.9300781488418579,
      "loss": 0.9627,
      "nll_loss": 0.9957031011581421,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.03035888634622097,
      "rewards/margins": 0.016194915398955345,
      "rewards/rejected": -0.04656982421875,
      "step": 1530
    },
    {
      "epoch": 0.11219174589298073,
      "grad_norm": 1.3199462325360787,
      "learning_rate": 1.2741179785940638e-06,
      "log_odds_chosen": 0.37200927734375,
      "log_odds_ratio": -0.669238269329071,
      "logits/chosen": -0.8525390625,
      "logits/rejected": -0.840039074420929,
      "logps/chosen": -0.5772460699081421,
      "logps/rejected": -0.803906261920929,
      "loss": 0.9615,
      "nll_loss": 0.9595702886581421,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.02887573279440403,
      "rewards/margins": 0.011301040649414062,
      "rewards/rejected": -0.04017944261431694,
      "step": 1540
    },
    {
      "epoch": 0.11292026372345464,
      "grad_norm": 1.6364303010818286,
      "learning_rate": 1.270001270001905e-06,
      "log_odds_chosen": 0.614575207233429,
      "log_odds_ratio": -0.5663086175918579,
      "logits/chosen": -0.947460949420929,
      "logits/rejected": -0.868945300579071,
      "logps/chosen": -0.551953136920929,
      "logps/rejected": -0.883984386920929,
      "loss": 0.9594,
      "nll_loss": 0.8955078125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02761230431497097,
      "rewards/margins": 0.016609573736786842,
      "rewards/rejected": -0.04420166090130806,
      "step": 1550
    },
    {
      "epoch": 0.11364878155392853,
      "grad_norm": 1.1883102478914753,
      "learning_rate": 1.2659242088545834e-06,
      "log_odds_chosen": 0.4511962831020355,
      "log_odds_ratio": -0.611035168170929,
      "logits/chosen": -0.982226550579071,
      "logits/rejected": -0.884960949420929,
      "logps/chosen": -0.6053711175918579,
      "logps/rejected": -0.867382824420929,
      "loss": 0.9453,
      "nll_loss": 0.8861328363418579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.030242919921875,
      "rewards/margins": 0.013089751824736595,
      "rewards/rejected": -0.043365478515625,
      "step": 1560
    },
    {
      "epoch": 0.11437729938440243,
      "grad_norm": 1.3371544417310577,
      "learning_rate": 1.261886162812672e-06,
      "log_odds_chosen": 0.47578126192092896,
      "log_odds_ratio": -0.622363269329071,
      "logits/chosen": -0.9498046636581421,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.6170898675918579,
      "logps/rejected": -0.87548828125,
      "loss": 0.9804,
      "nll_loss": 0.950390636920929,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.03086547926068306,
      "rewards/margins": 0.01291809044778347,
      "rewards/rejected": -0.04375610500574112,
      "step": 1570
    },
    {
      "epoch": 0.11510581721487634,
      "grad_norm": 1.284974784009936,
      "learning_rate": 1.257886513566569e-06,
      "log_odds_chosen": 0.3924560546875,
      "log_odds_ratio": -0.607617199420929,
      "logits/chosen": -0.916796863079071,
      "logits/rejected": -0.8363281488418579,
      "logps/chosen": -0.589160144329071,
      "logps/rejected": -0.8099609613418579,
      "loss": 0.9537,
      "nll_loss": 0.923046886920929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.02945556677877903,
      "rewards/margins": 0.011016654781997204,
      "rewards/rejected": -0.04046020656824112,
      "step": 1580
    },
    {
      "epoch": 0.11583433504535023,
      "grad_norm": 1.3420827275901521,
      "learning_rate": 1.253924656438798e-06,
      "log_odds_chosen": 0.652026355266571,
      "log_odds_ratio": -0.5401366949081421,
      "logits/chosen": -0.962109386920929,
      "logits/rejected": -0.8604491949081421,
      "logps/chosen": -0.5962890386581421,
      "logps/rejected": -0.960742175579071,
      "loss": 0.974,
      "nll_loss": 0.946093738079071,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.029815673828125,
      "rewards/margins": 0.018260955810546875,
      "rewards/rejected": -0.04807128757238388,
      "step": 1590
    },
    {
      "epoch": 0.11656285287582413,
      "grad_norm": 1.3076136477290436,
      "learning_rate": 1.25e-06,
      "log_odds_chosen": 0.688720703125,
      "log_odds_ratio": -0.525390625,
      "logits/chosen": -0.9271484613418579,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.606152355670929,
      "logps/rejected": -1.0046875476837158,
      "loss": 0.9741,
      "nll_loss": 0.931835949420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.03029174730181694,
      "rewards/margins": 0.01995544508099556,
      "rewards/rejected": -0.05021972581744194,
      "step": 1600
    },
    {
      "epoch": 0.11729137070629804,
      "grad_norm": 1.3361197712375694,
      "learning_rate": 1.246111965698067e-06,
      "log_odds_chosen": 0.7662597894668579,
      "log_odds_ratio": -0.5550781488418579,
      "logits/chosen": -0.964648425579071,
      "logits/rejected": -0.837695300579071,
      "logps/chosen": -0.56982421875,
      "logps/rejected": -1.025390625,
      "loss": 0.9642,
      "nll_loss": 0.9447265863418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.02850341796875,
      "rewards/margins": 0.022796630859375,
      "rewards/rejected": -0.05131835862994194,
      "step": 1610
    },
    {
      "epoch": 0.11801988853677194,
      "grad_norm": 1.2855754652522504,
      "learning_rate": 1.2422599874998834e-06,
      "log_odds_chosen": 0.6658691167831421,
      "log_odds_ratio": -0.539843738079071,
      "logits/chosen": -0.904101550579071,
      "logits/rejected": -0.841796875,
      "logps/chosen": -0.576855480670929,
      "logps/rejected": -0.9595702886581421,
      "loss": 0.9563,
      "nll_loss": 0.9291015863418579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02882080152630806,
      "rewards/margins": 0.019127655774354935,
      "rewards/rejected": -0.04798584058880806,
      "step": 1620
    },
    {
      "epoch": 0.11874840636724583,
      "grad_norm": 1.3150100058761525,
      "learning_rate": 1.238443511545175e-06,
      "log_odds_chosen": 0.807177722454071,
      "log_odds_ratio": -0.4892578125,
      "logits/chosen": -0.9273437261581421,
      "logits/rejected": -0.8257812261581421,
      "logps/chosen": -0.620800793170929,
      "logps/rejected": -1.105859398841858,
      "loss": 0.953,
      "nll_loss": 0.9984375238418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.031036376953125,
      "rewards/margins": 0.02426452562212944,
      "rewards/rejected": -0.0552978515625,
      "step": 1630
    },
    {
      "epoch": 0.11947692419771974,
      "grad_norm": 1.2545064108952282,
      "learning_rate": 1.2346619958119873e-06,
      "log_odds_chosen": 0.6653808355331421,
      "log_odds_ratio": -0.5630859136581421,
      "logits/chosen": -0.989062488079071,
      "logits/rejected": -0.908984363079071,
      "logps/chosen": -0.6268554925918579,
      "logps/rejected": -0.98876953125,
      "loss": 0.9612,
      "nll_loss": 0.9462890625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03135376051068306,
      "rewards/margins": 0.018077850341796875,
      "rewards/rejected": -0.0494384765625,
      "step": 1640
    },
    {
      "epoch": 0.12020544202819364,
      "grad_norm": 1.278951850838904,
      "learning_rate": 1.2309149097933274e-06,
      "log_odds_chosen": 0.5755981206893921,
      "log_odds_ratio": -0.54833984375,
      "logits/chosen": -0.942187488079071,
      "logits/rejected": -0.872265636920929,
      "logps/chosen": -0.53515625,
      "logps/rejected": -0.840039074420929,
      "loss": 0.9435,
      "nll_loss": 0.9214843511581421,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02673950232565403,
      "rewards/margins": 0.015283966436982155,
      "rewards/rejected": -0.04202880710363388,
      "step": 1650
    },
    {
      "epoch": 0.12093395985866755,
      "grad_norm": 1.3508267466163653,
      "learning_rate": 1.2272017341845401e-06,
      "log_odds_chosen": 0.5066894292831421,
      "log_odds_ratio": -0.604296863079071,
      "logits/chosen": -0.89453125,
      "logits/rejected": -0.8345702886581421,
      "logps/chosen": -0.5733398199081421,
      "logps/rejected": -0.8443359136581421,
      "loss": 0.9446,
      "nll_loss": 0.943554699420929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.028656005859375,
      "rewards/margins": 0.013578033074736595,
      "rewards/rejected": -0.04222412034869194,
      "step": 1660
    },
    {
      "epoch": 0.12166247768914144,
      "grad_norm": 1.3823525644671513,
      "learning_rate": 1.223521960580991e-06,
      "log_odds_chosen": 0.647106945514679,
      "log_odds_ratio": -0.56640625,
      "logits/chosen": -0.9521484375,
      "logits/rejected": -0.892382800579071,
      "logps/chosen": -0.6172851324081421,
      "logps/rejected": -0.987109363079071,
      "loss": 0.9513,
      "nll_loss": 0.9537109136581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.03087768517434597,
      "rewards/margins": 0.018463134765625,
      "rewards/rejected": -0.04936523362994194,
      "step": 1670
    },
    {
      "epoch": 0.12239099551961534,
      "grad_norm": 1.3435049123567973,
      "learning_rate": 1.2198750911856664e-06,
      "log_odds_chosen": 0.538989245891571,
      "log_odds_ratio": -0.5931640863418579,
      "logits/chosen": -0.9624999761581421,
      "logits/rejected": -0.900390625,
      "logps/chosen": -0.5772460699081421,
      "logps/rejected": -0.864453136920929,
      "loss": 0.9582,
      "nll_loss": 0.906054675579071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.02887573279440403,
      "rewards/margins": 0.014349746517837048,
      "rewards/rejected": -0.04322509840130806,
      "step": 1680
    },
    {
      "epoch": 0.12311951335008925,
      "grad_norm": 1.2816799341263028,
      "learning_rate": 1.2162606385262997e-06,
      "log_odds_chosen": 0.614428699016571,
      "log_odds_ratio": -0.5751953125,
      "logits/chosen": -0.986523449420929,
      "logits/rejected": -0.8984375,
      "logps/chosen": -0.6356445550918579,
      "logps/rejected": -0.99365234375,
      "loss": 0.9326,
      "nll_loss": 0.8257812261581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.03178710862994194,
      "rewards/margins": 0.01796264573931694,
      "rewards/rejected": -0.04974975436925888,
      "step": 1690
    },
    {
      "epoch": 0.12384803118056315,
      "grad_norm": 1.4596403301161887,
      "learning_rate": 1.2126781251816649e-06,
      "log_odds_chosen": 0.653881847858429,
      "log_odds_ratio": -0.5506836175918579,
      "logits/chosen": -0.9359375238418579,
      "logits/rejected": -0.8628906011581421,
      "logps/chosen": -0.582714855670929,
      "logps/rejected": -0.961718738079071,
      "loss": 0.9375,
      "nll_loss": 0.857226550579071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02913818322122097,
      "rewards/margins": 0.01894226111471653,
      "rewards/rejected": -0.04805908352136612,
      "step": 1700
    },
    {
      "epoch": 0.12457654901103704,
      "grad_norm": 1.3220127324428235,
      "learning_rate": 1.2091270835166862e-06,
      "log_odds_chosen": 0.6234070062637329,
      "log_odds_ratio": -0.5816406011581421,
      "logits/chosen": -0.9111328125,
      "logits/rejected": -0.822460949420929,
      "logps/chosen": -0.5565429925918579,
      "logps/rejected": -0.9146484136581421,
      "loss": 0.9573,
      "nll_loss": 0.960156261920929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.02780761756002903,
      "rewards/margins": 0.0178985595703125,
      "rewards/rejected": -0.04573974758386612,
      "step": 1710
    },
    {
      "epoch": 0.12530506684151094,
      "grad_norm": 1.327484855675949,
      "learning_rate": 1.2056070554260305e-06,
      "log_odds_chosen": 0.622668445110321,
      "log_odds_ratio": -0.549023449420929,
      "logits/chosen": -0.981249988079071,
      "logits/rejected": -0.8853515386581421,
      "logps/chosen": -0.5819336175918579,
      "logps/rejected": -0.9375,
      "loss": 0.9236,
      "nll_loss": 0.885546863079071,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.02909545972943306,
      "rewards/margins": 0.017787933349609375,
      "rewards/rejected": -0.04691161960363388,
      "step": 1720
    },
    {
      "epoch": 0.12603358467198486,
      "grad_norm": 1.2523125475887857,
      "learning_rate": 1.2021175920858626e-06,
      "log_odds_chosen": 0.651416003704071,
      "log_odds_ratio": -0.576464831829071,
      "logits/chosen": -0.9912109375,
      "logits/rejected": -0.877148449420929,
      "logps/chosen": -0.607617199420929,
      "logps/rejected": -1.0119140148162842,
      "loss": 0.9576,
      "nll_loss": 0.9458984136581421,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.03037109412252903,
      "rewards/margins": 0.020226668566465378,
      "rewards/rejected": -0.05057372897863388,
      "step": 1730
    },
    {
      "epoch": 0.12676210250245876,
      "grad_norm": 1.3183942364486774,
      "learning_rate": 1.1986582537134606e-06,
      "log_odds_chosen": 0.564453125,
      "log_odds_ratio": -0.561816394329071,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.8951171636581421,
      "logps/chosen": -0.593945324420929,
      "logps/rejected": -0.925000011920929,
      "loss": 0.9538,
      "nll_loss": 0.931640625,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.02965698204934597,
      "rewards/margins": 0.01659240759909153,
      "rewards/rejected": -0.04625244066119194,
      "step": 1740
    },
    {
      "epoch": 0.12749062033293265,
      "grad_norm": 1.2454443371504929,
      "learning_rate": 1.1952286093343937e-06,
      "log_odds_chosen": 0.59912109375,
      "log_odds_ratio": -0.5645507574081421,
      "logits/chosen": -0.983203113079071,
      "logits/rejected": -0.9232422113418579,
      "logps/chosen": -0.5702148675918579,
      "logps/rejected": -0.8984375,
      "loss": 0.9521,
      "nll_loss": 0.916015625,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02849121019244194,
      "rewards/margins": 0.016418838873505592,
      "rewards/rejected": -0.04493408277630806,
      "step": 1750
    },
    {
      "epoch": 0.12821913816340655,
      "grad_norm": 1.6059630447509174,
      "learning_rate": 1.1918282365569903e-06,
      "log_odds_chosen": 0.5472656488418579,
      "log_odds_ratio": -0.5777343511581421,
      "logits/chosen": -0.9771484136581421,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.55810546875,
      "logps/rejected": -0.886914074420929,
      "loss": 0.9354,
      "nll_loss": 0.869140625,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.02788085862994194,
      "rewards/margins": 0.01643676683306694,
      "rewards/rejected": -0.04433593899011612,
      "step": 1760
    },
    {
      "epoch": 0.12894765599388044,
      "grad_norm": 1.2895080667249628,
      "learning_rate": 1.1884567213538209e-06,
      "log_odds_chosen": 0.681384265422821,
      "log_odds_ratio": -0.543164074420929,
      "logits/chosen": -0.9759765863418579,
      "logits/rejected": -0.8695312738418579,
      "logps/chosen": -0.563183605670929,
      "logps/rejected": -0.9720703363418579,
      "loss": 0.9625,
      "nll_loss": 0.9292968511581421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02816162072122097,
      "rewards/margins": 0.02044830285012722,
      "rewards/rejected": -0.04863281175494194,
      "step": 1770
    },
    {
      "epoch": 0.12967617382435434,
      "grad_norm": 1.432007243285182,
      "learning_rate": 1.1851136578499433e-06,
      "log_odds_chosen": 0.6414794921875,
      "log_odds_ratio": -0.5909179449081421,
      "logits/chosen": -0.9937499761581421,
      "logits/rejected": -0.9205077886581421,
      "logps/chosen": -0.590527355670929,
      "logps/rejected": -0.9935547113418579,
      "loss": 0.9375,
      "nll_loss": 0.920117199420929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02952880784869194,
      "rewards/margins": 0.02018432691693306,
      "rewards/rejected": -0.04968871921300888,
      "step": 1780
    },
    {
      "epoch": 0.13040469165482826,
      "grad_norm": 1.2978825032201369,
      "learning_rate": 1.181798648117664e-06,
      "log_odds_chosen": 0.8204101324081421,
      "log_odds_ratio": -0.5387207269668579,
      "logits/chosen": -0.955859363079071,
      "logits/rejected": -0.883984386920929,
      "logps/chosen": -0.6548827886581421,
      "logps/rejected": -1.182226538658142,
      "loss": 0.9678,
      "nll_loss": 0.943164050579071,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.03273925930261612,
      "rewards/margins": 0.0263824462890625,
      "rewards/rejected": -0.05911865085363388,
      "step": 1790
    },
    {
      "epoch": 0.13113320948530216,
      "grad_norm": 1.3145089191382204,
      "learning_rate": 1.1785113019775794e-06,
      "log_odds_chosen": 0.609326183795929,
      "log_odds_ratio": -0.555957019329071,
      "logits/chosen": -0.9691406488418579,
      "logits/rejected": -0.9037109613418579,
      "logps/chosen": -0.5562499761581421,
      "logps/rejected": -0.9136718511581421,
      "loss": 0.956,
      "nll_loss": 0.862109363079071,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.02777709998190403,
      "rewards/margins": 0.01788940466940403,
      "rewards/rejected": -0.04570312425494194,
      "step": 1800
    },
    {
      "epoch": 0.13186172731577606,
      "grad_norm": 1.3842440828057032,
      "learning_rate": 1.1752512368056712e-06,
      "log_odds_chosen": 0.697174072265625,
      "log_odds_ratio": -0.554492175579071,
      "logits/chosen": -0.9544922113418579,
      "logits/rejected": -0.8531249761581421,
      "logps/chosen": -0.561328113079071,
      "logps/rejected": -0.985156238079071,
      "loss": 0.9704,
      "nll_loss": 0.9281250238418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.02808837965130806,
      "rewards/margins": 0.021233748644590378,
      "rewards/rejected": -0.04929199069738388,
      "step": 1810
    },
    {
      "epoch": 0.13259024514624995,
      "grad_norm": 1.4436770025771462,
      "learning_rate": 1.1720180773462387e-06,
      "log_odds_chosen": 0.578808605670929,
      "log_odds_ratio": -0.624804675579071,
      "logits/chosen": -0.958984375,
      "logits/rejected": -0.9212890863418579,
      "logps/chosen": -0.607714831829071,
      "logps/rejected": -0.9820312261581421,
      "loss": 0.9504,
      "nll_loss": 0.870312511920929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03035888634622097,
      "rewards/margins": 0.01871032640337944,
      "rewards/rejected": -0.04906005784869194,
      "step": 1820
    },
    {
      "epoch": 0.13331876297672385,
      "grad_norm": 1.3539859055516792,
      "learning_rate": 1.168811455530461e-06,
      "log_odds_chosen": 0.653076171875,
      "log_odds_ratio": -0.554394543170929,
      "logits/chosen": -0.94921875,
      "logits/rejected": -0.8306640386581421,
      "logps/chosen": -0.5801757574081421,
      "logps/rejected": -0.942187488079071,
      "loss": 0.9469,
      "nll_loss": 0.904296875,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.02900390699505806,
      "rewards/margins": 0.01810913160443306,
      "rewards/rejected": -0.04714355617761612,
      "step": 1830
    },
    {
      "epoch": 0.13404728080719774,
      "grad_norm": 1.2259541612926108,
      "learning_rate": 1.1656310103003923e-06,
      "log_odds_chosen": 0.47148436307907104,
      "log_odds_ratio": -0.607421875,
      "logits/chosen": -0.9671875238418579,
      "logits/rejected": -0.8916015625,
      "logps/chosen": -0.6221679449081421,
      "logps/rejected": -0.90625,
      "loss": 0.9347,
      "nll_loss": 0.958203136920929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03109130822122097,
      "rewards/margins": 0.014196014031767845,
      "rewards/rejected": -0.04530029371380806,
      "step": 1840
    },
    {
      "epoch": 0.13477579863767167,
      "grad_norm": 1.3244950854265143,
      "learning_rate": 1.162476387438193e-06,
      "log_odds_chosen": 0.6208740472793579,
      "log_odds_ratio": -0.5630859136581421,
      "logits/chosen": -0.912890613079071,
      "logits/rejected": -0.8568359613418579,
      "logps/chosen": -0.5648437738418579,
      "logps/rejected": -0.9144531488418579,
      "loss": 0.9453,
      "nll_loss": 0.951367199420929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02823486365377903,
      "rewards/margins": 0.01750030554831028,
      "rewards/rejected": -0.045745849609375,
      "step": 1850
    },
    {
      "epoch": 0.13550431646814556,
      "grad_norm": 1.3244488977997921,
      "learning_rate": 1.1593472394004206e-06,
      "log_odds_chosen": 0.647448718547821,
      "log_odds_ratio": -0.582714855670929,
      "logits/chosen": -0.9322265386581421,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.6055663824081421,
      "logps/rejected": -1.016992211341858,
      "loss": 0.9589,
      "nll_loss": 0.9193359613418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.030303955078125,
      "rewards/margins": 0.020558929070830345,
      "rewards/rejected": -0.05085449293255806,
      "step": 1860
    },
    {
      "epoch": 0.13623283429861946,
      "grad_norm": 1.3233942331915476,
      "learning_rate": 1.1562432251572007e-06,
      "log_odds_chosen": 0.7266601324081421,
      "log_odds_ratio": -0.521777331829071,
      "logits/chosen": -0.9232422113418579,
      "logits/rejected": -0.835156261920929,
      "logps/chosen": -0.5946289300918579,
      "logps/rejected": -1.015234351158142,
      "loss": 0.9771,
      "nll_loss": 0.962695300579071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02974243089556694,
      "rewards/margins": 0.02100372314453125,
      "rewards/rejected": -0.05074463039636612,
      "step": 1870
    },
    {
      "epoch": 0.13696135212909336,
      "grad_norm": 1.3253484327708271,
      "learning_rate": 1.1531640100361064e-06,
      "log_odds_chosen": 0.6776367425918579,
      "log_odds_ratio": -0.5423828363418579,
      "logits/chosen": -0.956835925579071,
      "logits/rejected": -0.877148449420929,
      "logps/chosen": -0.5625,
      "logps/rejected": -0.974414050579071,
      "loss": 0.946,
      "nll_loss": 0.9105468988418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02811889722943306,
      "rewards/margins": 0.02058105543255806,
      "rewards/rejected": -0.04872436448931694,
      "step": 1880
    },
    {
      "epoch": 0.13768986995956725,
      "grad_norm": 1.2919358446389717,
      "learning_rate": 1.1501092655705905e-06,
      "log_odds_chosen": 0.6642822027206421,
      "log_odds_ratio": -0.5667968988418579,
      "logits/chosen": -0.9384765625,
      "logits/rejected": -0.868359386920929,
      "logps/chosen": -0.5782226324081421,
      "logps/rejected": -0.9957031011581421,
      "loss": 0.9576,
      "nll_loss": 0.966601550579071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02890625037252903,
      "rewards/margins": 0.02089233323931694,
      "rewards/rejected": -0.04978637769818306,
      "step": 1890
    },
    {
      "epoch": 0.13841838779004115,
      "grad_norm": 1.3787442425304846,
      "learning_rate": 1.1470786693528087e-06,
      "log_odds_chosen": 0.7455078363418579,
      "log_odds_ratio": -0.5323241949081421,
      "logits/chosen": -0.9593750238418579,
      "logits/rejected": -0.8466796875,
      "logps/chosen": -0.6182616949081421,
      "logps/rejected": -1.051367163658142,
      "loss": 0.9465,
      "nll_loss": 0.9332031011581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.03089599683880806,
      "rewards/margins": 0.021697234362363815,
      "rewards/rejected": -0.0526123046875,
      "step": 1900
    },
    {
      "epoch": 0.13914690562051507,
      "grad_norm": 1.294896327352322,
      "learning_rate": 1.144071904890689e-06,
      "log_odds_chosen": 0.7734375,
      "log_odds_ratio": -0.52294921875,
      "logits/chosen": -0.9466797113418579,
      "logits/rejected": -0.8935546875,
      "logps/chosen": -0.5801757574081421,
      "logps/rejected": -1.0333983898162842,
      "loss": 0.9115,
      "nll_loss": 0.861328125,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02901611290872097,
      "rewards/margins": 0.02261199988424778,
      "rewards/rejected": -0.0516357421875,
      "step": 1910
    },
    {
      "epoch": 0.13987542345098897,
      "grad_norm": 1.2823046082289764,
      "learning_rate": 1.1410886614690962e-06,
      "log_odds_chosen": 0.48529052734375,
      "log_odds_ratio": -0.595898449420929,
      "logits/chosen": -0.941601574420929,
      "logits/rejected": -0.851757824420929,
      "logps/chosen": -0.54931640625,
      "logps/rejected": -0.79541015625,
      "loss": 0.9527,
      "nll_loss": 0.871874988079071,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.02744751051068306,
      "rewards/margins": 0.012314987368881702,
      "rewards/rejected": -0.03978271409869194,
      "step": 1920
    },
    {
      "epoch": 0.14060394128146286,
      "grad_norm": 1.3470490767216472,
      "learning_rate": 1.1381286340149635e-06,
      "log_odds_chosen": 0.682666003704071,
      "log_odds_ratio": -0.546679675579071,
      "logits/chosen": -0.919726550579071,
      "logits/rejected": -0.8228515386581421,
      "logps/chosen": -0.5894531011581421,
      "logps/rejected": -0.989062488079071,
      "loss": 0.9305,
      "nll_loss": 0.8626953363418579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02946167066693306,
      "rewards/margins": 0.01997833326458931,
      "rewards/rejected": -0.0494384765625,
      "step": 1930
    },
    {
      "epoch": 0.14133245911193676,
      "grad_norm": 1.3278970568945683,
      "learning_rate": 1.1351915229662496e-06,
      "log_odds_chosen": 0.7845703363418579,
      "log_odds_ratio": -0.5106445550918579,
      "logits/chosen": -0.9310547113418579,
      "logits/rejected": -0.8701171875,
      "logps/chosen": -0.558300793170929,
      "logps/rejected": -1.0066406726837158,
      "loss": 0.9355,
      "nll_loss": 0.9085937738418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.027923583984375,
      "rewards/margins": 0.02241058275103569,
      "rewards/rejected": -0.05032958835363388,
      "step": 1940
    },
    {
      "epoch": 0.14206097694241065,
      "grad_norm": 1.2836092960119463,
      "learning_rate": 1.1322770341445958e-06,
      "log_odds_chosen": 0.732421875,
      "log_odds_ratio": -0.531445324420929,
      "logits/chosen": -0.949414074420929,
      "logits/rejected": -0.8519531488418579,
      "logps/chosen": -0.560546875,
      "logps/rejected": -0.9671875238418579,
      "loss": 0.9427,
      "nll_loss": 0.9017578363418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0280303955078125,
      "rewards/margins": 0.0203094482421875,
      "rewards/rejected": -0.04835205152630806,
      "step": 1950
    },
    {
      "epoch": 0.14278949477288458,
      "grad_norm": 1.2930982826501454,
      "learning_rate": 1.1293848786315642e-06,
      "log_odds_chosen": 0.541210949420929,
      "log_odds_ratio": -0.6036132574081421,
      "logits/chosen": -0.972851574420929,
      "logits/rejected": -0.876757800579071,
      "logps/chosen": -0.61083984375,
      "logps/rejected": -0.943554699420929,
      "loss": 0.9536,
      "nll_loss": 0.98828125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.030548095703125,
      "rewards/margins": 0.016637612134218216,
      "rewards/rejected": -0.04718017578125,
      "step": 1960
    },
    {
      "epoch": 0.14351801260335847,
      "grad_norm": 1.4837226598761686,
      "learning_rate": 1.1265147726483323e-06,
      "log_odds_chosen": 0.883544921875,
      "log_odds_ratio": -0.512890636920929,
      "logits/chosen": -0.9701172113418579,
      "logits/rejected": -0.8812500238418579,
      "logps/chosen": -0.556347668170929,
      "logps/rejected": -1.0828125476837158,
      "loss": 0.9281,
      "nll_loss": 0.903124988079071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02781982347369194,
      "rewards/margins": 0.026349639520049095,
      "rewards/rejected": -0.05412597581744194,
      "step": 1970
    },
    {
      "epoch": 0.14424653043383237,
      "grad_norm": 1.3751410014618848,
      "learning_rate": 1.1236664374387369e-06,
      "log_odds_chosen": 0.684741199016571,
      "log_odds_ratio": -0.5445312261581421,
      "logits/chosen": -0.9146484136581421,
      "logits/rejected": -0.8257812261581421,
      "logps/chosen": -0.608593761920929,
      "logps/rejected": -1.012304663658142,
      "loss": 0.9409,
      "nll_loss": 0.9173828363418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.03044433519244194,
      "rewards/margins": 0.020191192626953125,
      "rewards/rejected": -0.05063476413488388,
      "step": 1980
    },
    {
      "epoch": 0.14497504826430627,
      "grad_norm": 10.46038280014368,
      "learning_rate": 1.120839599155551e-06,
      "log_odds_chosen": 0.550854504108429,
      "log_odds_ratio": -0.5843750238418579,
      "logits/chosen": -0.9380859136581421,
      "logits/rejected": -0.854687511920929,
      "logps/chosen": -0.6268554925918579,
      "logps/rejected": -0.9517577886581421,
      "loss": 0.9395,
      "nll_loss": 0.918749988079071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.03134765475988388,
      "rewards/margins": 0.016242217272520065,
      "rewards/rejected": -0.04757080227136612,
      "step": 1990
    },
    {
      "epoch": 0.14570356609478016,
      "grad_norm": 1.3749440836669078,
      "learning_rate": 1.118033988749895e-06,
      "log_odds_chosen": 0.807666003704071,
      "log_odds_ratio": -0.50341796875,
      "logits/chosen": -0.91015625,
      "logits/rejected": -0.846875011920929,
      "logps/chosen": -0.616992175579071,
      "logps/rejected": -1.106054663658142,
      "loss": 0.9461,
      "nll_loss": 0.8974609375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.03085937537252903,
      "rewards/margins": 0.02443542517721653,
      "rewards/rejected": -0.05526123195886612,
      "step": 2000
    },
    {
      "epoch": 0.14643208392525406,
      "grad_norm": 1.3492245988210907,
      "learning_rate": 1.1152493418636764e-06,
      "log_odds_chosen": 0.696582019329071,
      "log_odds_ratio": -0.5462890863418579,
      "logits/chosen": -0.9349609613418579,
      "logits/rejected": -0.858203113079071,
      "logps/chosen": -0.5687500238418579,
      "logps/rejected": -0.967578113079071,
      "loss": 0.9384,
      "nll_loss": 0.8666015863418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.02843017503619194,
      "rewards/margins": 0.019966889172792435,
      "rewards/rejected": -0.04837646335363388,
      "step": 2010
    },
    {
      "epoch": 0.14716060175572798,
      "grad_norm": 1.400204671204439,
      "learning_rate": 1.112485398724962e-06,
      "log_odds_chosen": 0.8431152105331421,
      "log_odds_ratio": -0.518359363079071,
      "logits/chosen": -0.9222656488418579,
      "logits/rejected": -0.83203125,
      "logps/chosen": -0.551464855670929,
      "logps/rejected": -1.066796898841858,
      "loss": 0.9445,
      "nll_loss": 0.883984386920929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.02756958082318306,
      "rewards/margins": 0.025755692273378372,
      "rewards/rejected": -0.05332031100988388,
      "step": 2020
    },
    {
      "epoch": 0.14788911958620188,
      "grad_norm": 1.3495868570714293,
      "learning_rate": 1.1097419040461884e-06,
      "log_odds_chosen": 0.8280273675918579,
      "log_odds_ratio": -0.4803710877895355,
      "logits/chosen": -0.952343761920929,
      "logits/rejected": -0.8646484613418579,
      "logps/chosen": -0.5425781011581421,
      "logps/rejected": -1.009374976158142,
      "loss": 0.9291,
      "nll_loss": 0.898632824420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.027130126953125,
      "rewards/margins": 0.02330780029296875,
      "rewards/rejected": -0.0504150390625,
      "step": 2030
    },
    {
      "epoch": 0.14861763741667577,
      "grad_norm": 1.2627254572160034,
      "learning_rate": 1.1070186069251193e-06,
      "log_odds_chosen": 0.624713122844696,
      "log_odds_ratio": -0.592578113079071,
      "logits/chosen": -0.954882800579071,
      "logits/rejected": -0.8832031488418579,
      "logps/chosen": -0.582812488079071,
      "logps/rejected": -0.9847656488418579,
      "loss": 0.9301,
      "nll_loss": 0.9593750238418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.02913818322122097,
      "rewards/margins": 0.020095061510801315,
      "rewards/rejected": -0.04926757887005806,
      "step": 2040
    },
    {
      "epoch": 0.14934615524714967,
      "grad_norm": 1.5086741234118817,
      "learning_rate": 1.1043152607484655e-06,
      "log_odds_chosen": 0.863964855670929,
      "log_odds_ratio": -0.4957031309604645,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.8916015625,
      "logps/chosen": -0.6045898199081421,
      "logps/rejected": -1.1388671398162842,
      "loss": 0.9382,
      "nll_loss": 0.972460925579071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.03023071214556694,
      "rewards/margins": 0.0267486572265625,
      "rewards/rejected": -0.05699462816119194,
      "step": 2050
    },
    {
      "epoch": 0.15007467307762357,
      "grad_norm": 1.3530086520833953,
      "learning_rate": 1.1016316230980794e-06,
      "log_odds_chosen": 0.7586914300918579,
      "log_odds_ratio": -0.522900402545929,
      "logits/chosen": -0.9892578125,
      "logits/rejected": -0.906054675579071,
      "logps/chosen": -0.5499023199081421,
      "logps/rejected": -0.9781249761581421,
      "loss": 0.9442,
      "nll_loss": 0.854687511920929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02750854566693306,
      "rewards/margins": 0.021379852667450905,
      "rewards/rejected": -0.04890746995806694,
      "step": 2060
    },
    {
      "epoch": 0.15080319090809746,
      "grad_norm": 1.3247770000632113,
      "learning_rate": 1.098967455659645e-06,
      "log_odds_chosen": 0.7410644292831421,
      "log_odds_ratio": -0.5425781011581421,
      "logits/chosen": -0.954296886920929,
      "logits/rejected": -0.8736327886581421,
      "logps/chosen": -0.584667980670929,
      "logps/rejected": -1.014062523841858,
      "loss": 0.9465,
      "nll_loss": 0.883593738079071,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02922363206744194,
      "rewards/margins": 0.021475981920957565,
      "rewards/rejected": -0.05068359524011612,
      "step": 2070
    },
    {
      "epoch": 0.15153170873857139,
      "grad_norm": 1.3269718847423708,
      "learning_rate": 1.0963225241337867e-06,
      "log_odds_chosen": 0.5111938714981079,
      "log_odds_ratio": -0.6070312261581421,
      "logits/chosen": -0.9615234136581421,
      "logits/rejected": -0.8687499761581421,
      "logps/chosen": -0.60888671875,
      "logps/rejected": -0.925488293170929,
      "loss": 0.9372,
      "nll_loss": 0.8998047113418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.03046875074505806,
      "rewards/margins": 0.01577911339700222,
      "rewards/rejected": -0.04624023288488388,
      "step": 2080
    },
    {
      "epoch": 0.15226022656904528,
      "grad_norm": 1.4669329271055571,
      "learning_rate": 1.093696598149518e-06,
      "log_odds_chosen": 0.747631847858429,
      "log_odds_ratio": -0.544921875,
      "logits/chosen": -0.966601550579071,
      "logits/rejected": -0.900390625,
      "logps/chosen": -0.5731445550918579,
      "logps/rejected": -1.020898461341858,
      "loss": 0.941,
      "nll_loss": 0.938281238079071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.02866210974752903,
      "rewards/margins": 0.022378157824277878,
      "rewards/rejected": -0.05107421800494194,
      "step": 2090
    },
    {
      "epoch": 0.15298874439951918,
      "grad_norm": 1.3879374786688168,
      "learning_rate": 1.091089451179962e-06,
      "log_odds_chosen": 0.727020263671875,
      "log_odds_ratio": -0.5693359375,
      "logits/chosen": -0.8980468511581421,
      "logits/rejected": -0.8607422113418579,
      "logps/chosen": -0.5638672113418579,
      "logps/rejected": -0.9468749761581421,
      "loss": 0.9321,
      "nll_loss": 0.9703124761581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02821655198931694,
      "rewards/margins": 0.019098663702607155,
      "rewards/rejected": -0.04732055589556694,
      "step": 2100
    },
    {
      "epoch": 0.15371726222999307,
      "grad_norm": 1.4563523285091753,
      "learning_rate": 1.0885008604602703e-06,
      "log_odds_chosen": 0.6163574457168579,
      "log_odds_ratio": -0.562792956829071,
      "logits/chosen": -1.031640648841858,
      "logits/rejected": -0.9371093511581421,
      "logps/chosen": -0.5897461175918579,
      "logps/rejected": -0.947070300579071,
      "loss": 0.9445,
      "nll_loss": 0.9359375238418579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02949829027056694,
      "rewards/margins": 0.017831802368164062,
      "rewards/rejected": -0.04732666164636612,
      "step": 2110
    },
    {
      "epoch": 0.15444578006046697,
      "grad_norm": 1.4881778706091766,
      "learning_rate": 1.0859306069076736e-06,
      "log_odds_chosen": 0.718798816204071,
      "log_odds_ratio": -0.532910168170929,
      "logits/chosen": -0.993945300579071,
      "logits/rejected": -0.891406238079071,
      "logps/chosen": -0.578125,
      "logps/rejected": -1.006445288658142,
      "loss": 0.9467,
      "nll_loss": 0.8753906488418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.028900146484375,
      "rewards/margins": 0.021448517218232155,
      "rewards/rejected": -0.05034179612994194,
      "step": 2120
    },
    {
      "epoch": 0.1551742978909409,
      "grad_norm": 1.3888405220618039,
      "learning_rate": 1.083378475043599e-06,
      "log_odds_chosen": 0.602294921875,
      "log_odds_ratio": -0.579394519329071,
      "logits/chosen": -0.9730468988418579,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.5956054925918579,
      "logps/rejected": -0.9642578363418579,
      "loss": 0.9406,
      "nll_loss": 0.954296886920929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02976684644818306,
      "rewards/margins": 0.01840667799115181,
      "rewards/rejected": -0.048187255859375,
      "step": 2130
    },
    {
      "epoch": 0.1559028157214148,
      "grad_norm": 1.4330401366045449,
      "learning_rate": 1.0808442529177925e-06,
      "log_odds_chosen": 0.5767822265625,
      "log_odds_ratio": -0.5771484375,
      "logits/chosen": -0.966992199420929,
      "logits/rejected": -0.8759765625,
      "logps/chosen": -0.585156261920929,
      "logps/rejected": -0.9271484613418579,
      "loss": 0.9477,
      "nll_loss": 0.873242199420929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.029266357421875,
      "rewards/margins": 0.01708221435546875,
      "rewards/rejected": -0.04633789137005806,
      "step": 2140
    },
    {
      "epoch": 0.15663133355188869,
      "grad_norm": 1.2979917057905808,
      "learning_rate": 1.0783277320343842e-06,
      "log_odds_chosen": 0.7793213129043579,
      "log_odds_ratio": -0.5399414300918579,
      "logits/chosen": -0.952343761920929,
      "logits/rejected": -0.9173828363418579,
      "logps/chosen": -0.5462890863418579,
      "logps/rejected": -0.980664074420929,
      "loss": 0.9199,
      "nll_loss": 0.9271484613418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02732544019818306,
      "rewards/margins": 0.02170104905962944,
      "rewards/rejected": -0.04903564602136612,
      "step": 2150
    },
    {
      "epoch": 0.15735985138236258,
      "grad_norm": 1.609614572291044,
      "learning_rate": 1.075828707279838e-06,
      "log_odds_chosen": 0.5834106206893921,
      "log_odds_ratio": -0.5975586175918579,
      "logits/chosen": -0.961718738079071,
      "logits/rejected": -0.8798828125,
      "logps/chosen": -0.61572265625,
      "logps/rejected": -0.985546886920929,
      "loss": 0.9341,
      "nll_loss": 0.9205077886581421,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.03078613243997097,
      "rewards/margins": 0.01851501502096653,
      "rewards/rejected": -0.04931640625,
      "step": 2160
    },
    {
      "epoch": 0.15808836921283648,
      "grad_norm": 1.3220596523780337,
      "learning_rate": 1.0733469768527298e-06,
      "log_odds_chosen": 0.5291992425918579,
      "log_odds_ratio": -0.6136718988418579,
      "logits/chosen": -0.9652343988418579,
      "logits/rejected": -0.863476574420929,
      "logps/chosen": -0.6241210699081421,
      "logps/rejected": -0.9365234375,
      "loss": 0.9126,
      "nll_loss": 0.985156238079071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03120117262005806,
      "rewards/margins": 0.015605544671416283,
      "rewards/rejected": -0.04682006686925888,
      "step": 2170
    },
    {
      "epoch": 0.15881688704331037,
      "grad_norm": 2.737431532242556,
      "learning_rate": 1.0708823421952984e-06,
      "log_odds_chosen": 0.789355456829071,
      "log_odds_ratio": -0.5165039300918579,
      "logits/chosen": -0.9583984613418579,
      "logits/rejected": -0.880078136920929,
      "logps/chosen": -0.5708984136581421,
      "logps/rejected": -1.0261719226837158,
      "loss": 0.9564,
      "nll_loss": 0.9263671636581421,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02855834923684597,
      "rewards/margins": 0.022779082879424095,
      "rewards/rejected": -0.05131835862994194,
      "step": 2180
    },
    {
      "epoch": 0.1595454048737843,
      "grad_norm": 1.4128314598833083,
      "learning_rate": 1.0684346079267208e-06,
      "log_odds_chosen": 0.738720715045929,
      "log_odds_ratio": -0.540771484375,
      "logits/chosen": -0.9789062738418579,
      "logits/rejected": -0.896484375,
      "logps/chosen": -0.6084960699081421,
      "logps/rejected": -1.0730469226837158,
      "loss": 0.9311,
      "nll_loss": 0.860156238079071,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.030426025390625,
      "rewards/margins": 0.023181915283203125,
      "rewards/rejected": -0.05364990234375,
      "step": 2190
    },
    {
      "epoch": 0.1602739227042582,
      "grad_norm": 1.41933225407361,
      "learning_rate": 1.066003581778052e-06,
      "log_odds_chosen": 0.6357421875,
      "log_odds_ratio": -0.5640624761581421,
      "logits/chosen": -0.931640625,
      "logits/rejected": -0.84375,
      "logps/chosen": -0.59375,
      "logps/rejected": -0.971875011920929,
      "loss": 0.9278,
      "nll_loss": 0.9701172113418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02971801720559597,
      "rewards/margins": 0.01886901818215847,
      "rewards/rejected": -0.048583984375,
      "step": 2200
    },
    {
      "epoch": 0.1610024405347321,
      "grad_norm": 1.3528059687543945,
      "learning_rate": 1.0635890745287928e-06,
      "log_odds_chosen": 0.581787109375,
      "log_odds_ratio": -0.561718761920929,
      "logits/chosen": -0.9681640863418579,
      "logits/rejected": -0.883593738079071,
      "logps/chosen": -0.574023425579071,
      "logps/rejected": -0.90673828125,
      "loss": 0.9183,
      "nll_loss": 0.91015625,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02870483323931694,
      "rewards/margins": 0.016632843762636185,
      "rewards/rejected": -0.045379638671875,
      "step": 2210
    },
    {
      "epoch": 0.16173095836520598,
      "grad_norm": 1.3110003149621545,
      "learning_rate": 1.0611908999450224e-06,
      "log_odds_chosen": 0.5518554449081421,
      "log_odds_ratio": -0.5751953125,
      "logits/chosen": -0.8929687738418579,
      "logits/rejected": -0.796875,
      "logps/chosen": -0.5667968988418579,
      "logps/rejected": -0.856738269329071,
      "loss": 0.9543,
      "nll_loss": 0.960742175579071,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.02835693396627903,
      "rewards/margins": 0.014482880011200905,
      "rewards/rejected": -0.04284057766199112,
      "step": 2220
    },
    {
      "epoch": 0.16245947619567988,
      "grad_norm": 1.3380503798579797,
      "learning_rate": 1.058808874719067e-06,
      "log_odds_chosen": 0.6761230230331421,
      "log_odds_ratio": -0.565136730670929,
      "logits/chosen": -0.943359375,
      "logits/rejected": -0.8753906488418579,
      "logps/chosen": -0.5801757574081421,
      "logps/rejected": -0.9966796636581421,
      "loss": 0.9407,
      "nll_loss": 0.9072265625,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02904663048684597,
      "rewards/margins": 0.02077484130859375,
      "rewards/rejected": -0.04981078952550888,
      "step": 2230
    },
    {
      "epoch": 0.16318799402615378,
      "grad_norm": 1.3663932879454408,
      "learning_rate": 1.0564428184106459e-06,
      "log_odds_chosen": 1.0329468250274658,
      "log_odds_ratio": -0.4588378965854645,
      "logits/chosen": -0.9945312738418579,
      "logits/rejected": -0.8935546875,
      "logps/chosen": -0.5478515625,
      "logps/rejected": -1.149023413658142,
      "loss": 0.9323,
      "nll_loss": 0.873242199420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02739257737994194,
      "rewards/margins": 0.03003540076315403,
      "rewards/rejected": -0.05747070163488388,
      "step": 2240
    },
    {
      "epoch": 0.1639165118566277,
      "grad_norm": 1.2837389397497168,
      "learning_rate": 1.0540925533894598e-06,
      "log_odds_chosen": 0.6560424566268921,
      "log_odds_ratio": -0.552929699420929,
      "logits/chosen": -0.94921875,
      "logits/rejected": -0.8642578125,
      "logps/chosen": -0.594921886920929,
      "logps/rejected": -0.992968738079071,
      "loss": 0.9308,
      "nll_loss": 0.9271484613418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.029754638671875,
      "rewards/margins": 0.01989135704934597,
      "rewards/rejected": -0.04961548000574112,
      "step": 2250
    },
    {
      "epoch": 0.1646450296871016,
      "grad_norm": 1.3782789025745,
      "learning_rate": 1.0517579047791782e-06,
      "log_odds_chosen": 0.8399902582168579,
      "log_odds_ratio": -0.47797852754592896,
      "logits/chosen": -0.9712890386581421,
      "logits/rejected": -0.9017578363418579,
      "logps/chosen": -0.585253894329071,
      "logps/rejected": -1.0808594226837158,
      "loss": 0.946,
      "nll_loss": 0.987109363079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02925414964556694,
      "rewards/margins": 0.02478790283203125,
      "rewards/rejected": -0.0540771484375,
      "step": 2260
    },
    {
      "epoch": 0.1653735475175755,
      "grad_norm": 1.408731133882941,
      "learning_rate": 1.049438700402784e-06,
      "log_odds_chosen": 0.7229980230331421,
      "log_odds_ratio": -0.563671886920929,
      "logits/chosen": -0.949999988079071,
      "logits/rejected": -0.854296863079071,
      "logps/chosen": -0.590136706829071,
      "logps/rejected": -1.013671875,
      "loss": 0.9127,
      "nll_loss": 0.897265613079071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.029510498046875,
      "rewards/margins": 0.02112731896340847,
      "rewards/rejected": -0.05064697191119194,
      "step": 2270
    },
    {
      "epoch": 0.1661020653480494,
      "grad_norm": 1.4214137228684858,
      "learning_rate": 1.0471347707292389e-06,
      "log_odds_chosen": 0.7998291254043579,
      "log_odds_ratio": -0.5074218511581421,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.904101550579071,
      "logps/chosen": -0.548828125,
      "logps/rejected": -1.0041015148162842,
      "loss": 0.9246,
      "nll_loss": 0.892773449420929,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02744140662252903,
      "rewards/margins": 0.02276000939309597,
      "rewards/rejected": -0.05019531399011612,
      "step": 2280
    },
    {
      "epoch": 0.16683058317852328,
      "grad_norm": 1.5439807576207776,
      "learning_rate": 1.0448459488214322e-06,
      "log_odds_chosen": 0.8351074457168579,
      "log_odds_ratio": -0.5286620855331421,
      "logits/chosen": -0.9310547113418579,
      "logits/rejected": -0.8876953125,
      "logps/chosen": -0.61962890625,
      "logps/rejected": -1.134765625,
      "loss": 0.9477,
      "nll_loss": 0.919140636920929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.03098144568502903,
      "rewards/margins": 0.02578277513384819,
      "rewards/rejected": -0.05679931491613388,
      "step": 2290
    },
    {
      "epoch": 0.1675591010089972,
      "grad_norm": 1.4270538592079818,
      "learning_rate": 1.042572070285374e-06,
      "log_odds_chosen": 0.564404308795929,
      "log_odds_ratio": -0.5894531011581421,
      "logits/chosen": -0.9535156488418579,
      "logits/rejected": -0.8828125,
      "logps/chosen": -0.600390613079071,
      "logps/rejected": -0.9349609613418579,
      "loss": 0.9182,
      "nll_loss": 0.8785156011581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.03004150465130806,
      "rewards/margins": 0.01670532301068306,
      "rewards/rejected": -0.04672851413488388,
      "step": 2300
    },
    {
      "epoch": 0.1682876188394711,
      "grad_norm": 1.3646222940339758,
      "learning_rate": 1.0403129732205989e-06,
      "log_odds_chosen": 0.6390746831893921,
      "log_odds_ratio": -0.5411132574081421,
      "logits/chosen": -0.938281238079071,
      "logits/rejected": -0.8804687261581421,
      "logps/chosen": -0.6041015386581421,
      "logps/rejected": -0.986328125,
      "loss": 0.9288,
      "nll_loss": 0.9830077886581421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03018798865377903,
      "rewards/margins": 0.019163131713867188,
      "rewards/rejected": -0.04933471605181694,
      "step": 2310
    },
    {
      "epoch": 0.169016136669945,
      "grad_norm": 1.2456790418650379,
      "learning_rate": 1.0380684981717496e-06,
      "log_odds_chosen": 0.64892578125,
      "log_odds_ratio": -0.551953136920929,
      "logits/chosen": -0.9525390863418579,
      "logits/rejected": -0.917773425579071,
      "logps/chosen": -0.6026366949081421,
      "logps/rejected": -0.9720703363418579,
      "loss": 0.9337,
      "nll_loss": 0.939648449420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.030120849609375,
      "rewards/margins": 0.01848297193646431,
      "rewards/rejected": -0.04855956882238388,
      "step": 2320
    },
    {
      "epoch": 0.1697446545004189,
      "grad_norm": 1.433100316753579,
      "learning_rate": 1.0358384880813022e-06,
      "log_odds_chosen": 0.7541748285293579,
      "log_odds_ratio": -0.5796874761581421,
      "logits/chosen": -0.9593750238418579,
      "logits/rejected": -0.867382824420929,
      "logps/chosen": -0.6332031488418579,
      "logps/rejected": -1.111328125,
      "loss": 0.9336,
      "nll_loss": 0.907031238079071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0316619873046875,
      "rewards/margins": 0.02389984205365181,
      "rewards/rejected": -0.05559081956744194,
      "step": 2330
    },
    {
      "epoch": 0.1704731723308928,
      "grad_norm": 1.4872222900653267,
      "learning_rate": 1.033622788243404e-06,
      "log_odds_chosen": 0.6416991949081421,
      "log_odds_ratio": -0.5550781488418579,
      "logits/chosen": -0.935742199420929,
      "logits/rejected": -0.863085925579071,
      "logps/chosen": -0.5634765625,
      "logps/rejected": -0.9361327886581421,
      "loss": 0.923,
      "nll_loss": 0.913867175579071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02817993238568306,
      "rewards/margins": 0.018640900030732155,
      "rewards/rejected": -0.04682006686925888,
      "step": 2340
    },
    {
      "epoch": 0.1712016901613667,
      "grad_norm": 1.4182256547547933,
      "learning_rate": 1.0314212462587935e-06,
      "log_odds_chosen": 0.5360351800918579,
      "log_odds_ratio": -0.571582019329071,
      "logits/chosen": -0.9302734136581421,
      "logits/rejected": -0.8744140863418579,
      "logps/chosen": -0.567187488079071,
      "logps/rejected": -0.853515625,
      "loss": 0.915,
      "nll_loss": 0.889843761920929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.02836303785443306,
      "rewards/margins": 0.01428375206887722,
      "rewards/rejected": -0.04262695461511612,
      "step": 2350
    },
    {
      "epoch": 0.1719302079918406,
      "grad_norm": 1.3828020347488907,
      "learning_rate": 1.029233711990773e-06,
      "log_odds_chosen": 0.797070324420929,
      "log_odds_ratio": -0.535449206829071,
      "logits/chosen": -0.9339843988418579,
      "logits/rejected": -0.8441406488418579,
      "logps/chosen": -0.5619140863418579,
      "logps/rejected": -1.0593750476837158,
      "loss": 0.8964,
      "nll_loss": 0.854687511920929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.028076171875,
      "rewards/margins": 0.02489318884909153,
      "rewards/rejected": -0.05295410007238388,
      "step": 2360
    },
    {
      "epoch": 0.1726587258223145,
      "grad_norm": 1.575179819068865,
      "learning_rate": 1.0270600375222014e-06,
      "log_odds_chosen": 0.7242431640625,
      "log_odds_ratio": -0.5584472417831421,
      "logits/chosen": -0.936328113079071,
      "logits/rejected": -0.8482421636581421,
      "logps/chosen": -0.596386730670929,
      "logps/rejected": -1.0314452648162842,
      "loss": 0.9331,
      "nll_loss": 0.9076172113418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.029815673828125,
      "rewards/margins": 0.02179260179400444,
      "rewards/rejected": -0.05161132663488388,
      "step": 2370
    },
    {
      "epoch": 0.1733872436527884,
      "grad_norm": 1.5361587856253904,
      "learning_rate": 1.0249000771134847e-06,
      "log_odds_chosen": 0.880126953125,
      "log_odds_ratio": -0.473388671875,
      "logits/chosen": -0.927539050579071,
      "logits/rejected": -0.8408203125,
      "logps/chosen": -0.551562488079071,
      "logps/rejected": -1.0421874523162842,
      "loss": 0.9199,
      "nll_loss": 0.876757800579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02755126915872097,
      "rewards/margins": 0.02455444261431694,
      "rewards/rejected": -0.05213012546300888,
      "step": 2380
    },
    {
      "epoch": 0.1741157614832623,
      "grad_norm": 1.481738626103247,
      "learning_rate": 1.022753687161533e-06,
      "log_odds_chosen": 0.831103503704071,
      "log_odds_ratio": -0.5220702886581421,
      "logits/chosen": -0.9300781488418579,
      "logits/rejected": -0.8550781011581421,
      "logps/chosen": -0.599804699420929,
      "logps/rejected": -1.104882836341858,
      "loss": 0.9453,
      "nll_loss": 0.9222656488418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.02998657152056694,
      "rewards/margins": 0.02525482140481472,
      "rewards/rejected": -0.05524902418255806,
      "step": 2390
    },
    {
      "epoch": 0.1748442793137362,
      "grad_norm": 1.3775884508891134,
      "learning_rate": 1.0206207261596577e-06,
      "log_odds_chosen": 0.7951415777206421,
      "log_odds_ratio": -0.541308581829071,
      "logits/chosen": -0.9537109136581421,
      "logits/rejected": -0.844531238079071,
      "logps/chosen": -0.5850585699081421,
      "logps/rejected": -1.0544922351837158,
      "loss": 0.9192,
      "nll_loss": 0.848437488079071,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02922363206744194,
      "rewards/margins": 0.023496245965361595,
      "rewards/rejected": -0.05271606519818306,
      "step": 2400
    },
    {
      "epoch": 0.1755727971442101,
      "grad_norm": 1.506639950025661,
      "learning_rate": 1.0185010546583882e-06,
      "log_odds_chosen": 0.7869507074356079,
      "log_odds_ratio": -0.54345703125,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.913281261920929,
      "logps/chosen": -0.5928710699081421,
      "logps/rejected": -1.081445336341858,
      "loss": 0.9145,
      "nll_loss": 0.891406238079071,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.02963867224752903,
      "rewards/margins": 0.02444763109087944,
      "rewards/rejected": -0.05405273288488388,
      "step": 2410
    },
    {
      "epoch": 0.17630131497468401,
      "grad_norm": 1.5170077942434272,
      "learning_rate": 1.0163945352271773e-06,
      "log_odds_chosen": 0.763354480266571,
      "log_odds_ratio": -0.5152343511581421,
      "logits/chosen": -0.937695324420929,
      "logits/rejected": -0.8482421636581421,
      "logps/chosen": -0.563281238079071,
      "logps/rejected": -1.023046851158142,
      "loss": 0.9242,
      "nll_loss": 0.8863281011581421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02815551683306694,
      "rewards/margins": 0.023018646985292435,
      "rewards/rejected": -0.05117187649011612,
      "step": 2420
    },
    {
      "epoch": 0.1770298328051579,
      "grad_norm": 1.5287753958150976,
      "learning_rate": 1.0143010324169743e-06,
      "log_odds_chosen": 0.803332507610321,
      "log_odds_ratio": -0.5291992425918579,
      "logits/chosen": -0.9267578125,
      "logits/rejected": -0.850781261920929,
      "logps/chosen": -0.5615234375,
      "logps/rejected": -1.055761694908142,
      "loss": 0.9332,
      "nll_loss": 0.894335925579071,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.02810058556497097,
      "rewards/margins": 0.024681473150849342,
      "rewards/rejected": -0.05275879055261612,
      "step": 2430
    },
    {
      "epoch": 0.1777583506356318,
      "grad_norm": 1.405164592437613,
      "learning_rate": 1.0122204127236452e-06,
      "log_odds_chosen": 0.6588379144668579,
      "log_odds_ratio": -0.5733886957168579,
      "logits/chosen": -0.9517577886581421,
      "logits/rejected": -0.871874988079071,
      "logps/chosen": -0.58203125,
      "logps/rejected": -0.9642578363418579,
      "loss": 0.943,
      "nll_loss": 0.9175781011581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.029083251953125,
      "rewards/margins": 0.019109344109892845,
      "rewards/rejected": -0.04822998121380806,
      "step": 2440
    },
    {
      "epoch": 0.1784868684661057,
      "grad_norm": 1.7385119294915847,
      "learning_rate": 1.0101525445522107e-06,
      "log_odds_chosen": 0.8550781011581421,
      "log_odds_ratio": -0.508837878704071,
      "logits/chosen": -0.9605468511581421,
      "logits/rejected": -0.8837890625,
      "logps/chosen": -0.540820300579071,
      "logps/rejected": -1.0246093273162842,
      "loss": 0.9032,
      "nll_loss": 0.8384765386581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02703247033059597,
      "rewards/margins": 0.0241546630859375,
      "rewards/rejected": -0.05122070387005806,
      "step": 2450
    },
    {
      "epoch": 0.1792153862965796,
      "grad_norm": 1.4500580041882032,
      "learning_rate": 1.0080972981818898e-06,
      "log_odds_chosen": 0.695605456829071,
      "log_odds_ratio": -0.546191394329071,
      "logits/chosen": -0.9253906011581421,
      "logits/rejected": -0.842578113079071,
      "logps/chosen": -0.5858398675918579,
      "logps/rejected": -0.9847656488418579,
      "loss": 0.9488,
      "nll_loss": 0.912890613079071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02929077111184597,
      "rewards/margins": 0.01999206468462944,
      "rewards/rejected": -0.04930419847369194,
      "step": 2460
    },
    {
      "epoch": 0.17994390412705352,
      "grad_norm": 1.3282158741463947,
      "learning_rate": 1.0060545457319173e-06,
      "log_odds_chosen": 0.7539306879043579,
      "log_odds_ratio": -0.5611327886581421,
      "logits/chosen": -0.9486328363418579,
      "logits/rejected": -0.8218749761581421,
      "logps/chosen": -0.5960937738418579,
      "logps/rejected": -1.035546898841858,
      "loss": 0.9301,
      "nll_loss": 0.9154297113418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.02980956993997097,
      "rewards/margins": 0.02191925048828125,
      "rewards/rejected": -0.051727294921875,
      "step": 2470
    },
    {
      "epoch": 0.18067242195752742,
      "grad_norm": 1.4442953179379556,
      "learning_rate": 1.0040241611281238e-06,
      "log_odds_chosen": 0.819775402545929,
      "log_odds_ratio": -0.49101561307907104,
      "logits/chosen": -0.9306640625,
      "logits/rejected": -0.8314453363418579,
      "logps/chosen": -0.532421886920929,
      "logps/rejected": -0.973828136920929,
      "loss": 0.8933,
      "nll_loss": 0.8958984613418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02660522423684597,
      "rewards/margins": 0.02207488939166069,
      "rewards/rejected": -0.04862060397863388,
      "step": 2480
    },
    {
      "epoch": 0.18140093978800131,
      "grad_norm": 1.7175897312387,
      "learning_rate": 1.002006020070253e-06,
      "log_odds_chosen": 0.931689441204071,
      "log_odds_ratio": -0.4927734434604645,
      "logits/chosen": -0.968945324420929,
      "logits/rejected": -0.8705078363418579,
      "logps/chosen": -0.5643554925918579,
      "logps/rejected": -1.1193358898162842,
      "loss": 0.9291,
      "nll_loss": 0.8548828363418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02817993238568306,
      "rewards/margins": 0.027750205248594284,
      "rewards/rejected": -0.05595092847943306,
      "step": 2490
    },
    {
      "epoch": 0.1821294576184752,
      "grad_norm": 1.55979321358704,
      "learning_rate": 1.0000000000000002e-06,
      "log_odds_chosen": 0.8021606206893921,
      "log_odds_ratio": -0.528759777545929,
      "logits/chosen": -0.9390624761581421,
      "logits/rejected": -0.873828113079071,
      "logps/chosen": -0.5679687261581421,
      "logps/rejected": -1.00390625,
      "loss": 0.9038,
      "nll_loss": 0.881054699420929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02839965745806694,
      "rewards/margins": 0.02180328406393528,
      "rewards/rejected": -0.05023193359375,
      "step": 2500
    },
    {
      "epoch": 0.1828579754489491,
      "grad_norm": 1.5470719444387402,
      "learning_rate": 9.98005980069749e-07,
      "log_odds_chosen": 0.850170910358429,
      "log_odds_ratio": -0.5224609375,
      "logits/chosen": -0.927539050579071,
      "logits/rejected": -0.843554675579071,
      "logps/chosen": -0.595996081829071,
      "logps/rejected": -1.127539038658142,
      "loss": 0.9385,
      "nll_loss": 0.9369140863418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02980346605181694,
      "rewards/margins": 0.026580810546875,
      "rewards/rejected": -0.05638427659869194,
      "step": 2510
    },
    {
      "epoch": 0.183586493279423,
      "grad_norm": 1.744411201785703,
      "learning_rate": 9.960238411119948e-07,
      "log_odds_chosen": 0.9224609136581421,
      "log_odds_ratio": -0.49951171875,
      "logits/chosen": -0.954882800579071,
      "logits/rejected": -0.858593761920929,
      "logps/chosen": -0.5712890625,
      "logps/rejected": -1.1240234375,
      "loss": 0.9197,
      "nll_loss": 0.892578125,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02854614332318306,
      "rewards/margins": 0.02766571007668972,
      "rewards/rejected": -0.05622558668255806,
      "step": 2520
    },
    {
      "epoch": 0.18431501110989693,
      "grad_norm": 1.3625318256252719,
      "learning_rate": 9.9405346560943e-07,
      "log_odds_chosen": 0.85595703125,
      "log_odds_ratio": -0.4984374940395355,
      "logits/chosen": -0.9791015386581421,
      "logits/rejected": -0.883007824420929,
      "logps/chosen": -0.5860351324081421,
      "logps/rejected": -1.113671898841858,
      "loss": 0.9008,
      "nll_loss": 0.866015613079071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02929077111184597,
      "rewards/margins": 0.02637786790728569,
      "rewards/rejected": -0.05562744289636612,
      "step": 2530
    },
    {
      "epoch": 0.18504352894037082,
      "grad_norm": 1.9778913623311911,
      "learning_rate": 9.920947376656814e-07,
      "log_odds_chosen": 0.825439453125,
      "log_odds_ratio": -0.502734363079071,
      "logits/chosen": -0.990429699420929,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.5582031011581421,
      "logps/rejected": -1.0314452648162842,
      "loss": 0.9388,
      "nll_loss": 0.8941406011581421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02792968787252903,
      "rewards/margins": 0.0236358642578125,
      "rewards/rejected": -0.05156249925494194,
      "step": 2540
    },
    {
      "epoch": 0.18577204677084472,
      "grad_norm": 1.3691741742209838,
      "learning_rate": 9.901475429766744e-07,
      "log_odds_chosen": 0.759570300579071,
      "log_odds_ratio": -0.526562511920929,
      "logits/chosen": -1.019140601158142,
      "logits/rejected": -0.9095703363418579,
      "logps/chosen": -0.602832019329071,
      "logps/rejected": -1.0402343273162842,
      "loss": 0.9308,
      "nll_loss": 0.9193359613418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.03012695349752903,
      "rewards/margins": 0.0218505859375,
      "rewards/rejected": -0.05198974534869194,
      "step": 2550
    },
    {
      "epoch": 0.1865005646013186,
      "grad_norm": 1.697703694704515,
      "learning_rate": 9.882117688026186e-07,
      "log_odds_chosen": 0.9254394769668579,
      "log_odds_ratio": -0.48652344942092896,
      "logits/chosen": -0.9789062738418579,
      "logits/rejected": -0.869921863079071,
      "logps/chosen": -0.530566394329071,
      "logps/rejected": -1.0654296875,
      "loss": 0.9276,
      "nll_loss": 0.916015625,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02653198316693306,
      "rewards/margins": 0.02675170823931694,
      "rewards/rejected": -0.05324707180261612,
      "step": 2560
    },
    {
      "epoch": 0.1872290824317925,
      "grad_norm": 1.4945725271775336,
      "learning_rate": 9.862873039405896e-07,
      "log_odds_chosen": 0.619006335735321,
      "log_odds_ratio": -0.5694335699081421,
      "logits/chosen": -0.9574218988418579,
      "logits/rejected": -0.8833984136581421,
      "logps/chosen": -0.5804687738418579,
      "logps/rejected": -0.935546875,
      "loss": 0.94,
      "nll_loss": 0.9310547113418579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.029022216796875,
      "rewards/margins": 0.017742156982421875,
      "rewards/rejected": -0.04676513746380806,
      "step": 2570
    },
    {
      "epoch": 0.1879576002622664,
      "grad_norm": 1.4070218967643204,
      "learning_rate": 9.843740386976973e-07,
      "log_odds_chosen": 0.888476550579071,
      "log_odds_ratio": -0.4854492247104645,
      "logits/chosen": -0.948437511920929,
      "logits/rejected": -0.842578113079071,
      "logps/chosen": -0.569140613079071,
      "logps/rejected": -1.098242163658142,
      "loss": 0.9237,
      "nll_loss": 0.9095703363418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02843627892434597,
      "rewards/margins": 0.02643737755715847,
      "rewards/rejected": -0.05490722507238388,
      "step": 2580
    },
    {
      "epoch": 0.18868611809274033,
      "grad_norm": 1.446548768035602,
      "learning_rate": 9.824718648648244e-07,
      "log_odds_chosen": 0.978320300579071,
      "log_odds_ratio": -0.46342772245407104,
      "logits/chosen": -0.933789074420929,
      "logits/rejected": -0.847460925579071,
      "logps/chosen": -0.547070324420929,
      "logps/rejected": -1.089453101158142,
      "loss": 0.9123,
      "nll_loss": 0.8568359613418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02736816368997097,
      "rewards/margins": 0.02710266038775444,
      "rewards/rejected": -0.05451660230755806,
      "step": 2590
    },
    {
      "epoch": 0.18941463592321423,
      "grad_norm": 1.5535425679820252,
      "learning_rate": 9.805806756909204e-07,
      "log_odds_chosen": 0.9105224609375,
      "log_odds_ratio": -0.489990234375,
      "logits/chosen": -0.966601550579071,
      "logits/rejected": -0.897265613079071,
      "logps/chosen": -0.562792956829071,
      "logps/rejected": -1.0632812976837158,
      "loss": 0.9276,
      "nll_loss": 0.870898425579071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02813720703125,
      "rewards/margins": 0.02502136304974556,
      "rewards/rejected": -0.05316162109375,
      "step": 2600
    },
    {
      "epoch": 0.19014315375368812,
      "grad_norm": 1.5030145762470308,
      "learning_rate": 9.787003658578392e-07,
      "log_odds_chosen": 0.876953125,
      "log_odds_ratio": -0.4930175840854645,
      "logits/chosen": -0.982226550579071,
      "logits/rejected": -0.900390625,
      "logps/chosen": -0.5804687738418579,
      "logps/rejected": -1.1218750476837158,
      "loss": 0.9199,
      "nll_loss": 0.8720703125,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02902832068502903,
      "rewards/margins": 0.02703094482421875,
      "rewards/rejected": -0.05607910081744194,
      "step": 2610
    },
    {
      "epoch": 0.19087167158416202,
      "grad_norm": 1.5317271354103796,
      "learning_rate": 9.768308314557044e-07,
      "log_odds_chosen": 0.696093738079071,
      "log_odds_ratio": -0.550000011920929,
      "logits/chosen": -0.9681640863418579,
      "logits/rejected": -0.865039050579071,
      "logps/chosen": -0.5762695074081421,
      "logps/rejected": -0.9693359136581421,
      "loss": 0.9007,
      "nll_loss": 0.877148449420929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02880859375,
      "rewards/margins": 0.01968231238424778,
      "rewards/rejected": -0.0484619140625,
      "step": 2620
    },
    {
      "epoch": 0.1916001894146359,
      "grad_norm": 1.3918164124435848,
      "learning_rate": 9.749719699587899e-07,
      "log_odds_chosen": 0.6759277582168579,
      "log_odds_ratio": -0.5650879144668579,
      "logits/chosen": -0.9140625,
      "logits/rejected": -0.857421875,
      "logps/chosen": -0.591601550579071,
      "logps/rejected": -0.986523449420929,
      "loss": 0.9133,
      "nll_loss": 0.913867175579071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02955932542681694,
      "rewards/margins": 0.019737815484404564,
      "rewards/rejected": -0.04932861402630806,
      "step": 2630
    },
    {
      "epoch": 0.19232870724510984,
      "grad_norm": 1.3312972437311297,
      "learning_rate": 9.731236802019038e-07,
      "log_odds_chosen": 0.753430187702179,
      "log_odds_ratio": -0.53955078125,
      "logits/chosen": -0.9388672113418579,
      "logits/rejected": -0.903124988079071,
      "logps/chosen": -0.568359375,
      "logps/rejected": -1.0265624523162842,
      "loss": 0.8873,
      "nll_loss": 0.851757824420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02842407301068306,
      "rewards/margins": 0.02291259728372097,
      "rewards/rejected": -0.05129394680261612,
      "step": 2640
    },
    {
      "epoch": 0.19305722507558373,
      "grad_norm": 1.4395476174174195,
      "learning_rate": 9.712858623572642e-07,
      "log_odds_chosen": 0.6625000238418579,
      "log_odds_ratio": -0.5691894292831421,
      "logits/chosen": -0.9507812261581421,
      "logits/rejected": -0.863085925579071,
      "logps/chosen": -0.591992199420929,
      "logps/rejected": -1.0166015625,
      "loss": 0.9135,
      "nll_loss": 0.87890625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02958984300494194,
      "rewards/margins": 0.02124938927590847,
      "rewards/rejected": -0.05084838718175888,
      "step": 2650
    },
    {
      "epoch": 0.19378574290605763,
      "grad_norm": 1.333084127878223,
      "learning_rate": 9.694584179118515e-07,
      "log_odds_chosen": 0.586010754108429,
      "log_odds_ratio": -0.5689452886581421,
      "logits/chosen": -0.922656238079071,
      "logits/rejected": -0.845898449420929,
      "logps/chosen": -0.6011718511581421,
      "logps/rejected": -0.9339843988418579,
      "loss": 0.9173,
      "nll_loss": 0.965039074420929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03005371056497097,
      "rewards/margins": 0.01663970947265625,
      "rewards/rejected": -0.04669189453125,
      "step": 2660
    },
    {
      "epoch": 0.19451426073653152,
      "grad_norm": 1.5228973739500893,
      "learning_rate": 9.676412496452296e-07,
      "log_odds_chosen": 0.6689453125,
      "log_odds_ratio": -0.57666015625,
      "logits/chosen": -0.9701172113418579,
      "logits/rejected": -0.8617187738418579,
      "logps/chosen": -0.642285168170929,
      "logps/rejected": -1.0546875,
      "loss": 0.9153,
      "nll_loss": 0.9052734375,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.032135009765625,
      "rewards/margins": 0.02063598670065403,
      "rewards/rejected": -0.052734375,
      "step": 2670
    },
    {
      "epoch": 0.19524277856700542,
      "grad_norm": 1.3232016236463282,
      "learning_rate": 9.658342616078198e-07,
      "log_odds_chosen": 0.715869128704071,
      "log_odds_ratio": -0.5782715082168579,
      "logits/chosen": -0.954882800579071,
      "logits/rejected": -0.8394531011581421,
      "logps/chosen": -0.5960937738418579,
      "logps/rejected": -1.060546875,
      "loss": 0.8872,
      "nll_loss": 0.907031238079071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02979736402630806,
      "rewards/margins": 0.02321319654583931,
      "rewards/rejected": -0.05303955078125,
      "step": 2680
    },
    {
      "epoch": 0.19597129639747932,
      "grad_norm": 1.3943594501252696,
      "learning_rate": 9.640373590996239e-07,
      "log_odds_chosen": 1.007714867591858,
      "log_odds_ratio": -0.4468750059604645,
      "logits/chosen": -0.9662109613418579,
      "logits/rejected": -0.8720703125,
      "logps/chosen": -0.5367187261581421,
      "logps/rejected": -1.138281226158142,
      "loss": 0.9279,
      "nll_loss": 0.866992175579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02681884728372097,
      "rewards/margins": 0.03012695349752903,
      "rewards/rejected": -0.05694580078125,
      "step": 2690
    },
    {
      "epoch": 0.19669981422795324,
      "grad_norm": 1.832570364338214,
      "learning_rate": 9.622504486493764e-07,
      "log_odds_chosen": 0.7696288824081421,
      "log_odds_ratio": -0.5298827886581421,
      "logits/chosen": -0.962890625,
      "logits/rejected": -0.9013671875,
      "logps/chosen": -0.586132824420929,
      "logps/rejected": -1.0359375476837158,
      "loss": 0.9179,
      "nll_loss": 0.923828125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02930297888815403,
      "rewards/margins": 0.02248230017721653,
      "rewards/rejected": -0.051788330078125,
      "step": 2700
    },
    {
      "epoch": 0.19742833205842714,
      "grad_norm": 1.6108389359005961,
      "learning_rate": 9.604734379941232e-07,
      "log_odds_chosen": 0.81854248046875,
      "log_odds_ratio": -0.49824219942092896,
      "logits/chosen": -0.962695300579071,
      "logits/rejected": -0.8890625238418579,
      "logps/chosen": -0.569042980670929,
      "logps/rejected": -1.022851586341858,
      "loss": 0.9039,
      "nll_loss": 0.885937511920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02843933179974556,
      "rewards/margins": 0.022705078125,
      "rewards/rejected": -0.05113525316119194,
      "step": 2710
    },
    {
      "epoch": 0.19815684988890103,
      "grad_norm": 1.3818585279308315,
      "learning_rate": 9.58706236059213e-07,
      "log_odds_chosen": 0.780322253704071,
      "log_odds_ratio": -0.527148425579071,
      "logits/chosen": -0.9722656011581421,
      "logits/rejected": -0.84765625,
      "logps/chosen": -0.6005859375,
      "logps/rejected": -1.0701172351837158,
      "loss": 0.9178,
      "nll_loss": 0.883593738079071,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.03001708909869194,
      "rewards/margins": 0.02352294884622097,
      "rewards/rejected": -0.05354003980755806,
      "step": 2720
    },
    {
      "epoch": 0.19888536771937493,
      "grad_norm": 1.3944610032297158,
      "learning_rate": 9.56948752938691e-07,
      "log_odds_chosen": 0.509960949420929,
      "log_odds_ratio": -0.6094726324081421,
      "logits/chosen": -0.926953136920929,
      "logits/rejected": -0.8658202886581421,
      "logps/chosen": -0.5888671875,
      "logps/rejected": -0.87890625,
      "loss": 0.9013,
      "nll_loss": 0.897167980670929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.02943725511431694,
      "rewards/margins": 0.014508438296616077,
      "rewards/rejected": -0.043914794921875,
      "step": 2730
    },
    {
      "epoch": 0.19961388554984882,
      "grad_norm": 1.3705668971023235,
      "learning_rate": 9.552008998760876e-07,
      "log_odds_chosen": 0.906054675579071,
      "log_odds_ratio": -0.48828125,
      "logits/chosen": -0.9527343511581421,
      "logits/rejected": -0.8980468511581421,
      "logps/chosen": -0.5201171636581421,
      "logps/rejected": -1.032812476158142,
      "loss": 0.9059,
      "nll_loss": 0.8285156488418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02598876878619194,
      "rewards/margins": 0.02564697340130806,
      "rewards/rejected": -0.05162353441119194,
      "step": 2740
    },
    {
      "epoch": 0.20034240338032272,
      "grad_norm": 1.5743111169886062,
      "learning_rate": 9.534625892455924e-07,
      "log_odds_chosen": 0.7911132574081421,
      "log_odds_ratio": -0.5125976800918579,
      "logits/chosen": -0.9242187738418579,
      "logits/rejected": -0.863476574420929,
      "logps/chosen": -0.6029297113418579,
      "logps/rejected": -1.070898413658142,
      "loss": 0.9071,
      "nll_loss": 0.893750011920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.03014526329934597,
      "rewards/margins": 0.023410797119140625,
      "rewards/rejected": -0.05357665941119194,
      "step": 2750
    },
    {
      "epoch": 0.20107092121079664,
      "grad_norm": 1.5548354365426962,
      "learning_rate": 9.517337345336012e-07,
      "log_odds_chosen": 0.781115710735321,
      "log_odds_ratio": -0.565722644329071,
      "logits/chosen": -0.8910156488418579,
      "logits/rejected": -0.841601550579071,
      "logps/chosen": -0.547070324420929,
      "logps/rejected": -0.9775390625,
      "loss": 0.9253,
      "nll_loss": 0.9126952886581421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02735595777630806,
      "rewards/margins": 0.02154388464987278,
      "rewards/rejected": -0.04886474460363388,
      "step": 2760
    },
    {
      "epoch": 0.20179943904127054,
      "grad_norm": 1.5289144754390942,
      "learning_rate": 9.50014250320633e-07,
      "log_odds_chosen": 0.944042980670929,
      "log_odds_ratio": -0.4794921875,
      "logits/chosen": -0.962890625,
      "logits/rejected": -0.88671875,
      "logps/chosen": -0.5791015625,
      "logps/rejected": -1.1335937976837158,
      "loss": 0.8942,
      "nll_loss": 0.903124988079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0289306640625,
      "rewards/margins": 0.0277099609375,
      "rewards/rejected": -0.05662841722369194,
      "step": 2770
    },
    {
      "epoch": 0.20252795687174444,
      "grad_norm": 1.404199109758169,
      "learning_rate": 9.483040522636021e-07,
      "log_odds_chosen": 0.834057629108429,
      "log_odds_ratio": -0.4957031309604645,
      "logits/chosen": -0.970507800579071,
      "logits/rejected": -0.868945300579071,
      "logps/chosen": -0.6142578125,
      "logps/rejected": -1.124609351158142,
      "loss": 0.9219,
      "nll_loss": 0.8603515625,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.03070678748190403,
      "rewards/margins": 0.025556182488799095,
      "rewards/rejected": -0.05624999850988388,
      "step": 2780
    },
    {
      "epoch": 0.20325647470221833,
      "grad_norm": 1.3759458299136738,
      "learning_rate": 9.466030570784414e-07,
      "log_odds_chosen": 0.864306628704071,
      "log_odds_ratio": -0.49970704317092896,
      "logits/chosen": -0.91796875,
      "logits/rejected": -0.848828136920929,
      "logps/chosen": -0.5933593511581421,
      "logps/rejected": -1.1173827648162842,
      "loss": 0.9142,
      "nll_loss": 0.8955078125,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02967529371380806,
      "rewards/margins": 0.026209259405732155,
      "rewards/rejected": -0.05584716796875,
      "step": 2790
    },
    {
      "epoch": 0.20398499253269223,
      "grad_norm": 1.4559530354843078,
      "learning_rate": 9.449111825230681e-07,
      "log_odds_chosen": 0.668774425983429,
      "log_odds_ratio": -0.572558581829071,
      "logits/chosen": -0.934765636920929,
      "logits/rejected": -0.8443359136581421,
      "logps/chosen": -0.576367199420929,
      "logps/rejected": -0.978515625,
      "loss": 0.9163,
      "nll_loss": 0.853515625,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.02880859375,
      "rewards/margins": 0.02005920372903347,
      "rewards/rejected": -0.04893188551068306,
      "step": 2800
    },
    {
      "epoch": 0.20471351036316615,
      "grad_norm": 1.4802209619015467,
      "learning_rate": 9.432283473806812e-07,
      "log_odds_chosen": 0.6825195550918579,
      "log_odds_ratio": -0.51806640625,
      "logits/chosen": -0.9332031011581421,
      "logits/rejected": -0.859375,
      "logps/chosen": -0.577929675579071,
      "logps/rejected": -0.973437488079071,
      "loss": 0.9079,
      "nll_loss": 0.844921886920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02886352501809597,
      "rewards/margins": 0.01979827880859375,
      "rewards/rejected": -0.04866943508386612,
      "step": 2810
    },
    {
      "epoch": 0.20544202819364005,
      "grad_norm": 1.3924124517117695,
      "learning_rate": 9.415544714433869e-07,
      "log_odds_chosen": 0.7648559808731079,
      "log_odds_ratio": -0.5335937738418579,
      "logits/chosen": -0.941601574420929,
      "logits/rejected": -0.8521484136581421,
      "logps/chosen": -0.60107421875,
      "logps/rejected": -1.065039038658142,
      "loss": 0.9108,
      "nll_loss": 0.864062488079071,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03007202222943306,
      "rewards/margins": 0.02317199669778347,
      "rewards/rejected": -0.05319824069738388,
      "step": 2820
    },
    {
      "epoch": 0.20617054602411394,
      "grad_norm": 1.4523339550853125,
      "learning_rate": 9.398894754961406e-07,
      "log_odds_chosen": 0.7548828125,
      "log_odds_ratio": -0.541210949420929,
      "logits/chosen": -0.940625011920929,
      "logits/rejected": -0.875,
      "logps/chosen": -0.60400390625,
      "logps/rejected": -1.0642578601837158,
      "loss": 0.9188,
      "nll_loss": 0.9320312738418579,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.03018798865377903,
      "rewards/margins": 0.02300720289349556,
      "rewards/rejected": -0.05317382887005806,
      "step": 2830
    },
    {
      "epoch": 0.20689906385458784,
      "grad_norm": 1.4017456953401064,
      "learning_rate": 9.38233281301002e-07,
      "log_odds_chosen": 0.823925793170929,
      "log_odds_ratio": -0.500781238079071,
      "logits/chosen": -0.927539050579071,
      "logits/rejected": -0.845898449420929,
      "logps/chosen": -0.564648449420929,
      "logps/rejected": -1.050195336341858,
      "loss": 0.8974,
      "nll_loss": 0.8970702886581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02821044996380806,
      "rewards/margins": 0.024295425042510033,
      "rewards/rejected": -0.05250244215130806,
      "step": 2840
    },
    {
      "epoch": 0.20762758168506174,
      "grad_norm": 1.3691719831960945,
      "learning_rate": 9.365858115816941e-07,
      "log_odds_chosen": 0.723095715045929,
      "log_odds_ratio": -0.546875,
      "logits/chosen": -0.931835949420929,
      "logits/rejected": -0.849414050579071,
      "logps/chosen": -0.576171875,
      "logps/rejected": -0.984570324420929,
      "loss": 0.9006,
      "nll_loss": 0.9066406488418579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02879028394818306,
      "rewards/margins": 0.02043609693646431,
      "rewards/rejected": -0.04926757887005806,
      "step": 2850
    },
    {
      "epoch": 0.20835609951553563,
      "grad_norm": 1.6414140964802635,
      "learning_rate": 9.349469900084572e-07,
      "log_odds_chosen": 0.714599609375,
      "log_odds_ratio": -0.566699206829071,
      "logits/chosen": -0.956835925579071,
      "logits/rejected": -0.893359363079071,
      "logps/chosen": -0.6142578125,
      "logps/rejected": -1.0400390625,
      "loss": 0.9314,
      "nll_loss": 0.977734386920929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.03072509728372097,
      "rewards/margins": 0.02126159705221653,
      "rewards/rejected": -0.052001953125,
      "step": 2860
    },
    {
      "epoch": 0.20908461734600955,
      "grad_norm": 1.4514646156585926,
      "learning_rate": 9.333167411831968e-07,
      "log_odds_chosen": 0.798632800579071,
      "log_odds_ratio": -0.5328124761581421,
      "logits/chosen": -0.898632824420929,
      "logits/rejected": -0.8128906488418579,
      "logps/chosen": -0.5782226324081421,
      "logps/rejected": -1.0478515625,
      "loss": 0.8985,
      "nll_loss": 0.911914050579071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02889404259622097,
      "rewards/margins": 0.023468017578125,
      "rewards/rejected": -0.05239257961511612,
      "step": 2870
    },
    {
      "epoch": 0.20981313517648345,
      "grad_norm": 1.4197242569446238,
      "learning_rate": 9.316949906249125e-07,
      "log_odds_chosen": 0.804150402545929,
      "log_odds_ratio": -0.51806640625,
      "logits/chosen": -0.9593750238418579,
      "logits/rejected": -0.8843749761581421,
      "logps/chosen": -0.5941406488418579,
      "logps/rejected": -1.0798828601837158,
      "loss": 0.8911,
      "nll_loss": 0.866015613079071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02973632887005806,
      "rewards/margins": 0.024299049749970436,
      "rewards/rejected": -0.05405273288488388,
      "step": 2880
    },
    {
      "epoch": 0.21054165300695735,
      "grad_norm": 1.4281139985716156,
      "learning_rate": 9.300816647554058e-07,
      "log_odds_chosen": 0.861035168170929,
      "log_odds_ratio": -0.47265625,
      "logits/chosen": -0.9623047113418579,
      "logits/rejected": -0.823437511920929,
      "logps/chosen": -0.58447265625,
      "logps/rejected": -1.1033203601837158,
      "loss": 0.9156,
      "nll_loss": 0.8638671636581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02921142615377903,
      "rewards/margins": 0.02597503736615181,
      "rewards/rejected": -0.05516357347369194,
      "step": 2890
    },
    {
      "epoch": 0.21127017083743124,
      "grad_norm": 1.5457383855030953,
      "learning_rate": 9.284766908852594e-07,
      "log_odds_chosen": 0.6578124761581421,
      "log_odds_ratio": -0.550585925579071,
      "logits/chosen": -0.9478515386581421,
      "logits/rejected": -0.8541015386581421,
      "logps/chosen": -0.600781261920929,
      "logps/rejected": -1.002343773841858,
      "loss": 0.9149,
      "nll_loss": 0.8861328363418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.030029296875,
      "rewards/margins": 0.02008667029440403,
      "rewards/rejected": -0.05008544772863388,
      "step": 2900
    },
    {
      "epoch": 0.21199868866790514,
      "grad_norm": 1.4501702533471332,
      "learning_rate": 9.26879997200081e-07,
      "log_odds_chosen": 0.992968738079071,
      "log_odds_ratio": -0.4598632752895355,
      "logits/chosen": -0.9576171636581421,
      "logits/rejected": -0.867968738079071,
      "logps/chosen": -0.5712890625,
      "logps/rejected": -1.1843750476837158,
      "loss": 0.9157,
      "nll_loss": 0.865039050579071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02855834923684597,
      "rewards/margins": 0.03065795823931694,
      "rewards/rejected": -0.05921630933880806,
      "step": 2910
    },
    {
      "epoch": 0.21272720649837903,
      "grad_norm": 1.58477681978126,
      "learning_rate": 9.252915127470066e-07,
      "log_odds_chosen": 0.799511730670929,
      "log_odds_ratio": -0.52294921875,
      "logits/chosen": -0.9296875,
      "logits/rejected": -0.8353515863418579,
      "logps/chosen": -0.58056640625,
      "logps/rejected": -1.0595703125,
      "loss": 0.9067,
      "nll_loss": 0.892382800579071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02903442457318306,
      "rewards/margins": 0.023952484130859375,
      "rewards/rejected": -0.05299682542681694,
      "step": 2920
    },
    {
      "epoch": 0.21345572432885296,
      "grad_norm": 1.4685650560542338,
      "learning_rate": 9.23711167421458e-07,
      "log_odds_chosen": 0.673095703125,
      "log_odds_ratio": -0.5782226324081421,
      "logits/chosen": -0.942578136920929,
      "logits/rejected": -0.827343761920929,
      "logps/chosen": -0.577832043170929,
      "logps/rejected": -0.9976562261581421,
      "loss": 0.9021,
      "nll_loss": 0.854687511920929,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02890625037252903,
      "rewards/margins": 0.02094421349465847,
      "rewards/rejected": -0.04986572265625,
      "step": 2930
    },
    {
      "epoch": 0.21418424215932685,
      "grad_norm": 1.4711228282574715,
      "learning_rate": 9.221388919541469e-07,
      "log_odds_chosen": 0.750415027141571,
      "log_odds_ratio": -0.5352538824081421,
      "logits/chosen": -0.944531261920929,
      "logits/rejected": -0.862500011920929,
      "logps/chosen": -0.6005859375,
      "logps/rejected": -1.068359375,
      "loss": 0.8945,
      "nll_loss": 0.838671863079071,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03001098707318306,
      "rewards/margins": 0.023381805047392845,
      "rewards/rejected": -0.05339965969324112,
      "step": 2940
    },
    {
      "epoch": 0.21491275998980075,
      "grad_norm": 1.345776672005385,
      "learning_rate": 9.205746178983235e-07,
      "log_odds_chosen": 0.648510754108429,
      "log_odds_ratio": -0.547070324420929,
      "logits/chosen": -0.8949218988418579,
      "logits/rejected": -0.830078125,
      "logps/chosen": -0.5648437738418579,
      "logps/rejected": -0.9169921875,
      "loss": 0.9221,
      "nll_loss": 0.861132800579071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02825927734375,
      "rewards/margins": 0.017594527453184128,
      "rewards/rejected": -0.04583740234375,
      "step": 2950
    },
    {
      "epoch": 0.21564127782027465,
      "grad_norm": 1.4120078677101533,
      "learning_rate": 9.190182776172598e-07,
      "log_odds_chosen": 0.5892089605331421,
      "log_odds_ratio": -0.571093738079071,
      "logits/chosen": -0.9427734613418579,
      "logits/rejected": -0.8423827886581421,
      "logps/chosen": -0.5765625238418579,
      "logps/rejected": -0.9234374761581421,
      "loss": 0.935,
      "nll_loss": 0.8453124761581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.02882080152630806,
      "rewards/margins": 0.017369842156767845,
      "rewards/rejected": -0.04616088792681694,
      "step": 2960
    },
    {
      "epoch": 0.21636979565074854,
      "grad_norm": 1.4001362638095882,
      "learning_rate": 9.174698042719672e-07,
      "log_odds_chosen": 0.977246105670929,
      "log_odds_ratio": -0.4593749940395355,
      "logits/chosen": -0.9595702886581421,
      "logits/rejected": -0.8681640625,
      "logps/chosen": -0.531445324420929,
      "logps/rejected": -1.0964844226837158,
      "loss": 0.8857,
      "nll_loss": 0.8896484375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02657470665872097,
      "rewards/margins": 0.02825317345559597,
      "rewards/rejected": -0.0548095703125,
      "step": 2970
    },
    {
      "epoch": 0.21709831348122247,
      "grad_norm": 1.4994509472052573,
      "learning_rate": 9.159291318091397e-07,
      "log_odds_chosen": 0.976757824420929,
      "log_odds_ratio": -0.47187501192092896,
      "logits/chosen": -0.938281238079071,
      "logits/rejected": -0.8193359375,
      "logps/chosen": -0.562792956829071,
      "logps/rejected": -1.1462891101837158,
      "loss": 0.9108,
      "nll_loss": 0.8824218511581421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02816162072122097,
      "rewards/margins": 0.02915344201028347,
      "rewards/rejected": -0.05734863132238388,
      "step": 2980
    },
    {
      "epoch": 0.21782683131169636,
      "grad_norm": 1.3245465243469383,
      "learning_rate": 9.143961949493189e-07,
      "log_odds_chosen": 0.7958984375,
      "log_odds_ratio": -0.541308581829071,
      "logits/chosen": -0.9439452886581421,
      "logits/rejected": -0.881640613079071,
      "logps/chosen": -0.6170898675918579,
      "logps/rejected": -1.1042969226837158,
      "loss": 0.8808,
      "nll_loss": 0.879687488079071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03087768517434597,
      "rewards/margins": 0.0243682861328125,
      "rewards/rejected": -0.05522460862994194,
      "step": 2990
    },
    {
      "epoch": 0.21855534914217026,
      "grad_norm": 1.332184451105708,
      "learning_rate": 9.128709291752768e-07,
      "log_odds_chosen": 0.982531726360321,
      "log_odds_ratio": -0.48652344942092896,
      "logits/chosen": -0.955078125,
      "logits/rejected": -0.8570312261581421,
      "logps/chosen": -0.58349609375,
      "logps/rejected": -1.1980469226837158,
      "loss": 0.8731,
      "nll_loss": 0.804882824420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02915039099752903,
      "rewards/margins": 0.030735015869140625,
      "rewards/rejected": -0.05991210788488388,
      "step": 3000
    },
    {
      "epoch": 0.21928386697264415,
      "grad_norm": 1.3507962648714655,
      "learning_rate": 9.113532707206116e-07,
      "log_odds_chosen": 0.8505493402481079,
      "log_odds_ratio": -0.510302722454071,
      "logits/chosen": -0.9306640625,
      "logits/rejected": -0.8521484136581421,
      "logps/chosen": -0.6041015386581421,
      "logps/rejected": -1.145898461341858,
      "loss": 0.9156,
      "nll_loss": 0.900585949420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03019409254193306,
      "rewards/margins": 0.027099991217255592,
      "rewards/rejected": -0.05736694484949112,
      "step": 3010
    },
    {
      "epoch": 0.22001238480311805,
      "grad_norm": 1.4871790658734243,
      "learning_rate": 9.098431565585488e-07,
      "log_odds_chosen": 0.730480968952179,
      "log_odds_ratio": -0.558789074420929,
      "logits/chosen": -0.925000011920929,
      "logits/rejected": -0.8373047113418579,
      "logps/chosen": -0.558789074420929,
      "logps/rejected": -0.9652343988418579,
      "loss": 0.9097,
      "nll_loss": 0.918164074420929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02794189378619194,
      "rewards/margins": 0.020308684557676315,
      "rewards/rejected": -0.04829101637005806,
      "step": 3020
    },
    {
      "epoch": 0.22074090263359195,
      "grad_norm": 1.5493139003247753,
      "learning_rate": 9.083405243909494e-07,
      "log_odds_chosen": 0.7729736566543579,
      "log_odds_ratio": -0.542919933795929,
      "logits/chosen": -0.96875,
      "logits/rejected": -0.857617199420929,
      "logps/chosen": -0.563769519329071,
      "logps/rejected": -1.042871117591858,
      "loss": 0.908,
      "nll_loss": 0.846875011920929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.02819213829934597,
      "rewards/margins": 0.023925017565488815,
      "rewards/rejected": -0.05211181566119194,
      "step": 3030
    },
    {
      "epoch": 0.22146942046406587,
      "grad_norm": 1.4794214562799732,
      "learning_rate": 9.068453126375147e-07,
      "log_odds_chosen": 0.757617175579071,
      "log_odds_ratio": -0.53173828125,
      "logits/chosen": -0.9332031011581421,
      "logits/rejected": -0.8583984375,
      "logps/chosen": -0.574023425579071,
      "logps/rejected": -1.053125023841858,
      "loss": 0.923,
      "nll_loss": 0.8841797113418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02867431566119194,
      "rewards/margins": 0.02397308312356472,
      "rewards/rejected": -0.05267333984375,
      "step": 3040
    },
    {
      "epoch": 0.22219793829453977,
      "grad_norm": 1.3241126754684722,
      "learning_rate": 9.053574604251853e-07,
      "log_odds_chosen": 0.7998046875,
      "log_odds_ratio": -0.523242175579071,
      "logits/chosen": -0.9369140863418579,
      "logits/rejected": -0.874218761920929,
      "logps/chosen": -0.550488293170929,
      "logps/rejected": -1.0232422351837158,
      "loss": 0.9208,
      "nll_loss": 0.851367175579071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.027496337890625,
      "rewards/margins": 0.02363433875143528,
      "rewards/rejected": -0.05114135891199112,
      "step": 3050
    },
    {
      "epoch": 0.22292645612501366,
      "grad_norm": 1.4033273394185453,
      "learning_rate": 9.03876907577734e-07,
      "log_odds_chosen": 0.9085693359375,
      "log_odds_ratio": -0.48442381620407104,
      "logits/chosen": -0.948046863079071,
      "logits/rejected": -0.818164050579071,
      "logps/chosen": -0.5367187261581421,
      "logps/rejected": -1.070898413658142,
      "loss": 0.9106,
      "nll_loss": 0.841796875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.026824951171875,
      "rewards/margins": 0.026747893542051315,
      "rewards/rejected": -0.05355224758386612,
      "step": 3060
    },
    {
      "epoch": 0.22365497395548756,
      "grad_norm": 1.586978729844288,
      "learning_rate": 9.024035946055421e-07,
      "log_odds_chosen": 0.8028320074081421,
      "log_odds_ratio": -0.5350586175918579,
      "logits/chosen": -0.8931640386581421,
      "logits/rejected": -0.808789074420929,
      "logps/chosen": -0.6258789300918579,
      "logps/rejected": -1.1335937976837158,
      "loss": 0.9131,
      "nll_loss": 0.882617175579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.03128661960363388,
      "rewards/margins": 0.02539977990090847,
      "rewards/rejected": -0.05667724460363388,
      "step": 3070
    },
    {
      "epoch": 0.22438349178596145,
      "grad_norm": 1.4353623599257552,
      "learning_rate": 9.00937462695559e-07,
      "log_odds_chosen": 0.6742187738418579,
      "log_odds_ratio": -0.5411132574081421,
      "logits/chosen": -0.9615234136581421,
      "logits/rejected": -0.8662109375,
      "logps/chosen": -0.5542968511581421,
      "logps/rejected": -0.9619140625,
      "loss": 0.8827,
      "nll_loss": 0.8447265625,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.0277099609375,
      "rewards/margins": 0.02041015587747097,
      "rewards/rejected": -0.04810791090130806,
      "step": 3080
    },
    {
      "epoch": 0.22511200961643535,
      "grad_norm": 1.3440018357449843,
      "learning_rate": 8.994784537014432e-07,
      "log_odds_chosen": 0.9351562261581421,
      "log_odds_ratio": -0.47607421875,
      "logits/chosen": -0.964062511920929,
      "logits/rejected": -0.869140625,
      "logps/chosen": -0.5400390625,
      "logps/rejected": -1.0871093273162842,
      "loss": 0.901,
      "nll_loss": 0.8255859613418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0269775390625,
      "rewards/margins": 0.027328109368681908,
      "rewards/rejected": -0.05431518703699112,
      "step": 3090
    },
    {
      "epoch": 0.22584052744690927,
      "grad_norm": 1.4635992984895636,
      "learning_rate": 8.980265101338747e-07,
      "log_odds_chosen": 0.848706066608429,
      "log_odds_ratio": -0.489501953125,
      "logits/chosen": -0.9312499761581421,
      "logits/rejected": -0.8695312738418579,
      "logps/chosen": -0.581347644329071,
      "logps/rejected": -1.0714843273162842,
      "loss": 0.8939,
      "nll_loss": 0.8384765386581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02904663048684597,
      "rewards/margins": 0.02454834058880806,
      "rewards/rejected": -0.05355224758386612,
      "step": 3100
    },
    {
      "epoch": 0.22656904527738317,
      "grad_norm": 1.5388486148796512,
      "learning_rate": 8.965815751510408e-07,
      "log_odds_chosen": 1.0378906726837158,
      "log_odds_ratio": -0.42949217557907104,
      "logits/chosen": -0.91796875,
      "logits/rejected": -0.7855468988418579,
      "logps/chosen": -0.5204101800918579,
      "logps/rejected": -1.1124999523162842,
      "loss": 0.8719,
      "nll_loss": 0.8531249761581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02601928636431694,
      "rewards/margins": 0.02957458421587944,
      "rewards/rejected": -0.05560302734375,
      "step": 3110
    },
    {
      "epoch": 0.22729756310785706,
      "grad_norm": 3.1256367830454233,
      "learning_rate": 8.951435925492912e-07,
      "log_odds_chosen": 0.848895251750946,
      "log_odds_ratio": -0.5264648199081421,
      "logits/chosen": -0.949414074420929,
      "logits/rejected": -0.8558593988418579,
      "logps/chosen": -0.561230480670929,
      "logps/rejected": -1.081640601158142,
      "loss": 0.8823,
      "nll_loss": 0.8394531011581421,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02805786207318306,
      "rewards/margins": 0.026039505377411842,
      "rewards/rejected": -0.05412597581744194,
      "step": 3120
    },
    {
      "epoch": 0.22802608093833096,
      "grad_norm": 1.4331307455992304,
      "learning_rate": 8.93712506753953e-07,
      "log_odds_chosen": 0.788378894329071,
      "log_odds_ratio": -0.5249999761581421,
      "logits/chosen": -0.921875,
      "logits/rejected": -0.841992199420929,
      "logps/chosen": -0.5650390386581421,
      "logps/rejected": -1.020898461341858,
      "loss": 0.885,
      "nll_loss": 0.875,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02824096754193306,
      "rewards/margins": 0.022850800305604935,
      "rewards/rejected": -0.05106201022863388,
      "step": 3130
    },
    {
      "epoch": 0.22875459876880486,
      "grad_norm": 1.3870836132601616,
      "learning_rate": 8.922882628103122e-07,
      "log_odds_chosen": 0.6246093511581421,
      "log_odds_ratio": -0.587695300579071,
      "logits/chosen": -0.9429687261581421,
      "logits/rejected": -0.859375,
      "logps/chosen": -0.56494140625,
      "logps/rejected": -0.931933581829071,
      "loss": 0.9035,
      "nll_loss": 0.829882800579071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02826538123190403,
      "rewards/margins": 0.01834564283490181,
      "rewards/rejected": -0.04659423977136612,
      "step": 3140
    },
    {
      "epoch": 0.22948311659927878,
      "grad_norm": 1.6798383966230093,
      "learning_rate": 8.90870806374748e-07,
      "log_odds_chosen": 0.8128906488418579,
      "log_odds_ratio": -0.515820324420929,
      "logits/chosen": -0.9488281011581421,
      "logits/rejected": -0.863476574420929,
      "logps/chosen": -0.6231445074081421,
      "logps/rejected": -1.1076171398162842,
      "loss": 0.9106,
      "nll_loss": 0.891796886920929,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0311279296875,
      "rewards/margins": 0.02423400804400444,
      "rewards/rejected": -0.05537109449505806,
      "step": 3150
    },
    {
      "epoch": 0.23021163442975268,
      "grad_norm": 1.415435720017948,
      "learning_rate": 8.894600837060251e-07,
      "log_odds_chosen": 0.9644531011581421,
      "log_odds_ratio": -0.45976561307907104,
      "logits/chosen": -0.9677734375,
      "logits/rejected": -0.8539062738418579,
      "logps/chosen": -0.518750011920929,
      "logps/rejected": -1.0564453601837158,
      "loss": 0.8959,
      "nll_loss": 0.8662109375,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02593994140625,
      "rewards/margins": 0.026871109381318092,
      "rewards/rejected": -0.052825927734375,
      "step": 3160
    },
    {
      "epoch": 0.23094015226022657,
      "grad_norm": 1.7569065242787367,
      "learning_rate": 8.880560416567349e-07,
      "log_odds_chosen": 0.7228149175643921,
      "log_odds_ratio": -0.5487304925918579,
      "logits/chosen": -0.8490234613418579,
      "logits/rejected": -0.8041015863418579,
      "logps/chosen": -0.5785156488418579,
      "logps/rejected": -0.9996093511581421,
      "loss": 0.8998,
      "nll_loss": 0.8480468988418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02892456017434597,
      "rewards/margins": 0.02107391320168972,
      "rewards/rejected": -0.04998779296875,
      "step": 3170
    },
    {
      "epoch": 0.23166867009070047,
      "grad_norm": 1.331613204089301,
      "learning_rate": 8.866586276648859e-07,
      "log_odds_chosen": 0.6680663824081421,
      "log_odds_ratio": -0.545605480670929,
      "logits/chosen": -0.952929675579071,
      "logits/rejected": -0.827343761920929,
      "logps/chosen": -0.5707031488418579,
      "logps/rejected": -0.9515625238418579,
      "loss": 0.9201,
      "nll_loss": 0.879687488079071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02856140211224556,
      "rewards/margins": 0.018971633166074753,
      "rewards/rejected": -0.04754638671875,
      "step": 3180
    },
    {
      "epoch": 0.23239718792117436,
      "grad_norm": 1.581499027427193,
      "learning_rate": 8.852677897456389e-07,
      "log_odds_chosen": 0.6622070074081421,
      "log_odds_ratio": -0.5540527105331421,
      "logits/chosen": -0.973828136920929,
      "logits/rejected": -0.862109363079071,
      "logps/chosen": -0.555859386920929,
      "logps/rejected": -0.9359375238418579,
      "loss": 0.9004,
      "nll_loss": 0.8407226800918579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02779540978372097,
      "rewards/margins": 0.01901702955365181,
      "rewards/rejected": -0.04683838039636612,
      "step": 3190
    },
    {
      "epoch": 0.23312570575164826,
      "grad_norm": 1.6907908899770634,
      "learning_rate": 8.838834764831844e-07,
      "log_odds_chosen": 0.741992175579071,
      "log_odds_ratio": -0.569531261920929,
      "logits/chosen": -0.974414050579071,
      "logits/rejected": -0.875,
      "logps/chosen": -0.600292980670929,
      "logps/rejected": -1.0458984375,
      "loss": 0.8719,
      "nll_loss": 0.8529297113418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.03001708909869194,
      "rewards/margins": 0.02227020263671875,
      "rewards/rejected": -0.05225219577550888,
      "step": 3200
    },
    {
      "epoch": 0.23385422358212218,
      "grad_norm": 1.5308861157234481,
      "learning_rate": 8.825056370227597e-07,
      "log_odds_chosen": 0.7325683832168579,
      "log_odds_ratio": -0.527294933795929,
      "logits/chosen": -0.917187511920929,
      "logits/rejected": -0.835742175579071,
      "logps/chosen": -0.57666015625,
      "logps/rejected": -0.993945300579071,
      "loss": 0.8957,
      "nll_loss": 0.843945324420929,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02886962890625,
      "rewards/margins": 0.02084198035299778,
      "rewards/rejected": -0.04965820163488388,
      "step": 3210
    },
    {
      "epoch": 0.23458274141259608,
      "grad_norm": 1.5417426023783956,
      "learning_rate": 8.811342210628018e-07,
      "log_odds_chosen": 0.7958984375,
      "log_odds_ratio": -0.49541014432907104,
      "logits/chosen": -0.9521484375,
      "logits/rejected": -0.8228515386581421,
      "logps/chosen": -0.567675769329071,
      "logps/rejected": -1.0419921875,
      "loss": 0.9145,
      "nll_loss": 0.878710925579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02839355543255806,
      "rewards/margins": 0.023651123046875,
      "rewards/rejected": -0.05206298828125,
      "step": 3220
    },
    {
      "epoch": 0.23531125924306998,
      "grad_norm": 1.7346027954656311,
      "learning_rate": 8.797691788472336e-07,
      "log_odds_chosen": 0.7486327886581421,
      "log_odds_ratio": -0.5220702886581421,
      "logits/chosen": -0.9380859136581421,
      "logits/rejected": -0.8521484136581421,
      "logps/chosen": -0.5918945074081421,
      "logps/rejected": -1.015039086341858,
      "loss": 0.899,
      "nll_loss": 0.9076172113418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02960205078125,
      "rewards/margins": 0.02117614820599556,
      "rewards/rejected": -0.05077514797449112,
      "step": 3230
    },
    {
      "epoch": 0.23603977707354387,
      "grad_norm": 1.4244537526510177,
      "learning_rate": 8.784104611578832e-07,
      "log_odds_chosen": 0.8416992425918579,
      "log_odds_ratio": -0.4893554747104645,
      "logits/chosen": -0.9457031488418579,
      "logits/rejected": -0.848437488079071,
      "logps/chosen": -0.561328113079071,
      "logps/rejected": -1.0300781726837158,
      "loss": 0.8936,
      "nll_loss": 0.839648425579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02809448167681694,
      "rewards/margins": 0.02339324913918972,
      "rewards/rejected": -0.05147705227136612,
      "step": 3240
    },
    {
      "epoch": 0.23676829490401777,
      "grad_norm": 1.5958979814206269,
      "learning_rate": 8.770580193070293e-07,
      "log_odds_chosen": 1.0061523914337158,
      "log_odds_ratio": -0.4327148497104645,
      "logits/chosen": -0.92578125,
      "logits/rejected": -0.850781261920929,
      "logps/chosen": -0.527636706829071,
      "logps/rejected": -1.0867187976837158,
      "loss": 0.9004,
      "nll_loss": 0.8822265863418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.026397705078125,
      "rewards/margins": 0.02797546423971653,
      "rewards/rejected": -0.0543212890625,
      "step": 3250
    },
    {
      "epoch": 0.23749681273449166,
      "grad_norm": 1.7697307005198555,
      "learning_rate": 8.757118051300735e-07,
      "log_odds_chosen": 0.8946288824081421,
      "log_odds_ratio": -0.49726563692092896,
      "logits/chosen": -0.959765613079071,
      "logits/rejected": -0.83984375,
      "logps/chosen": -0.579785168170929,
      "logps/rejected": -1.0973632335662842,
      "loss": 0.8983,
      "nll_loss": 0.8509765863418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02898559533059597,
      "rewards/margins": 0.02583770826458931,
      "rewards/rejected": -0.054901123046875,
      "step": 3260
    },
    {
      "epoch": 0.2382253305649656,
      "grad_norm": 1.469383909094762,
      "learning_rate": 8.743717709783363e-07,
      "log_odds_chosen": 0.6790527105331421,
      "log_odds_ratio": -0.5619140863418579,
      "logits/chosen": -0.963671863079071,
      "logits/rejected": -0.8773437738418579,
      "logps/chosen": -0.557666003704071,
      "logps/rejected": -0.915234386920929,
      "loss": 0.9123,
      "nll_loss": 0.807812511920929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0279083251953125,
      "rewards/margins": 0.01788330078125,
      "rewards/rejected": -0.04578857496380806,
      "step": 3270
    },
    {
      "epoch": 0.23895384839543948,
      "grad_norm": 1.46747668921437,
      "learning_rate": 8.730378697119729e-07,
      "log_odds_chosen": 0.6919921636581421,
      "log_odds_ratio": -0.556933581829071,
      "logits/chosen": -0.9742187261581421,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.619824230670929,
      "logps/rejected": -1.015039086341858,
      "loss": 0.9152,
      "nll_loss": 0.924609363079071,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03101196326315403,
      "rewards/margins": 0.019748687744140625,
      "rewards/rejected": -0.05072021484375,
      "step": 3280
    },
    {
      "epoch": 0.23968236622591338,
      "grad_norm": 1.5039486308037944,
      "learning_rate": 8.717100546930084e-07,
      "log_odds_chosen": 1.0233886241912842,
      "log_odds_ratio": -0.446044921875,
      "logits/chosen": -0.968554675579071,
      "logits/rejected": -0.851757824420929,
      "logps/chosen": -0.5303710699081421,
      "logps/rejected": -1.138281226158142,
      "loss": 0.8899,
      "nll_loss": 0.8246093988418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02653198316693306,
      "rewards/margins": 0.03038635291159153,
      "rewards/rejected": -0.05692138522863388,
      "step": 3290
    },
    {
      "epoch": 0.24041088405638728,
      "grad_norm": 1.350718040665698,
      "learning_rate": 8.703882797784894e-07,
      "log_odds_chosen": 0.795117199420929,
      "log_odds_ratio": -0.49443358182907104,
      "logits/chosen": -0.938281238079071,
      "logits/rejected": -0.8330078125,
      "logps/chosen": -0.605175793170929,
      "logps/rejected": -1.054101586341858,
      "loss": 0.895,
      "nll_loss": 0.9693359136581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.03026733361184597,
      "rewards/margins": 0.022430419921875,
      "rewards/rejected": -0.05270995944738388,
      "step": 3300
    },
    {
      "epoch": 0.24113940188686117,
      "grad_norm": 1.7486273528350769,
      "learning_rate": 8.690724993137478e-07,
      "log_odds_chosen": 0.59100341796875,
      "log_odds_ratio": -0.5973144769668579,
      "logits/chosen": -0.9380859136581421,
      "logits/rejected": -0.879687488079071,
      "logps/chosen": -0.60595703125,
      "logps/rejected": -0.995800793170929,
      "loss": 0.9108,
      "nll_loss": 0.9556640386581421,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.03029174730181694,
      "rewards/margins": 0.01952209509909153,
      "rewards/rejected": -0.0498046875,
      "step": 3310
    },
    {
      "epoch": 0.2418679197173351,
      "grad_norm": 1.4247923501203736,
      "learning_rate": 8.677626681257792e-07,
      "log_odds_chosen": 0.854785144329071,
      "log_odds_ratio": -0.4901367127895355,
      "logits/chosen": -0.9515625238418579,
      "logits/rejected": -0.87109375,
      "logps/chosen": -0.5943359136581421,
      "logps/rejected": -1.0984375476837158,
      "loss": 0.8853,
      "nll_loss": 0.825976550579071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02969970740377903,
      "rewards/margins": 0.02519531175494194,
      "rewards/rejected": -0.054931640625,
      "step": 3320
    },
    {
      "epoch": 0.242596437547809,
      "grad_norm": 1.5608635313501997,
      "learning_rate": 8.664587415167274e-07,
      "log_odds_chosen": 0.764892578125,
      "log_odds_ratio": -0.543164074420929,
      "logits/chosen": -0.931835949420929,
      "logits/rejected": -0.851757824420929,
      "logps/chosen": -0.601367175579071,
      "logps/rejected": -1.044531226158142,
      "loss": 0.8889,
      "nll_loss": 0.8642578125,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.03007202222943306,
      "rewards/margins": 0.022115325555205345,
      "rewards/rejected": -0.05220947414636612,
      "step": 3330
    },
    {
      "epoch": 0.2433249553782829,
      "grad_norm": 1.5062804754209616,
      "learning_rate": 8.651606752574786e-07,
      "log_odds_chosen": 0.9329468011856079,
      "log_odds_ratio": -0.49028319120407104,
      "logits/chosen": -0.931640625,
      "logits/rejected": -0.81640625,
      "logps/chosen": -0.5458984375,
      "logps/rejected": -1.0556640625,
      "loss": 0.8819,
      "nll_loss": 0.88671875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02731933631002903,
      "rewards/margins": 0.025493621826171875,
      "rewards/rejected": -0.05280761793255806,
      "step": 3340
    },
    {
      "epoch": 0.24405347320875678,
      "grad_norm": 1.4091259182402542,
      "learning_rate": 8.638684255813602e-07,
      "log_odds_chosen": 0.6739746332168579,
      "log_odds_ratio": -0.5716797113418579,
      "logits/chosen": -0.910351574420929,
      "logits/rejected": -0.8173828125,
      "logps/chosen": -0.598339855670929,
      "logps/rejected": -1.0109374523162842,
      "loss": 0.9062,
      "nll_loss": 0.890820324420929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.02991943433880806,
      "rewards/margins": 0.0206146240234375,
      "rewards/rejected": -0.050537109375,
      "step": 3350
    },
    {
      "epoch": 0.24478199103923068,
      "grad_norm": 1.438218098550839,
      "learning_rate": 8.625819491779427e-07,
      "log_odds_chosen": 0.768847644329071,
      "log_odds_ratio": -0.508496105670929,
      "logits/chosen": -0.9248046875,
      "logits/rejected": -0.8267577886581421,
      "logps/chosen": -0.5767577886581421,
      "logps/rejected": -1.0451171398162842,
      "loss": 0.8903,
      "nll_loss": 0.8740234375,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02882690355181694,
      "rewards/margins": 0.02344970777630806,
      "rewards/rejected": -0.05225830152630806,
      "step": 3360
    },
    {
      "epoch": 0.24551050886970457,
      "grad_norm": 1.4273083258293784,
      "learning_rate": 8.613012031869432e-07,
      "log_odds_chosen": 0.7571471929550171,
      "log_odds_ratio": -0.533496081829071,
      "logits/chosen": -0.916210949420929,
      "logits/rejected": -0.8187500238418579,
      "logps/chosen": -0.540722668170929,
      "logps/rejected": -0.9800781011581421,
      "loss": 0.8821,
      "nll_loss": 0.863085925579071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02701416052877903,
      "rewards/margins": 0.02197876013815403,
      "rewards/rejected": -0.04902954027056694,
      "step": 3370
    },
    {
      "epoch": 0.2462390267001785,
      "grad_norm": 1.5039807090081188,
      "learning_rate": 8.600261451922269e-07,
      "log_odds_chosen": 1.181640625,
      "log_odds_ratio": -0.4149414002895355,
      "logits/chosen": -0.9345703125,
      "logits/rejected": -0.842578113079071,
      "logps/chosen": -0.5321289300918579,
      "logps/rejected": -1.2677733898162842,
      "loss": 0.8992,
      "nll_loss": 0.8587890863418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.026580810546875,
      "rewards/margins": 0.03682556003332138,
      "rewards/rejected": -0.06345214694738388,
      "step": 3380
    },
    {
      "epoch": 0.2469675445306524,
      "grad_norm": 1.490199889976916,
      "learning_rate": 8.587567332159079e-07,
      "log_odds_chosen": 0.9180663824081421,
      "log_odds_ratio": -0.5150390863418579,
      "logits/chosen": -0.970898449420929,
      "logits/rejected": -0.8515625,
      "logps/chosen": -0.557324230670929,
      "logps/rejected": -1.1279296875,
      "loss": 0.9204,
      "nll_loss": 0.863085925579071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02784423902630806,
      "rewards/margins": 0.02854614332318306,
      "rewards/rejected": -0.05640869215130806,
      "step": 3390
    },
    {
      "epoch": 0.2476960623611263,
      "grad_norm": 1.4609217103008818,
      "learning_rate": 8.574929257125441e-07,
      "log_odds_chosen": 0.8778320550918579,
      "log_odds_ratio": -0.539843738079071,
      "logits/chosen": -0.935742199420929,
      "logits/rejected": -0.884570300579071,
      "logps/chosen": -0.590039074420929,
      "logps/rejected": -1.105859398841858,
      "loss": 0.8752,
      "nll_loss": 0.8363281488418579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02951965294778347,
      "rewards/margins": 0.02579956129193306,
      "rewards/rejected": -0.05528564378619194,
      "step": 3400
    },
    {
      "epoch": 0.24842458019160019,
      "grad_norm": 1.476689622238305,
      "learning_rate": 8.562346815634272e-07,
      "log_odds_chosen": 0.7204834222793579,
      "log_odds_ratio": -0.5301758050918579,
      "logits/chosen": -0.958789050579071,
      "logits/rejected": -0.8666015863418579,
      "logps/chosen": -0.5770508050918579,
      "logps/rejected": -1.0242187976837158,
      "loss": 0.8928,
      "nll_loss": 0.8701171875,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02885131910443306,
      "rewards/margins": 0.02237396314740181,
      "rewards/rejected": -0.051239013671875,
      "step": 3410
    },
    {
      "epoch": 0.24915309802207408,
      "grad_norm": 1.493129076954762,
      "learning_rate": 8.549819600709619e-07,
      "log_odds_chosen": 0.8148193359375,
      "log_odds_ratio": -0.521777331829071,
      "logits/chosen": -0.915820300579071,
      "logits/rejected": -0.81640625,
      "logps/chosen": -0.597460925579071,
      "logps/rejected": -1.096093773841858,
      "loss": 0.9073,
      "nll_loss": 0.9058593511581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.029876708984375,
      "rewards/margins": 0.024900436401367188,
      "rewards/rejected": -0.05479736253619194,
      "step": 3420
    },
    {
      "epoch": 0.24988161585254798,
      "grad_norm": 1.4231173677740727,
      "learning_rate": 8.537347209531384e-07,
      "log_odds_chosen": 0.947460949420929,
      "log_odds_ratio": -0.48359376192092896,
      "logits/chosen": -0.947070300579071,
      "logits/rejected": -0.859375,
      "logps/chosen": -0.5323241949081421,
      "logps/rejected": -1.0810546875,
      "loss": 0.8866,
      "nll_loss": 0.8394531011581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02663574181497097,
      "rewards/margins": 0.02742309495806694,
      "rewards/rejected": -0.05400390550494194,
      "step": 3430
    },
    {
      "epoch": 0.2506101336830219,
      "grad_norm": 1.9349241527576937,
      "learning_rate": 8.52492924338092e-07,
      "log_odds_chosen": 0.843554675579071,
      "log_odds_ratio": -0.502734363079071,
      "logits/chosen": -0.939257800579071,
      "logits/rejected": -0.8505859375,
      "logps/chosen": -0.60693359375,
      "logps/rejected": -1.1271483898162842,
      "loss": 0.8792,
      "nll_loss": 0.8832031488418579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.03034668043255806,
      "rewards/margins": 0.02598876878619194,
      "rewards/rejected": -0.05629882961511612,
      "step": 3440
    },
    {
      "epoch": 0.25133865151349577,
      "grad_norm": 1.4930459205860565,
      "learning_rate": 8.512565307587487e-07,
      "log_odds_chosen": 0.743298351764679,
      "log_odds_ratio": -0.54052734375,
      "logits/chosen": -0.983593761920929,
      "logits/rejected": -0.8623046875,
      "logps/chosen": -0.6011718511581421,
      "logps/rejected": -1.048242211341858,
      "loss": 0.8988,
      "nll_loss": 0.883984386920929,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.03007202222943306,
      "rewards/margins": 0.022317886352539062,
      "rewards/rejected": -0.05238037183880806,
      "step": 3450
    },
    {
      "epoch": 0.2520671693439697,
      "grad_norm": 1.574146280453646,
      "learning_rate": 8.500255011475575e-07,
      "log_odds_chosen": 0.762561023235321,
      "log_odds_ratio": -0.5669921636581421,
      "logits/chosen": -0.9166015386581421,
      "logits/rejected": -0.8519531488418579,
      "logps/chosen": -0.5933593511581421,
      "logps/rejected": -1.0759766101837158,
      "loss": 0.9038,
      "nll_loss": 0.860156238079071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.02963867224752903,
      "rewards/margins": 0.024152373895049095,
      "rewards/rejected": -0.0538330078125,
      "step": 3460
    },
    {
      "epoch": 0.2527956871744436,
      "grad_norm": 1.6269323155999924,
      "learning_rate": 8.48799796831305e-07,
      "log_odds_chosen": 1.0134766101837158,
      "log_odds_ratio": -0.4601074159145355,
      "logits/chosen": -0.938671886920929,
      "logits/rejected": -0.8353515863418579,
      "logps/chosen": -0.563281238079071,
      "logps/rejected": -1.1552734375,
      "loss": 0.8854,
      "nll_loss": 0.853320300579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02817382849752903,
      "rewards/margins": 0.02960205078125,
      "rewards/rejected": -0.05779419094324112,
      "step": 3470
    },
    {
      "epoch": 0.2535242050049175,
      "grad_norm": 1.567764575915131,
      "learning_rate": 8.475793795260132e-07,
      "log_odds_chosen": 0.85784912109375,
      "log_odds_ratio": -0.53369140625,
      "logits/chosen": -0.9642578363418579,
      "logits/rejected": -0.840039074420929,
      "logps/chosen": -0.6410156488418579,
      "logps/rejected": -1.1886718273162842,
      "loss": 0.9065,
      "nll_loss": 0.8619140386581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03201904147863388,
      "rewards/margins": 0.02736663818359375,
      "rewards/rejected": -0.05943603441119194,
      "step": 3480
    },
    {
      "epoch": 0.2542527228353914,
      "grad_norm": 1.5923747626809492,
      "learning_rate": 8.463642113319158e-07,
      "log_odds_chosen": 0.813720703125,
      "log_odds_ratio": -0.5340820550918579,
      "logits/chosen": -0.9117187261581421,
      "logits/rejected": -0.8216797113418579,
      "logps/chosen": -0.5811523199081421,
      "logps/rejected": -1.0769531726837158,
      "loss": 0.903,
      "nll_loss": 0.8265625238418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02907714806497097,
      "rewards/margins": 0.02477111853659153,
      "rewards/rejected": -0.05384521558880806,
      "step": 3490
    },
    {
      "epoch": 0.2549812406658653,
      "grad_norm": 1.37163074637473,
      "learning_rate": 8.451542547285166e-07,
      "log_odds_chosen": 0.907421886920929,
      "log_odds_ratio": -0.53173828125,
      "logits/chosen": -0.9058593511581421,
      "logits/rejected": -0.8492187261581421,
      "logps/chosen": -0.6075195074081421,
      "logps/rejected": -1.179296851158142,
      "loss": 0.8801,
      "nll_loss": 0.8656250238418579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.03037719801068306,
      "rewards/margins": 0.02854461595416069,
      "rewards/rejected": -0.05889892578125,
      "step": 3500
    },
    {
      "epoch": 0.2557097584963392,
      "grad_norm": 1.579798419222017,
      "learning_rate": 8.439494725697223e-07,
      "log_odds_chosen": 0.9288574457168579,
      "log_odds_ratio": -0.47685545682907104,
      "logits/chosen": -0.9437500238418579,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.5580078363418579,
      "logps/rejected": -1.0869140625,
      "loss": 0.8747,
      "nll_loss": 0.8394531011581421,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02787475660443306,
      "rewards/margins": 0.02646484412252903,
      "rewards/rejected": -0.05438232421875,
      "step": 3510
    },
    {
      "epoch": 0.2564382763268131,
      "grad_norm": 1.4255163276092582,
      "learning_rate": 8.427498280790526e-07,
      "log_odds_chosen": 0.732470691204071,
      "log_odds_ratio": -0.5699218511581421,
      "logits/chosen": -0.9798828363418579,
      "logits/rejected": -0.8880859613418579,
      "logps/chosen": -0.6162109375,
      "logps/rejected": -1.095312476158142,
      "loss": 0.894,
      "nll_loss": 0.881054699420929,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.03082275390625,
      "rewards/margins": 0.02388153038918972,
      "rewards/rejected": -0.0546875,
      "step": 3520
    },
    {
      "epoch": 0.257166794157287,
      "grad_norm": 1.5633142073892945,
      "learning_rate": 8.415552848449264e-07,
      "log_odds_chosen": 0.8133789300918579,
      "log_odds_ratio": -0.4950195252895355,
      "logits/chosen": -0.964062511920929,
      "logits/rejected": -0.880664050579071,
      "logps/chosen": -0.583203136920929,
      "logps/rejected": -1.0828125476837158,
      "loss": 0.8978,
      "nll_loss": 0.900195300579071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02916870079934597,
      "rewards/margins": 0.02493438683450222,
      "rewards/rejected": -0.05410156399011612,
      "step": 3530
    },
    {
      "epoch": 0.2578953119877609,
      "grad_norm": 1.544337300637992,
      "learning_rate": 8.40365806816018e-07,
      "log_odds_chosen": 0.843945324420929,
      "log_odds_ratio": -0.502246081829071,
      "logits/chosen": -0.928906261920929,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -0.5453125238418579,
      "logps/rejected": -1.0546875,
      "loss": 0.8905,
      "nll_loss": 0.836132824420929,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02722778357565403,
      "rewards/margins": 0.025499725714325905,
      "rewards/rejected": -0.05275879055261612,
      "step": 3540
    },
    {
      "epoch": 0.2586238298182348,
      "grad_norm": 1.7941323153159834,
      "learning_rate": 8.391813582966891e-07,
      "log_odds_chosen": 0.602825939655304,
      "log_odds_ratio": -0.587597668170929,
      "logits/chosen": -0.9443359375,
      "logits/rejected": -0.875781238079071,
      "logps/chosen": -0.58935546875,
      "logps/rejected": -0.970507800579071,
      "loss": 0.8924,
      "nll_loss": 0.8705078363418579,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.02945556677877903,
      "rewards/margins": 0.019066238775849342,
      "rewards/rejected": -0.04852294921875,
      "step": 3550
    },
    {
      "epoch": 0.2593523476487087,
      "grad_norm": 1.4994165315897447,
      "learning_rate": 8.380019039424888e-07,
      "log_odds_chosen": 0.6723998785018921,
      "log_odds_ratio": -0.533886730670929,
      "logits/chosen": -0.9359375238418579,
      "logits/rejected": -0.8492187261581421,
      "logps/chosen": -0.586132824420929,
      "logps/rejected": -0.9658203125,
      "loss": 0.8974,
      "nll_loss": 0.8779296875,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02930908277630806,
      "rewards/margins": 0.01901702955365181,
      "rewards/rejected": -0.048309326171875,
      "step": 3560
    },
    {
      "epoch": 0.2600808654791826,
      "grad_norm": 1.5343518155996114,
      "learning_rate": 8.368274087557231e-07,
      "log_odds_chosen": 0.7240234613418579,
      "log_odds_ratio": -0.5361328125,
      "logits/chosen": -0.955078125,
      "logits/rejected": -0.843945324420929,
      "logps/chosen": -0.59521484375,
      "logps/rejected": -1.018945336341858,
      "loss": 0.8736,
      "nll_loss": 0.873046875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02977294847369194,
      "rewards/margins": 0.02118225023150444,
      "rewards/rejected": -0.05096435546875,
      "step": 3570
    },
    {
      "epoch": 0.26080938330965653,
      "grad_norm": 1.8356870379475305,
      "learning_rate": 8.356578380810946e-07,
      "log_odds_chosen": 0.77392578125,
      "log_odds_ratio": -0.5033203363418579,
      "logits/chosen": -0.9378906488418579,
      "logits/rejected": -0.852734386920929,
      "logps/chosen": -0.5611327886581421,
      "logps/rejected": -1.0041015148162842,
      "loss": 0.8759,
      "nll_loss": 0.826953113079071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02805786207318306,
      "rewards/margins": 0.022186279296875,
      "rewards/rejected": -0.05020751804113388,
      "step": 3580
    },
    {
      "epoch": 0.2615379011401304,
      "grad_norm": 1.4618509524506902,
      "learning_rate": 8.344931576014064e-07,
      "log_odds_chosen": 0.763232409954071,
      "log_odds_ratio": -0.5165039300918579,
      "logits/chosen": -0.9345703125,
      "logits/rejected": -0.8421875238418579,
      "logps/chosen": -0.5986328125,
      "logps/rejected": -1.043359398841858,
      "loss": 0.8926,
      "nll_loss": 0.8822265863418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02993164025247097,
      "rewards/margins": 0.02222900465130806,
      "rewards/rejected": -0.05216064304113388,
      "step": 3590
    },
    {
      "epoch": 0.2622664189706043,
      "grad_norm": 1.4742461255335868,
      "learning_rate": 8.333333333333333e-07,
      "log_odds_chosen": 0.957324206829071,
      "log_odds_ratio": -0.46918946504592896,
      "logits/chosen": -0.9195312261581421,
      "logits/rejected": -0.796679675579071,
      "logps/chosen": -0.5648437738418579,
      "logps/rejected": -1.143945336341858,
      "loss": 0.8877,
      "nll_loss": 0.8333984613418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.028228759765625,
      "rewards/margins": 0.02896423265337944,
      "rewards/rejected": -0.05717773362994194,
      "step": 3600
    },
    {
      "epoch": 0.2629949368010782,
      "grad_norm": 1.5687611512458783,
      "learning_rate": 8.321783316232578e-07,
      "log_odds_chosen": 0.7798827886581421,
      "log_odds_ratio": -0.5140380859375,
      "logits/chosen": -0.924023449420929,
      "logits/rejected": -0.844531238079071,
      "logps/chosen": -0.5902343988418579,
      "logps/rejected": -1.056640625,
      "loss": 0.8725,
      "nll_loss": 0.8828125,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02951660193502903,
      "rewards/margins": 0.02330932579934597,
      "rewards/rejected": -0.05283202975988388,
      "step": 3610
    },
    {
      "epoch": 0.2637234546315521,
      "grad_norm": 1.3420819706859284,
      "learning_rate": 8.310281191431671e-07,
      "log_odds_chosen": 0.623583972454071,
      "log_odds_ratio": -0.58544921875,
      "logits/chosen": -0.96484375,
      "logits/rejected": -0.886914074420929,
      "logps/chosen": -0.6114257574081421,
      "logps/rejected": -0.971484363079071,
      "loss": 0.8773,
      "nll_loss": 0.8896484375,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.03057861328125,
      "rewards/margins": 0.01799926720559597,
      "rewards/rejected": -0.04856567457318306,
      "step": 3620
    },
    {
      "epoch": 0.264451972462026,
      "grad_norm": 1.4577103126773563,
      "learning_rate": 8.298826628866154e-07,
      "log_odds_chosen": 0.935839831829071,
      "log_odds_ratio": -0.4634765684604645,
      "logits/chosen": -0.96875,
      "logits/rejected": -0.851367175579071,
      "logps/chosen": -0.5770508050918579,
      "logps/rejected": -1.141992211341858,
      "loss": 0.8825,
      "nll_loss": 0.883984386920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02886962890625,
      "rewards/margins": 0.02822265587747097,
      "rewards/rejected": -0.05710449069738388,
      "step": 3630
    },
    {
      "epoch": 0.2651804902924999,
      "grad_norm": 1.503949033819343,
      "learning_rate": 8.287419301647449e-07,
      "log_odds_chosen": 0.793261706829071,
      "log_odds_ratio": -0.525585949420929,
      "logits/chosen": -0.940625011920929,
      "logits/rejected": -0.8119140863418579,
      "logps/chosen": -0.6001952886581421,
      "logps/rejected": -1.1044921875,
      "loss": 0.8843,
      "nll_loss": 0.8541015386581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02998657152056694,
      "rewards/margins": 0.0252227783203125,
      "rewards/rejected": -0.05524902418255806,
      "step": 3640
    },
    {
      "epoch": 0.2659090081229738,
      "grad_norm": 1.5276481466839764,
      "learning_rate": 8.27605888602368e-07,
      "log_odds_chosen": 0.783642590045929,
      "log_odds_ratio": -0.5279296636581421,
      "logits/chosen": -0.94140625,
      "logits/rejected": -0.868945300579071,
      "logps/chosen": -0.568554699420929,
      "logps/rejected": -1.0109374523162842,
      "loss": 0.888,
      "nll_loss": 0.866992175579071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02843627892434597,
      "rewards/margins": 0.022101592272520065,
      "rewards/rejected": -0.05057372897863388,
      "step": 3650
    },
    {
      "epoch": 0.2666375259534477,
      "grad_norm": 1.586876052413893,
      "learning_rate": 8.264745061341079e-07,
      "log_odds_chosen": 0.916699230670929,
      "log_odds_ratio": -0.49755859375,
      "logits/chosen": -0.939453125,
      "logits/rejected": -0.850390613079071,
      "logps/chosen": -0.5947265625,
      "logps/rejected": -1.1560547351837158,
      "loss": 0.9092,
      "nll_loss": 0.9251953363418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02971801720559597,
      "rewards/margins": 0.02809600904583931,
      "rewards/rejected": -0.05783691257238388,
      "step": 3660
    },
    {
      "epoch": 0.2673660437839216,
      "grad_norm": 1.419064233832202,
      "learning_rate": 8.253477510005973e-07,
      "log_odds_chosen": 0.8990722894668579,
      "log_odds_ratio": -0.4966796934604645,
      "logits/chosen": -0.9310547113418579,
      "logits/rejected": -0.863476574420929,
      "logps/chosen": -0.544921875,
      "logps/rejected": -1.0783202648162842,
      "loss": 0.8609,
      "nll_loss": 0.8203125,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.027252197265625,
      "rewards/margins": 0.02666015550494194,
      "rewards/rejected": -0.05388794094324112,
      "step": 3670
    },
    {
      "epoch": 0.2680945616143955,
      "grad_norm": 1.6918631033764964,
      "learning_rate": 8.24225591744734e-07,
      "log_odds_chosen": 0.784716784954071,
      "log_odds_ratio": -0.533886730670929,
      "logits/chosen": -0.9056640863418579,
      "logits/rejected": -0.8140624761581421,
      "logps/chosen": -0.576171875,
      "logps/rejected": -1.016015648841858,
      "loss": 0.8976,
      "nll_loss": 0.8755859136581421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02882690355181694,
      "rewards/margins": 0.021976470947265625,
      "rewards/rejected": -0.05083007737994194,
      "step": 3680
    },
    {
      "epoch": 0.26882307944486944,
      "grad_norm": 1.4431799865531587,
      "learning_rate": 8.231079972079914e-07,
      "log_odds_chosen": 0.749768078327179,
      "log_odds_ratio": -0.5430663824081421,
      "logits/chosen": -0.9761718511581421,
      "logits/rejected": -0.8521484136581421,
      "logps/chosen": -0.5947265625,
      "logps/rejected": -1.095117211341858,
      "loss": 0.8743,
      "nll_loss": 0.852343738079071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02973632887005806,
      "rewards/margins": 0.025009918957948685,
      "rewards/rejected": -0.05477295070886612,
      "step": 3690
    },
    {
      "epoch": 0.26955159727534334,
      "grad_norm": 1.5020166949090745,
      "learning_rate": 8.219949365267865e-07,
      "log_odds_chosen": 0.7906249761581421,
      "log_odds_ratio": -0.53759765625,
      "logits/chosen": -0.903515636920929,
      "logits/rejected": -0.8236328363418579,
      "logps/chosen": -0.5697265863418579,
      "logps/rejected": -1.0212891101837158,
      "loss": 0.8912,
      "nll_loss": 0.8675781488418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02850341796875,
      "rewards/margins": 0.02260284498333931,
      "rewards/rejected": -0.05108642578125,
      "step": 3700
    },
    {
      "epoch": 0.27028011510581723,
      "grad_norm": 1.5603049275440746,
      "learning_rate": 8.208863791288982e-07,
      "log_odds_chosen": 0.864453136920929,
      "log_odds_ratio": -0.5154784917831421,
      "logits/chosen": -0.923046886920929,
      "logits/rejected": -0.8345702886581421,
      "logps/chosen": -0.587890625,
      "logps/rejected": -1.124414086341858,
      "loss": 0.8951,
      "nll_loss": 0.8994140625,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02940673753619194,
      "rewards/margins": 0.026851654052734375,
      "rewards/rejected": -0.05623779445886612,
      "step": 3710
    },
    {
      "epoch": 0.2710086329362911,
      "grad_norm": 1.618927700165,
      "learning_rate": 8.197822947299412e-07,
      "log_odds_chosen": 1.102929711341858,
      "log_odds_ratio": -0.4439941346645355,
      "logits/chosen": -0.922656238079071,
      "logits/rejected": -0.829882800579071,
      "logps/chosen": -0.5445312261581421,
      "logps/rejected": -1.193359375,
      "loss": 0.8777,
      "nll_loss": 0.834179699420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02719726599752903,
      "rewards/margins": 0.03246154636144638,
      "rewards/rejected": -0.05964965745806694,
      "step": 3720
    },
    {
      "epoch": 0.271737150766765,
      "grad_norm": 1.6342613467857443,
      "learning_rate": 8.186826533298912e-07,
      "log_odds_chosen": 0.758654773235321,
      "log_odds_ratio": -0.5391601324081421,
      "logits/chosen": -0.92578125,
      "logits/rejected": -0.8330078125,
      "logps/chosen": -0.591503918170929,
      "logps/rejected": -1.046484351158142,
      "loss": 0.8816,
      "nll_loss": 0.8890625238418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02956542931497097,
      "rewards/margins": 0.022768402472138405,
      "rewards/rejected": -0.05233154445886612,
      "step": 3730
    },
    {
      "epoch": 0.2724656685972389,
      "grad_norm": 1.7335563641201865,
      "learning_rate": 8.175874252096609e-07,
      "log_odds_chosen": 0.987597644329071,
      "log_odds_ratio": -0.48662108182907104,
      "logits/chosen": -0.896484375,
      "logits/rejected": -0.800585925579071,
      "logps/chosen": -0.592578113079071,
      "logps/rejected": -1.2019531726837158,
      "loss": 0.8865,
      "nll_loss": 0.9134765863418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02960815466940403,
      "rewards/margins": 0.03045959398150444,
      "rewards/rejected": -0.06008300930261612,
      "step": 3740
    },
    {
      "epoch": 0.2731941864277128,
      "grad_norm": 1.514956151492815,
      "learning_rate": 8.164965809277262e-07,
      "log_odds_chosen": 0.787890613079071,
      "log_odds_ratio": -0.5205078125,
      "logits/chosen": -0.9320312738418579,
      "logits/rejected": -0.8447265625,
      "logps/chosen": -0.559374988079071,
      "logps/rejected": -0.9917968511581421,
      "loss": 0.8843,
      "nll_loss": 0.869140625,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.0279541015625,
      "rewards/margins": 0.02164611779153347,
      "rewards/rejected": -0.04957275465130806,
      "step": 3750
    },
    {
      "epoch": 0.2739227042581867,
      "grad_norm": 1.6444784490622804,
      "learning_rate": 8.154100913168028e-07,
      "log_odds_chosen": 0.783886730670929,
      "log_odds_ratio": -0.4922851622104645,
      "logits/chosen": -0.976367175579071,
      "logits/rejected": -0.849414050579071,
      "logps/chosen": -0.553515613079071,
      "logps/rejected": -1.0339844226837158,
      "loss": 0.8736,
      "nll_loss": 0.823437511920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02770385704934597,
      "rewards/margins": 0.02398071251809597,
      "rewards/rejected": -0.05164794996380806,
      "step": 3760
    },
    {
      "epoch": 0.2746512220886606,
      "grad_norm": 1.5399729376241158,
      "learning_rate": 8.143279274805705e-07,
      "log_odds_chosen": 1.054711937904358,
      "log_odds_ratio": -0.45502930879592896,
      "logits/chosen": -0.9449218511581421,
      "logits/rejected": -0.8433593511581421,
      "logps/chosen": -0.5233398675918579,
      "logps/rejected": -1.1609375476837158,
      "loss": 0.8774,
      "nll_loss": 0.843945324420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02616577222943306,
      "rewards/margins": 0.03188476711511612,
      "rewards/rejected": -0.058013916015625,
      "step": 3770
    },
    {
      "epoch": 0.2753797399191345,
      "grad_norm": 1.6665405816258774,
      "learning_rate": 8.132500607904444e-07,
      "log_odds_chosen": 1.031091332435608,
      "log_odds_ratio": -0.4662109315395355,
      "logits/chosen": -0.9974609613418579,
      "logits/rejected": -0.882031261920929,
      "logps/chosen": -0.563183605670929,
      "logps/rejected": -1.1875,
      "loss": 0.8529,
      "nll_loss": 0.7679687738418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02816162072122097,
      "rewards/margins": 0.0313262939453125,
      "rewards/rejected": -0.05943603441119194,
      "step": 3780
    },
    {
      "epoch": 0.2761082577496084,
      "grad_norm": 1.4905300621487052,
      "learning_rate": 8.12176462882395e-07,
      "log_odds_chosen": 0.887133777141571,
      "log_odds_ratio": -0.49882811307907104,
      "logits/chosen": -0.9515625238418579,
      "logits/rejected": -0.861328125,
      "logps/chosen": -0.5967773199081421,
      "logps/rejected": -1.1173827648162842,
      "loss": 0.8844,
      "nll_loss": 0.833789050579071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02984619140625,
      "rewards/margins": 0.025994110852479935,
      "rewards/rejected": -0.05583496019244194,
      "step": 3790
    },
    {
      "epoch": 0.2768367755800823,
      "grad_norm": 1.586433258267976,
      "learning_rate": 8.111071056538128e-07,
      "log_odds_chosen": 0.959057629108429,
      "log_odds_ratio": -0.5113281011581421,
      "logits/chosen": -0.919140636920929,
      "logits/rejected": -0.8306640386581421,
      "logps/chosen": -0.547167956829071,
      "logps/rejected": -1.0791015625,
      "loss": 0.8577,
      "nll_loss": 0.845507800579071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02736205980181694,
      "rewards/margins": 0.02663726732134819,
      "rewards/rejected": -0.05396728590130806,
      "step": 3800
    },
    {
      "epoch": 0.27756529341055625,
      "grad_norm": 1.6069113486074968,
      "learning_rate": 8.100419612604182e-07,
      "log_odds_chosen": 0.8556152582168579,
      "log_odds_ratio": -0.5101562738418579,
      "logits/chosen": -0.9585937261581421,
      "logits/rejected": -0.8431640863418579,
      "logps/chosen": -0.5565429925918579,
      "logps/rejected": -1.0693359375,
      "loss": 0.8841,
      "nll_loss": 0.8970702886581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02783203125,
      "rewards/margins": 0.02556457556784153,
      "rewards/rejected": -0.05344238132238388,
      "step": 3810
    },
    {
      "epoch": 0.27829381124103014,
      "grad_norm": 2.067634492486163,
      "learning_rate": 8.08981002113217e-07,
      "log_odds_chosen": 1.0219604969024658,
      "log_odds_ratio": -0.4540039002895355,
      "logits/chosen": -0.947460949420929,
      "logits/rejected": -0.8324218988418579,
      "logps/chosen": -0.5791991949081421,
      "logps/rejected": -1.1716797351837158,
      "loss": 0.8827,
      "nll_loss": 0.826171875,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02895507775247097,
      "rewards/margins": 0.029688263311982155,
      "rewards/rejected": -0.05863036960363388,
      "step": 3820
    },
    {
      "epoch": 0.27902232907150404,
      "grad_norm": 1.912067352597151,
      "learning_rate": 8.079242008754989e-07,
      "log_odds_chosen": 0.9559570550918579,
      "log_odds_ratio": -0.510205090045929,
      "logits/chosen": -0.961132824420929,
      "logits/rejected": -0.858593761920929,
      "logps/chosen": -0.5909179449081421,
      "logps/rejected": -1.1886718273162842,
      "loss": 0.8631,
      "nll_loss": 0.8500000238418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02955322340130806,
      "rewards/margins": 0.02988586388528347,
      "rewards/rejected": -0.05942993238568306,
      "step": 3830
    },
    {
      "epoch": 0.27975084690197793,
      "grad_norm": 1.5151159922764625,
      "learning_rate": 8.068715304598786e-07,
      "log_odds_chosen": 0.9059082269668579,
      "log_odds_ratio": -0.487548828125,
      "logits/chosen": -0.940625011920929,
      "logits/rejected": -0.8228515386581421,
      "logps/chosen": -0.5376952886581421,
      "logps/rejected": -1.044531226158142,
      "loss": 0.8953,
      "nll_loss": 0.830273449420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02686767652630806,
      "rewards/margins": 0.025313567370176315,
      "rewards/rejected": -0.05220947414636612,
      "step": 3840
    },
    {
      "epoch": 0.28047936473245183,
      "grad_norm": 1.7107742970547475,
      "learning_rate": 8.058229640253803e-07,
      "log_odds_chosen": 0.9208984375,
      "log_odds_ratio": -0.49755859375,
      "logits/chosen": -0.9107421636581421,
      "logits/rejected": -0.850390613079071,
      "logps/chosen": -0.560546875,
      "logps/rejected": -1.126562476158142,
      "loss": 0.8837,
      "nll_loss": 0.804492175579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02800292894244194,
      "rewards/margins": 0.02826538123190403,
      "rewards/rejected": -0.05628662183880806,
      "step": 3850
    },
    {
      "epoch": 0.2812078825629257,
      "grad_norm": 1.5863816282506675,
      "learning_rate": 8.047784749745631e-07,
      "log_odds_chosen": 0.9679199457168579,
      "log_odds_ratio": -0.4789062440395355,
      "logits/chosen": -0.9779297113418579,
      "logits/rejected": -0.835742175579071,
      "logps/chosen": -0.587109386920929,
      "logps/rejected": -1.177343726158142,
      "loss": 0.8723,
      "nll_loss": 0.82568359375,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02935180626809597,
      "rewards/margins": 0.02949676476418972,
      "rewards/rejected": -0.05886230617761612,
      "step": 3860
    },
    {
      "epoch": 0.2819364003933996,
      "grad_norm": 1.5458039383733937,
      "learning_rate": 8.03738036950687e-07,
      "log_odds_chosen": 0.822558581829071,
      "log_odds_ratio": -0.52490234375,
      "logits/chosen": -0.961718738079071,
      "logits/rejected": -0.8642578125,
      "logps/chosen": -0.54150390625,
      "logps/rejected": -1.034570336341858,
      "loss": 0.8995,
      "nll_loss": 0.8466796875,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.02708129957318306,
      "rewards/margins": 0.02467651292681694,
      "rewards/rejected": -0.05174560472369194,
      "step": 3870
    },
    {
      "epoch": 0.2826649182238735,
      "grad_norm": 1.7113516840730356,
      "learning_rate": 8.027016238349195e-07,
      "log_odds_chosen": 0.750048816204071,
      "log_odds_ratio": -0.535449206829071,
      "logits/chosen": -0.947265625,
      "logits/rejected": -0.8882812261581421,
      "logps/chosen": -0.5665038824081421,
      "logps/rejected": -0.981640636920929,
      "loss": 0.8701,
      "nll_loss": 0.838085949420929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02833252027630806,
      "rewards/margins": 0.02075653150677681,
      "rewards/rejected": -0.049072265625,
      "step": 3880
    },
    {
      "epoch": 0.2833934360543474,
      "grad_norm": 1.6566105426717554,
      "learning_rate": 8.016692097435824e-07,
      "log_odds_chosen": 0.8487304449081421,
      "log_odds_ratio": -0.502685546875,
      "logits/chosen": -0.937695324420929,
      "logits/rejected": -0.8382812738418579,
      "logps/chosen": -0.5693359375,
      "logps/rejected": -1.0662109851837158,
      "loss": 0.862,
      "nll_loss": 0.872265636920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.028472900390625,
      "rewards/margins": 0.024840544909238815,
      "rewards/rejected": -0.05329589918255806,
      "step": 3890
    },
    {
      "epoch": 0.2841219538848213,
      "grad_norm": 1.5063092551609207,
      "learning_rate": 8.006407690254357e-07,
      "log_odds_chosen": 0.936816394329071,
      "log_odds_ratio": -0.524609386920929,
      "logits/chosen": -0.9115234613418579,
      "logits/rejected": -0.8248046636581421,
      "logps/chosen": -0.611035168170929,
      "logps/rejected": -1.213476538658142,
      "loss": 0.8791,
      "nll_loss": 0.8876953125,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.03053588792681694,
      "rewards/margins": 0.030132293701171875,
      "rewards/rejected": -0.06067504733800888,
      "step": 3900
    },
    {
      "epoch": 0.2848504717152952,
      "grad_norm": 1.6090352782379929,
      "learning_rate": 7.996162762590016e-07,
      "log_odds_chosen": 0.865063488483429,
      "log_odds_ratio": -0.511035144329071,
      "logits/chosen": -0.91796875,
      "logits/rejected": -0.7640625238418579,
      "logps/chosen": -0.589648425579071,
      "logps/rejected": -1.1179687976837158,
      "loss": 0.87,
      "nll_loss": 0.8880859613418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02950439415872097,
      "rewards/margins": 0.026430130004882812,
      "rewards/rejected": -0.05591430515050888,
      "step": 3910
    },
    {
      "epoch": 0.28557898954576916,
      "grad_norm": 1.508894731122892,
      "learning_rate": 7.98595706249925e-07,
      "log_odds_chosen": 0.7348877191543579,
      "log_odds_ratio": -0.584277331829071,
      "logits/chosen": -0.924023449420929,
      "logits/rejected": -0.848828136920929,
      "logps/chosen": -0.597949206829071,
      "logps/rejected": -1.0576171875,
      "loss": 0.8747,
      "nll_loss": 0.888671875,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02988891676068306,
      "rewards/margins": 0.02298736572265625,
      "rewards/rejected": -0.05288086086511612,
      "step": 3920
    },
    {
      "epoch": 0.28630750737624305,
      "grad_norm": 1.5941262844688338,
      "learning_rate": 7.975790340283705e-07,
      "log_odds_chosen": 0.867236316204071,
      "log_odds_ratio": -0.514941394329071,
      "logits/chosen": -0.9248046875,
      "logits/rejected": -0.8470703363418579,
      "logps/chosen": -0.549121081829071,
      "logps/rejected": -1.093164086341858,
      "loss": 0.888,
      "nll_loss": 0.8587890863418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02742919884622097,
      "rewards/margins": 0.027190780267119408,
      "rewards/rejected": -0.05466919019818306,
      "step": 3930
    },
    {
      "epoch": 0.28703602520671695,
      "grad_norm": 1.6680287027813154,
      "learning_rate": 7.965662348464579e-07,
      "log_odds_chosen": 1.02880859375,
      "log_odds_ratio": -0.4546875059604645,
      "logits/chosen": -0.9203125238418579,
      "logits/rejected": -0.860156238079071,
      "logps/chosen": -0.53857421875,
      "logps/rejected": -1.1326172351837158,
      "loss": 0.8823,
      "nll_loss": 0.8333984613418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02692871168255806,
      "rewards/margins": 0.029759978875517845,
      "rewards/rejected": -0.05668945237994194,
      "step": 3940
    },
    {
      "epoch": 0.28776454303719085,
      "grad_norm": 1.5235572955327947,
      "learning_rate": 7.9555728417573e-07,
      "log_odds_chosen": 0.880175769329071,
      "log_odds_ratio": -0.5101562738418579,
      "logits/chosen": -0.931445300579071,
      "logits/rejected": -0.817187488079071,
      "logps/chosen": -0.56103515625,
      "logps/rejected": -1.0906250476837158,
      "loss": 0.8756,
      "nll_loss": 0.83203125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.028045654296875,
      "rewards/margins": 0.02649078331887722,
      "rewards/rejected": -0.05455322191119194,
      "step": 3950
    },
    {
      "epoch": 0.28849306086766474,
      "grad_norm": 1.4014521266260338,
      "learning_rate": 7.945521577046602e-07,
      "log_odds_chosen": 0.7816406488418579,
      "log_odds_ratio": -0.5132812261581421,
      "logits/chosen": -0.9828125238418579,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.5699218511581421,
      "logps/rejected": -1.0390625,
      "loss": 0.8648,
      "nll_loss": 0.8248046636581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02847900427877903,
      "rewards/margins": 0.02344970777630806,
      "rewards/rejected": -0.05197753757238388,
      "step": 3960
    },
    {
      "epoch": 0.28922157869813864,
      "grad_norm": 1.5100853094633087,
      "learning_rate": 7.935508313361897e-07,
      "log_odds_chosen": 0.701855480670929,
      "log_odds_ratio": -0.5594726800918579,
      "logits/chosen": -0.938281238079071,
      "logits/rejected": -0.846484363079071,
      "logps/chosen": -0.540722668170929,
      "logps/rejected": -0.9439452886581421,
      "loss": 0.8928,
      "nll_loss": 0.837695300579071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.02704467810690403,
      "rewards/margins": 0.020200729370117188,
      "rewards/rejected": -0.04724731296300888,
      "step": 3970
    },
    {
      "epoch": 0.28995009652861253,
      "grad_norm": 1.4343131376280664,
      "learning_rate": 7.925532811853019e-07,
      "log_odds_chosen": 0.754406750202179,
      "log_odds_ratio": -0.509814441204071,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.8091796636581421,
      "logps/chosen": -0.5513671636581421,
      "logps/rejected": -0.96875,
      "loss": 0.8732,
      "nll_loss": 0.790820300579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02755126915872097,
      "rewards/margins": 0.02089080773293972,
      "rewards/rejected": -0.04845581203699112,
      "step": 3980
    },
    {
      "epoch": 0.29067861435908643,
      "grad_norm": 1.4851778901384896,
      "learning_rate": 7.915594835766295e-07,
      "log_odds_chosen": 0.7605956792831421,
      "log_odds_ratio": -0.556835949420929,
      "logits/chosen": -0.9052734375,
      "logits/rejected": -0.8115234375,
      "logps/chosen": -0.578906238079071,
      "logps/rejected": -1.0662109851837158,
      "loss": 0.8575,
      "nll_loss": 0.8818359375,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02894287183880806,
      "rewards/margins": 0.02438507042825222,
      "rewards/rejected": -0.05332031100988388,
      "step": 3990
    },
    {
      "epoch": 0.2914071321895603,
      "grad_norm": 1.505646160710882,
      "learning_rate": 7.905694150420949e-07,
      "log_odds_chosen": 0.878173828125,
      "log_odds_ratio": -0.4903808534145355,
      "logits/chosen": -0.944531261920929,
      "logits/rejected": -0.8388671875,
      "logps/chosen": -0.5830078125,
      "logps/rejected": -1.076171875,
      "loss": 0.859,
      "nll_loss": 0.8580077886581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.029144287109375,
      "rewards/margins": 0.024726103991270065,
      "rewards/rejected": -0.05385742336511612,
      "step": 4000
    },
    {
      "epoch": 0.2921356500200342,
      "grad_norm": 1.3844891467237288,
      "learning_rate": 7.895830523185819e-07,
      "log_odds_chosen": 0.914257824420929,
      "log_odds_ratio": -0.48164063692092896,
      "logits/chosen": -0.9808593988418579,
      "logits/rejected": -0.8677734136581421,
      "logps/chosen": -0.5723632574081421,
      "logps/rejected": -1.114843726158142,
      "loss": 0.8726,
      "nll_loss": 0.8021484613418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02861328050494194,
      "rewards/margins": 0.02709655836224556,
      "rewards/rejected": -0.05571288987994194,
      "step": 4010
    },
    {
      "epoch": 0.2928641678505081,
      "grad_norm": 1.6015914888874005,
      "learning_rate": 7.886003723456397e-07,
      "log_odds_chosen": 0.9903808832168579,
      "log_odds_ratio": -0.47998046875,
      "logits/chosen": -0.9458984136581421,
      "logits/rejected": -0.8208984136581421,
      "logps/chosen": -0.5721679925918579,
      "logps/rejected": -1.159570336341858,
      "loss": 0.8886,
      "nll_loss": 0.9007812738418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.028594970703125,
      "rewards/margins": 0.0293426513671875,
      "rewards/rejected": -0.05792236328125,
      "step": 4020
    },
    {
      "epoch": 0.29359268568098207,
      "grad_norm": 1.383524329499732,
      "learning_rate": 7.876213522632199e-07,
      "log_odds_chosen": 0.86962890625,
      "log_odds_ratio": -0.50927734375,
      "logits/chosen": -0.935742199420929,
      "logits/rejected": -0.8451172113418579,
      "logps/chosen": -0.601757824420929,
      "logps/rejected": -1.1544921398162842,
      "loss": 0.8686,
      "nll_loss": 0.9193359613418579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03008422814309597,
      "rewards/margins": 0.02761382982134819,
      "rewards/rejected": -0.057708740234375,
      "step": 4030
    },
    {
      "epoch": 0.29432120351145596,
      "grad_norm": 1.7795944572324638,
      "learning_rate": 7.866459694094408e-07,
      "log_odds_chosen": 0.9947265386581421,
      "log_odds_ratio": -0.4690918028354645,
      "logits/chosen": -0.9615234136581421,
      "logits/rejected": -0.818164050579071,
      "logps/chosen": -0.563281238079071,
      "logps/rejected": -1.155859351158142,
      "loss": 0.8488,
      "nll_loss": 0.7939453125,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02816162072122097,
      "rewards/margins": 0.02962646447122097,
      "rewards/rejected": -0.05778808519244194,
      "step": 4040
    },
    {
      "epoch": 0.29504972134192986,
      "grad_norm": 1.9324196629369352,
      "learning_rate": 7.856742013183862e-07,
      "log_odds_chosen": 0.89794921875,
      "log_odds_ratio": -0.47929686307907104,
      "logits/chosen": -0.9898437261581421,
      "logits/rejected": -0.883007824420929,
      "logps/chosen": -0.578808605670929,
      "logps/rejected": -1.1271483898162842,
      "loss": 0.8736,
      "nll_loss": 0.802734375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02894897386431694,
      "rewards/margins": 0.02740783616900444,
      "rewards/rejected": -0.05637206882238388,
      "step": 4050
    },
    {
      "epoch": 0.29577823917240376,
      "grad_norm": 1.4813546334600336,
      "learning_rate": 7.847060257179306e-07,
      "log_odds_chosen": 0.9251464605331421,
      "log_odds_ratio": -0.48095703125,
      "logits/chosen": -0.930859386920929,
      "logits/rejected": -0.7796875238418579,
      "logps/chosen": -0.562207043170929,
      "logps/rejected": -1.0900390148162842,
      "loss": 0.8753,
      "nll_loss": 0.871289074420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02811889722943306,
      "rewards/margins": 0.026401519775390625,
      "rewards/rejected": -0.05447997897863388,
      "step": 4060
    },
    {
      "epoch": 0.29650675700287765,
      "grad_norm": 1.8218615090318722,
      "learning_rate": 7.837414205275962e-07,
      "log_odds_chosen": 0.9300781488418579,
      "log_odds_ratio": -0.49565428495407104,
      "logits/chosen": -0.9339843988418579,
      "logits/rejected": -0.839062511920929,
      "logps/chosen": -0.544238269329071,
      "logps/rejected": -1.1025390625,
      "loss": 0.8525,
      "nll_loss": 0.857617199420929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0272216796875,
      "rewards/margins": 0.02795867994427681,
      "rewards/rejected": -0.05518188327550888,
      "step": 4070
    },
    {
      "epoch": 0.29723527483335155,
      "grad_norm": 1.5892598128790665,
      "learning_rate": 7.82780363856437e-07,
      "log_odds_chosen": 0.970898449420929,
      "log_odds_ratio": -0.47773438692092896,
      "logits/chosen": -0.9369140863418579,
      "logits/rejected": -0.8167968988418579,
      "logps/chosen": -0.5635741949081421,
      "logps/rejected": -1.1494140625,
      "loss": 0.8783,
      "nll_loss": 0.867382824420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02817382849752903,
      "rewards/margins": 0.029314422979950905,
      "rewards/rejected": -0.05750732496380806,
      "step": 4080
    },
    {
      "epoch": 0.29796379266382544,
      "grad_norm": 1.6324270109940213,
      "learning_rate": 7.818228340009527e-07,
      "log_odds_chosen": 0.8548339605331421,
      "log_odds_ratio": -0.550976574420929,
      "logits/chosen": -0.9263671636581421,
      "logits/rejected": -0.8519531488418579,
      "logps/chosen": -0.605761706829071,
      "logps/rejected": -1.0822265148162842,
      "loss": 0.8875,
      "nll_loss": 0.876269519329071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.03029174730181694,
      "rewards/margins": 0.023824309930205345,
      "rewards/rejected": -0.05408325046300888,
      "step": 4090
    },
    {
      "epoch": 0.29869231049429934,
      "grad_norm": 1.4765119327599514,
      "learning_rate": 7.808688094430305e-07,
      "log_odds_chosen": 0.815234363079071,
      "log_odds_ratio": -0.505175769329071,
      "logits/chosen": -0.8960937261581421,
      "logits/rejected": -0.871289074420929,
      "logps/chosen": -0.5796874761581421,
      "logps/rejected": -1.0517578125,
      "loss": 0.8603,
      "nll_loss": 0.8011718988418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02899169921875,
      "rewards/margins": 0.02365570142865181,
      "rewards/rejected": -0.05263672024011612,
      "step": 4100
    },
    {
      "epoch": 0.29942082832477324,
      "grad_norm": 1.6753842420424772,
      "learning_rate": 7.799182688479127e-07,
      "log_odds_chosen": 0.883105456829071,
      "log_odds_ratio": -0.48271483182907104,
      "logits/chosen": -0.919140636920929,
      "logits/rejected": -0.8363281488418579,
      "logps/chosen": -0.56201171875,
      "logps/rejected": -1.0626952648162842,
      "loss": 0.8709,
      "nll_loss": 0.8677734136581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02811889722943306,
      "rewards/margins": 0.02501373365521431,
      "rewards/rejected": -0.05313720554113388,
      "step": 4110
    },
    {
      "epoch": 0.30014934615524713,
      "grad_norm": 3.0317977853027838,
      "learning_rate": 7.789711910621948e-07,
      "log_odds_chosen": 1.0588867664337158,
      "log_odds_ratio": -0.4324707090854645,
      "logits/chosen": -0.955078125,
      "logits/rejected": -0.8091796636581421,
      "logps/chosen": -0.550488293170929,
      "logps/rejected": -1.184960961341858,
      "loss": 0.8998,
      "nll_loss": 0.913281261920929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02754516527056694,
      "rewards/margins": 0.03172607347369194,
      "rewards/rejected": -0.05926513671875,
      "step": 4120
    },
    {
      "epoch": 0.30087786398572103,
      "grad_norm": 1.6822221642303052,
      "learning_rate": 7.780275551118465e-07,
      "log_odds_chosen": 1.0626952648162842,
      "log_odds_ratio": -0.451416015625,
      "logits/chosen": -0.9232422113418579,
      "logits/rejected": -0.7984374761581421,
      "logps/chosen": -0.541210949420929,
      "logps/rejected": -1.1796875,
      "loss": 0.86,
      "nll_loss": 0.839648425579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02705078199505806,
      "rewards/margins": 0.031951904296875,
      "rewards/rejected": -0.05896606296300888,
      "step": 4130
    },
    {
      "epoch": 0.3016063818161949,
      "grad_norm": 1.4373571580224653,
      "learning_rate": 7.770873402002615e-07,
      "log_odds_chosen": 0.783642590045929,
      "log_odds_ratio": -0.5445312261581421,
      "logits/chosen": -0.9146484136581421,
      "logits/rejected": -0.834179699420929,
      "logps/chosen": -0.54736328125,
      "logps/rejected": -1.0099608898162842,
      "loss": 0.8504,
      "nll_loss": 0.8687499761581421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.02734375,
      "rewards/margins": 0.023108482360839844,
      "rewards/rejected": -0.05048828199505806,
      "step": 4140
    },
    {
      "epoch": 0.3023348996466689,
      "grad_norm": 1.4990375624943861,
      "learning_rate": 7.761505257063329e-07,
      "log_odds_chosen": 0.7701781988143921,
      "log_odds_ratio": -0.5712890625,
      "logits/chosen": -0.9486328363418579,
      "logits/rejected": -0.80078125,
      "logps/chosen": -0.574999988079071,
      "logps/rejected": -1.02734375,
      "loss": 0.8665,
      "nll_loss": 0.7767578363418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02875976637005806,
      "rewards/margins": 0.022634124383330345,
      "rewards/rejected": -0.05140380933880806,
      "step": 4150
    },
    {
      "epoch": 0.30306341747714277,
      "grad_norm": 1.5526110536940048,
      "learning_rate": 7.752170911825528e-07,
      "log_odds_chosen": 0.954541027545929,
      "log_odds_ratio": -0.4803710877895355,
      "logits/chosen": -0.9107421636581421,
      "logits/rejected": -0.791210949420929,
      "logps/chosen": -0.5897461175918579,
      "logps/rejected": -1.197851538658142,
      "loss": 0.8479,
      "nll_loss": 0.8648437261581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02950439415872097,
      "rewards/margins": 0.03040161170065403,
      "rewards/rejected": -0.05988769605755806,
      "step": 4160
    },
    {
      "epoch": 0.30379193530761667,
      "grad_norm": 1.5150212855620722,
      "learning_rate": 7.742870163531387e-07,
      "log_odds_chosen": 1.2048828601837158,
      "log_odds_ratio": -0.420654296875,
      "logits/chosen": -0.9027343988418579,
      "logits/rejected": -0.8121093511581421,
      "logps/chosen": -0.505664050579071,
      "logps/rejected": -1.169531226158142,
      "loss": 0.8527,
      "nll_loss": 0.8365234136581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0252685546875,
      "rewards/margins": 0.033172607421875,
      "rewards/rejected": -0.05844726413488388,
      "step": 4170
    },
    {
      "epoch": 0.30452045313809056,
      "grad_norm": 1.56939528486701,
      "learning_rate": 7.733602811121825e-07,
      "log_odds_chosen": 0.762622058391571,
      "log_odds_ratio": -0.511523425579071,
      "logits/chosen": -0.968945324420929,
      "logits/rejected": -0.859570324420929,
      "logps/chosen": -0.5824218988418579,
      "logps/rejected": -1.0128905773162842,
      "loss": 0.8484,
      "nll_loss": 0.8197265863418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02911987341940403,
      "rewards/margins": 0.02151641808450222,
      "rewards/rejected": -0.05061035230755806,
      "step": 4180
    },
    {
      "epoch": 0.30524897096856446,
      "grad_norm": 1.4372365746764728,
      "learning_rate": 7.724368655218262e-07,
      "log_odds_chosen": 0.8341308832168579,
      "log_odds_ratio": -0.508496105670929,
      "logits/chosen": -0.890429675579071,
      "logits/rejected": -0.7876952886581421,
      "logps/chosen": -0.5677734613418579,
      "logps/rejected": -1.0730469226837158,
      "loss": 0.8536,
      "nll_loss": 0.8226562738418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02837829664349556,
      "rewards/margins": 0.025275420397520065,
      "rewards/rejected": -0.053680419921875,
      "step": 4190
    },
    {
      "epoch": 0.30597748879903836,
      "grad_norm": 1.5318885355664305,
      "learning_rate": 7.715167498104596e-07,
      "log_odds_chosen": 0.9305664300918579,
      "log_odds_ratio": -0.46699219942092896,
      "logits/chosen": -0.9302734136581421,
      "logits/rejected": -0.8349609375,
      "logps/chosen": -0.5174804925918579,
      "logps/rejected": -1.0185546875,
      "loss": 0.8482,
      "nll_loss": 0.7816406488418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02587280236184597,
      "rewards/margins": 0.02502746507525444,
      "rewards/rejected": -0.05091552808880806,
      "step": 4200
    },
    {
      "epoch": 0.30670600662951225,
      "grad_norm": 1.4823420426223046,
      "learning_rate": 7.705999143709424e-07,
      "log_odds_chosen": 0.764941394329071,
      "log_odds_ratio": -0.5531250238418579,
      "logits/chosen": -0.9105468988418579,
      "logits/rejected": -0.8158203363418579,
      "logps/chosen": -0.536425769329071,
      "logps/rejected": -0.995898425579071,
      "loss": 0.8765,
      "nll_loss": 0.8238281011581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02683715894818306,
      "rewards/margins": 0.02294311486184597,
      "rewards/rejected": -0.049774169921875,
      "step": 4210
    },
    {
      "epoch": 0.30743452445998615,
      "grad_norm": 1.485564408022588,
      "learning_rate": 7.69686339758849e-07,
      "log_odds_chosen": 0.945849597454071,
      "log_odds_ratio": -0.4710449278354645,
      "logits/chosen": -0.9654296636581421,
      "logits/rejected": -0.859375,
      "logps/chosen": -0.5249999761581421,
      "logps/rejected": -1.076171875,
      "loss": 0.8565,
      "nll_loss": 0.7943359613418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0262451171875,
      "rewards/margins": 0.02753753587603569,
      "rewards/rejected": -0.053802490234375,
      "step": 4220
    },
    {
      "epoch": 0.30816304229046004,
      "grad_norm": 1.5713843346199057,
      "learning_rate": 7.687760066907376e-07,
      "log_odds_chosen": 0.8070312738418579,
      "log_odds_ratio": -0.53759765625,
      "logits/chosen": -0.959179699420929,
      "logits/rejected": -0.869140625,
      "logps/chosen": -0.6083984375,
      "logps/rejected": -1.085546851158142,
      "loss": 0.8791,
      "nll_loss": 0.8199218511581421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03040771558880806,
      "rewards/margins": 0.02386627160012722,
      "rewards/rejected": -0.05429687350988388,
      "step": 4230
    },
    {
      "epoch": 0.30889156012093394,
      "grad_norm": 1.6358161060277328,
      "learning_rate": 7.678688960424391e-07,
      "log_odds_chosen": 1.1162109375,
      "log_odds_ratio": -0.4189453125,
      "logits/chosen": -0.873828113079071,
      "logits/rejected": -0.792773425579071,
      "logps/chosen": -0.5003906488418579,
      "logps/rejected": -1.129296898841858,
      "loss": 0.8695,
      "nll_loss": 0.8091796636581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0250091552734375,
      "rewards/margins": 0.03148193284869194,
      "rewards/rejected": -0.05646972730755806,
      "step": 4240
    },
    {
      "epoch": 0.30962007795140783,
      "grad_norm": 1.6436065324620928,
      "learning_rate": 7.669649888473705e-07,
      "log_odds_chosen": 0.8707275390625,
      "log_odds_ratio": -0.5116211175918579,
      "logits/chosen": -0.9126952886581421,
      "logits/rejected": -0.8212890625,
      "logps/chosen": -0.5467773675918579,
      "logps/rejected": -1.050195336341858,
      "loss": 0.8601,
      "nll_loss": 0.84033203125,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02734375,
      "rewards/margins": 0.02514038048684597,
      "rewards/rejected": -0.05246581882238388,
      "step": 4250
    },
    {
      "epoch": 0.3103485957818818,
      "grad_norm": 1.6763325804813973,
      "learning_rate": 7.660642662948695e-07,
      "log_odds_chosen": 0.9034789800643921,
      "log_odds_ratio": -0.494873046875,
      "logits/chosen": -0.9287109375,
      "logits/rejected": -0.8179687261581421,
      "logps/chosen": -0.553027331829071,
      "logps/rejected": -1.116796851158142,
      "loss": 0.8654,
      "nll_loss": 0.86328125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02764282189309597,
      "rewards/margins": 0.028214644640684128,
      "rewards/rejected": -0.05586547777056694,
      "step": 4260
    },
    {
      "epoch": 0.3110771136123557,
      "grad_norm": 1.6002407875760114,
      "learning_rate": 7.651667097285499e-07,
      "log_odds_chosen": 0.7691894769668579,
      "log_odds_ratio": -0.542285144329071,
      "logits/chosen": -0.89453125,
      "logits/rejected": -0.8173828125,
      "logps/chosen": -0.581835925579071,
      "logps/rejected": -1.026953101158142,
      "loss": 0.8664,
      "nll_loss": 0.8412109613418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02909545972943306,
      "rewards/margins": 0.022251129150390625,
      "rewards/rejected": -0.05135498195886612,
      "step": 4270
    },
    {
      "epoch": 0.3118056314428296,
      "grad_norm": 1.6144958923999309,
      "learning_rate": 7.642723006446789e-07,
      "log_odds_chosen": 0.8265380859375,
      "log_odds_ratio": -0.5098632574081421,
      "logits/chosen": -0.898242175579071,
      "logits/rejected": -0.8251953125,
      "logps/chosen": -0.5721679925918579,
      "logps/rejected": -1.05810546875,
      "loss": 0.8597,
      "nll_loss": 0.8355468511581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02858886681497097,
      "rewards/margins": 0.024306869134306908,
      "rewards/rejected": -0.05290527269244194,
      "step": 4280
    },
    {
      "epoch": 0.3125341492733035,
      "grad_norm": 1.7593155582757383,
      "learning_rate": 7.633810206905743e-07,
      "log_odds_chosen": 0.90960693359375,
      "log_odds_ratio": -0.49506837129592896,
      "logits/chosen": -0.921875,
      "logits/rejected": -0.8443359136581421,
      "logps/chosen": -0.564746081829071,
      "logps/rejected": -1.0779297351837158,
      "loss": 0.8589,
      "nll_loss": 0.826953113079071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02824706956744194,
      "rewards/margins": 0.025687027722597122,
      "rewards/rejected": -0.05391845852136612,
      "step": 4290
    },
    {
      "epoch": 0.31326266710377737,
      "grad_norm": 1.8410602637569518,
      "learning_rate": 7.624928516630234e-07,
      "log_odds_chosen": 0.969775378704071,
      "log_odds_ratio": -0.4906249940395355,
      "logits/chosen": -0.948437511920929,
      "logits/rejected": -0.8677734136581421,
      "logps/chosen": -0.5970703363418579,
      "logps/rejected": -1.1912109851837158,
      "loss": 0.8745,
      "nll_loss": 0.8436523675918579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02982788160443306,
      "rewards/margins": 0.0297393798828125,
      "rewards/rejected": -0.0595703125,
      "step": 4300
    },
    {
      "epoch": 0.31399118493425127,
      "grad_norm": 1.6798282057177292,
      "learning_rate": 7.616077755067217e-07,
      "log_odds_chosen": 1.020898461341858,
      "log_odds_ratio": -0.47050780057907104,
      "logits/chosen": -0.9935547113418579,
      "logits/rejected": -0.894335925579071,
      "logps/chosen": -0.5381835699081421,
      "logps/rejected": -1.097265601158142,
      "loss": 0.8679,
      "nll_loss": 0.800976574420929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02689209021627903,
      "rewards/margins": 0.028011322021484375,
      "rewards/rejected": -0.054931640625,
      "step": 4310
    },
    {
      "epoch": 0.31471970276472516,
      "grad_norm": 1.4745439465010646,
      "learning_rate": 7.607257743127307e-07,
      "log_odds_chosen": 0.8475097417831421,
      "log_odds_ratio": -0.4940429627895355,
      "logits/chosen": -0.938671886920929,
      "logits/rejected": -0.828320324420929,
      "logps/chosen": -0.5506836175918579,
      "logps/rejected": -1.0626952648162842,
      "loss": 0.868,
      "nll_loss": 0.857421875,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02753906324505806,
      "rewards/margins": 0.025665283203125,
      "rewards/rejected": -0.053192138671875,
      "step": 4320
    },
    {
      "epoch": 0.31544822059519906,
      "grad_norm": 1.9268688172750659,
      "learning_rate": 7.598468303169562e-07,
      "log_odds_chosen": 0.8641113042831421,
      "log_odds_ratio": -0.524121105670929,
      "logits/chosen": -0.888867199420929,
      "logits/rejected": -0.796875,
      "logps/chosen": -0.593945324420929,
      "logps/rejected": -1.1404297351837158,
      "loss": 0.8606,
      "nll_loss": 0.829394519329071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02968139573931694,
      "rewards/margins": 0.02732544019818306,
      "rewards/rejected": -0.05703125149011612,
      "step": 4330
    },
    {
      "epoch": 0.31617673842567295,
      "grad_norm": 1.6926604523038316,
      "learning_rate": 7.589709258986455e-07,
      "log_odds_chosen": 0.8819580078125,
      "log_odds_ratio": -0.5105956792831421,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.8349609375,
      "logps/chosen": -0.5777343511581421,
      "logps/rejected": -1.092187523841858,
      "loss": 0.8731,
      "nll_loss": 0.8599609136581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02890625037252903,
      "rewards/margins": 0.025681305676698685,
      "rewards/rejected": -0.05457763746380806,
      "step": 4340
    },
    {
      "epoch": 0.31690525625614685,
      "grad_norm": 1.4762787314874581,
      "learning_rate": 7.580980435789034e-07,
      "log_odds_chosen": 0.7232910394668579,
      "log_odds_ratio": -0.563183605670929,
      "logits/chosen": -0.9154297113418579,
      "logits/rejected": -0.843554675579071,
      "logps/chosen": -0.5677734613418579,
      "logps/rejected": -0.989062488079071,
      "loss": 0.8697,
      "nll_loss": 0.8218749761581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02836913987994194,
      "rewards/margins": 0.02106628380715847,
      "rewards/rejected": -0.04946289211511612,
      "step": 4350
    },
    {
      "epoch": 0.31763377408662075,
      "grad_norm": 1.4861042466112717,
      "learning_rate": 7.572281660192283e-07,
      "log_odds_chosen": 0.894763171672821,
      "log_odds_ratio": -0.5140625238418579,
      "logits/chosen": -0.913281261920929,
      "logits/rejected": -0.816601574420929,
      "logps/chosen": -0.576855480670929,
      "logps/rejected": -1.1064453125,
      "loss": 0.8576,
      "nll_loss": 0.8246093988418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02883300743997097,
      "rewards/margins": 0.02644500695168972,
      "rewards/rejected": -0.05527954176068306,
      "step": 4360
    },
    {
      "epoch": 0.3183622919170947,
      "grad_norm": 4.980430633809897,
      "learning_rate": 7.563612760200645e-07,
      "log_odds_chosen": 0.7821289300918579,
      "log_odds_ratio": -0.5311523675918579,
      "logits/chosen": -0.9287109375,
      "logits/rejected": -0.850390613079071,
      "logps/chosen": -0.555371105670929,
      "logps/rejected": -1.0281250476837158,
      "loss": 0.847,
      "nll_loss": 0.8031250238418579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02778930589556694,
      "rewards/margins": 0.023616790771484375,
      "rewards/rejected": -0.0513916015625,
      "step": 4370
    },
    {
      "epoch": 0.3190908097475686,
      "grad_norm": 2.2195355623949786,
      "learning_rate": 7.554973565193743e-07,
      "log_odds_chosen": 0.9759765863418579,
      "log_odds_ratio": -0.5013183355331421,
      "logits/chosen": -0.9371093511581421,
      "logits/rejected": -0.840039074420929,
      "logps/chosen": -0.6016601324081421,
      "logps/rejected": -1.1990234851837158,
      "loss": 0.8519,
      "nll_loss": 0.82421875,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.03010253980755806,
      "rewards/margins": 0.02988281287252903,
      "rewards/rejected": -0.05997314304113388,
      "step": 4380
    },
    {
      "epoch": 0.3198193275780425,
      "grad_norm": 1.4926465623043692,
      "learning_rate": 7.546363905912276e-07,
      "log_odds_chosen": 0.8623046875,
      "log_odds_ratio": -0.504345715045929,
      "logits/chosen": -0.907031238079071,
      "logits/rejected": -0.816210925579071,
      "logps/chosen": -0.5406249761581421,
      "logps/rejected": -1.054296851158142,
      "loss": 0.8595,
      "nll_loss": 0.800976574420929,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02702026441693306,
      "rewards/margins": 0.02569122239947319,
      "rewards/rejected": -0.05269775539636612,
      "step": 4390
    },
    {
      "epoch": 0.3205478454085164,
      "grad_norm": 1.589781613252637,
      "learning_rate": 7.537783614444091e-07,
      "log_odds_chosen": 1.066064476966858,
      "log_odds_ratio": -0.44697266817092896,
      "logits/chosen": -0.9222656488418579,
      "logits/rejected": -0.801953136920929,
      "logps/chosen": -0.5615234375,
      "logps/rejected": -1.201757788658142,
      "loss": 0.8566,
      "nll_loss": 0.8482421636581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02808837965130806,
      "rewards/margins": 0.03198394924402237,
      "rewards/rejected": -0.06005859375,
      "step": 4400
    },
    {
      "epoch": 0.3212763632389903,
      "grad_norm": 1.8660148110341992,
      "learning_rate": 7.529232524210427e-07,
      "log_odds_chosen": 0.7901366949081421,
      "log_odds_ratio": -0.5519043207168579,
      "logits/chosen": -0.949023425579071,
      "logits/rejected": -0.8783203363418579,
      "logps/chosen": -0.575976550579071,
      "logps/rejected": -1.0402343273162842,
      "loss": 0.8694,
      "nll_loss": 0.888476550579071,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02880859375,
      "rewards/margins": 0.023192595690488815,
      "rewards/rejected": -0.052001953125,
      "step": 4410
    },
    {
      "epoch": 0.3220048810694642,
      "grad_norm": 1.461517070589429,
      "learning_rate": 7.520710469952336e-07,
      "log_odds_chosen": 0.700024425983429,
      "log_odds_ratio": -0.551074206829071,
      "logits/chosen": -0.953320324420929,
      "logits/rejected": -0.837890625,
      "logps/chosen": -0.588574230670929,
      "logps/rejected": -1.0187499523162842,
      "loss": 0.8657,
      "nll_loss": 0.876171886920929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.02943725511431694,
      "rewards/margins": 0.0215130802243948,
      "rewards/rejected": -0.05092773586511612,
      "step": 4420
    },
    {
      "epoch": 0.3227333988999381,
      "grad_norm": 1.4549210264814458,
      "learning_rate": 7.512217287717264e-07,
      "log_odds_chosen": 0.69091796875,
      "log_odds_ratio": -0.5409179925918579,
      "logits/chosen": -0.9349609613418579,
      "logits/rejected": -0.811718761920929,
      "logps/chosen": -0.583203136920929,
      "logps/rejected": -0.9808593988418579,
      "loss": 0.8641,
      "nll_loss": 0.849609375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02915649488568306,
      "rewards/margins": 0.01985626295208931,
      "rewards/rejected": -0.04901123046875,
      "step": 4430
    },
    {
      "epoch": 0.32346191673041197,
      "grad_norm": 1.7487574626283238,
      "learning_rate": 7.503752814845804e-07,
      "log_odds_chosen": 0.856249988079071,
      "log_odds_ratio": -0.494140625,
      "logits/chosen": -0.9833984375,
      "logits/rejected": -0.850390613079071,
      "logps/chosen": -0.5713866949081421,
      "logps/rejected": -1.109375,
      "loss": 0.8378,
      "nll_loss": 0.837695300579071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02857055701315403,
      "rewards/margins": 0.02692871168255806,
      "rewards/rejected": -0.05550537258386612,
      "step": 4440
    },
    {
      "epoch": 0.32419043456088587,
      "grad_norm": 1.827494790533643,
      "learning_rate": 7.495316889958615e-07,
      "log_odds_chosen": 0.946337878704071,
      "log_odds_ratio": -0.46928709745407104,
      "logits/chosen": -0.9369140863418579,
      "logits/rejected": -0.8265625238418579,
      "logps/chosen": -0.592578113079071,
      "logps/rejected": -1.156640648841858,
      "loss": 0.8862,
      "nll_loss": 0.913281261920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02962646447122097,
      "rewards/margins": 0.02821502648293972,
      "rewards/rejected": -0.05783691257238388,
      "step": 4450
    },
    {
      "epoch": 0.32491895239135976,
      "grad_norm": 1.5963136363713497,
      "learning_rate": 7.486909352943498e-07,
      "log_odds_chosen": 1.0548584461212158,
      "log_odds_ratio": -0.4749999940395355,
      "logits/chosen": -0.917187511920929,
      "logits/rejected": -0.812695324420929,
      "logps/chosen": -0.5801757574081421,
      "logps/rejected": -1.248632788658142,
      "loss": 0.8716,
      "nll_loss": 0.830078125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02897338941693306,
      "rewards/margins": 0.03342742845416069,
      "rewards/rejected": -0.06240234524011612,
      "step": 4460
    },
    {
      "epoch": 0.32564747022183366,
      "grad_norm": 1.5364561235940872,
      "learning_rate": 7.478530044942631e-07,
      "log_odds_chosen": 0.892626941204071,
      "log_odds_ratio": -0.5245116949081421,
      "logits/chosen": -0.9351562261581421,
      "logits/rejected": -0.848437488079071,
      "logps/chosen": -0.590136706829071,
      "logps/rejected": -1.156640648841858,
      "loss": 0.8758,
      "nll_loss": 0.908007800579071,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.02949829027056694,
      "rewards/margins": 0.02834777906537056,
      "rewards/rejected": -0.05784912034869194,
      "step": 4470
    },
    {
      "epoch": 0.32637598805230755,
      "grad_norm": 1.5691605066732266,
      "learning_rate": 7.47017880833996e-07,
      "log_odds_chosen": 0.984667956829071,
      "log_odds_ratio": -0.4891601502895355,
      "logits/chosen": -0.911914050579071,
      "logits/rejected": -0.8140624761581421,
      "logps/chosen": -0.58935546875,
      "logps/rejected": -1.175390601158142,
      "loss": 0.8789,
      "nll_loss": 0.859179675579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02949829027056694,
      "rewards/margins": 0.02931060828268528,
      "rewards/rejected": -0.05881347507238388,
      "step": 4480
    },
    {
      "epoch": 0.3271045058827815,
      "grad_norm": 1.6561181119115407,
      "learning_rate": 7.461855486748755e-07,
      "log_odds_chosen": 1.0111815929412842,
      "log_odds_ratio": -0.48974609375,
      "logits/chosen": -0.943164050579071,
      "logits/rejected": -0.813671886920929,
      "logps/chosen": -0.5596679449081421,
      "logps/rejected": -1.174707055091858,
      "loss": 0.8742,
      "nll_loss": 0.837695300579071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02797851525247097,
      "rewards/margins": 0.03076782263815403,
      "rewards/rejected": -0.05872802808880806,
      "step": 4490
    },
    {
      "epoch": 0.3278330237132554,
      "grad_norm": 1.6294570217809938,
      "learning_rate": 7.4535599249993e-07,
      "log_odds_chosen": 0.764941394329071,
      "log_odds_ratio": -0.526171863079071,
      "logits/chosen": -0.896679699420929,
      "logits/rejected": -0.807421863079071,
      "logps/chosen": -0.54248046875,
      "logps/rejected": -1.0001952648162842,
      "loss": 0.8495,
      "nll_loss": 0.869140625,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02711181715130806,
      "rewards/margins": 0.02290649339556694,
      "rewards/rejected": -0.05003662034869194,
      "step": 4500
    },
    {
      "epoch": 0.3285615415437293,
      "grad_norm": 1.6504711434888506,
      "learning_rate": 7.445291969126747e-07,
      "log_odds_chosen": 0.928027331829071,
      "log_odds_ratio": -0.4603515565395355,
      "logits/chosen": -0.936328113079071,
      "logits/rejected": -0.801562488079071,
      "logps/chosen": -0.5155273675918579,
      "logps/rejected": -1.062890648841858,
      "loss": 0.8387,
      "nll_loss": 0.802539050579071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02576293982565403,
      "rewards/margins": 0.027414703741669655,
      "rewards/rejected": -0.05318603664636612,
      "step": 4510
    },
    {
      "epoch": 0.3292900593742032,
      "grad_norm": 1.5630359826279663,
      "learning_rate": 7.43705146635912e-07,
      "log_odds_chosen": 0.810546875,
      "log_odds_ratio": -0.520214855670929,
      "logits/chosen": -0.9144531488418579,
      "logits/rejected": -0.820507824420929,
      "logps/chosen": -0.5658203363418579,
      "logps/rejected": -1.041406273841858,
      "loss": 0.8752,
      "nll_loss": 0.7890625,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.028289794921875,
      "rewards/margins": 0.02376403845846653,
      "rewards/rejected": -0.05202636867761612,
      "step": 4520
    },
    {
      "epoch": 0.3300185772046771,
      "grad_norm": 1.5864770125136172,
      "learning_rate": 7.428838265105448e-07,
      "log_odds_chosen": 0.9333251714706421,
      "log_odds_ratio": -0.4690918028354645,
      "logits/chosen": -0.9105468988418579,
      "logits/rejected": -0.814453125,
      "logps/chosen": -0.55810546875,
      "logps/rejected": -1.0623047351837158,
      "loss": 0.8746,
      "nll_loss": 0.871289074420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02790527418255806,
      "rewards/margins": 0.025189971551299095,
      "rewards/rejected": -0.05308837816119194,
      "step": 4530
    },
    {
      "epoch": 0.330747095035151,
      "grad_norm": 1.6054259798012254,
      "learning_rate": 7.42065221494406e-07,
      "log_odds_chosen": 0.8402343988418579,
      "log_odds_ratio": -0.531542956829071,
      "logits/chosen": -0.938671886920929,
      "logits/rejected": -0.835742175579071,
      "logps/chosen": -0.539843738079071,
      "logps/rejected": -1.048242211341858,
      "loss": 0.8292,
      "nll_loss": 0.741406261920929,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02698974683880806,
      "rewards/margins": 0.02544555626809597,
      "rewards/rejected": -0.05239257961511612,
      "step": 4540
    },
    {
      "epoch": 0.3314756128656249,
      "grad_norm": 1.9395836907941761,
      "learning_rate": 7.412493166611012e-07,
      "log_odds_chosen": 0.863818347454071,
      "log_odds_ratio": -0.524218738079071,
      "logits/chosen": -0.890429675579071,
      "logits/rejected": -0.8082031011581421,
      "logps/chosen": -0.5572265386581421,
      "logps/rejected": -1.086328148841858,
      "loss": 0.8638,
      "nll_loss": 0.8207031488418579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02788696251809597,
      "rewards/margins": 0.026393890380859375,
      "rewards/rejected": -0.05429687350988388,
      "step": 4550
    },
    {
      "epoch": 0.3322041306960988,
      "grad_norm": 1.7787895882264908,
      "learning_rate": 7.404360971988655e-07,
      "log_odds_chosen": 0.926501452922821,
      "log_odds_ratio": -0.50244140625,
      "logits/chosen": -0.925585925579071,
      "logits/rejected": -0.841992199420929,
      "logps/chosen": -0.564160168170929,
      "logps/rejected": -1.100976586341858,
      "loss": 0.8568,
      "nll_loss": 0.8306640386581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02820434607565403,
      "rewards/margins": 0.02686767652630806,
      "rewards/rejected": -0.0550537109375,
      "step": 4560
    },
    {
      "epoch": 0.33293264852657267,
      "grad_norm": 1.744759234602506,
      "learning_rate": 7.396255484094341e-07,
      "log_odds_chosen": 0.909863293170929,
      "log_odds_ratio": -0.4913085997104645,
      "logits/chosen": -0.9166015386581421,
      "logits/rejected": -0.854687511920929,
      "logps/chosen": -0.5658203363418579,
      "logps/rejected": -1.1002929210662842,
      "loss": 0.8688,
      "nll_loss": 0.824023425579071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02831420861184597,
      "rewards/margins": 0.02672729454934597,
      "rewards/rejected": -0.05501709133386612,
      "step": 4570
    },
    {
      "epoch": 0.33366116635704657,
      "grad_norm": 1.5499655110257635,
      "learning_rate": 7.388176557069273e-07,
      "log_odds_chosen": 0.958984375,
      "log_odds_ratio": -0.472412109375,
      "logits/chosen": -0.924609363079071,
      "logits/rejected": -0.855664074420929,
      "logps/chosen": -0.543652355670929,
      "logps/rejected": -1.0703125,
      "loss": 0.8419,
      "nll_loss": 0.804882824420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02716064453125,
      "rewards/margins": 0.02637329138815403,
      "rewards/rejected": -0.05354003980755806,
      "step": 4580
    },
    {
      "epoch": 0.33438968418752046,
      "grad_norm": 1.5024466547390467,
      "learning_rate": 7.380124046167461e-07,
      "log_odds_chosen": 0.7918456792831421,
      "log_odds_ratio": -0.5198730230331421,
      "logits/chosen": -0.948046863079071,
      "logits/rejected": -0.8443359136581421,
      "logps/chosen": -0.5718749761581421,
      "logps/rejected": -1.037695288658142,
      "loss": 0.8446,
      "nll_loss": 0.824414074420929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02861938439309597,
      "rewards/margins": 0.02330169640481472,
      "rewards/rejected": -0.05186767503619194,
      "step": 4590
    },
    {
      "epoch": 0.3351182020179944,
      "grad_norm": 1.6521537733046818,
      "learning_rate": 7.372097807744858e-07,
      "log_odds_chosen": 0.88134765625,
      "log_odds_ratio": -0.45947265625,
      "logits/chosen": -0.930859386920929,
      "logits/rejected": -0.859179675579071,
      "logps/chosen": -0.53076171875,
      "logps/rejected": -1.037109375,
      "loss": 0.8603,
      "nll_loss": 0.847851574420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02653198316693306,
      "rewards/margins": 0.02533264085650444,
      "rewards/rejected": -0.0518798828125,
      "step": 4600
    },
    {
      "epoch": 0.3358467198484683,
      "grad_norm": 2.079608032658111,
      "learning_rate": 7.364097699248571e-07,
      "log_odds_chosen": 1.01953125,
      "log_odds_ratio": -0.4900878965854645,
      "logits/chosen": -0.950976550579071,
      "logits/rejected": -0.873242199420929,
      "logps/chosen": -0.5536133050918579,
      "logps/rejected": -1.180273413658142,
      "loss": 0.8454,
      "nll_loss": 0.7801758050918579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.027679443359375,
      "rewards/margins": 0.03128967434167862,
      "rewards/rejected": -0.0589599609375,
      "step": 4610
    },
    {
      "epoch": 0.3365752376789422,
      "grad_norm": 1.6788656554783654,
      "learning_rate": 7.356123579206247e-07,
      "log_odds_chosen": 0.8868774175643921,
      "log_odds_ratio": -0.494140625,
      "logits/chosen": -0.9437500238418579,
      "logits/rejected": -0.8570312261581421,
      "logps/chosen": -0.5533202886581421,
      "logps/rejected": -1.0847656726837158,
      "loss": 0.8474,
      "nll_loss": 0.841796875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02766723558306694,
      "rewards/margins": 0.026525115594267845,
      "rewards/rejected": -0.05421752855181694,
      "step": 4620
    },
    {
      "epoch": 0.3373037555094161,
      "grad_norm": 1.444704143140688,
      "learning_rate": 7.348175307215552e-07,
      "log_odds_chosen": 1.0972900390625,
      "log_odds_ratio": -0.4539550840854645,
      "logits/chosen": -0.8919922113418579,
      "logits/rejected": -0.8267577886581421,
      "logps/chosen": -0.5708984136581421,
      "logps/rejected": -1.2208983898162842,
      "loss": 0.8628,
      "nll_loss": 0.830273449420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02852783165872097,
      "rewards/margins": 0.032473374158144,
      "rewards/rejected": -0.06101074069738388,
      "step": 4630
    },
    {
      "epoch": 0.33803227333989,
      "grad_norm": 1.5977547692463485,
      "learning_rate": 7.340252743933794e-07,
      "log_odds_chosen": 1.1137206554412842,
      "log_odds_ratio": -0.4474121034145355,
      "logits/chosen": -0.914257824420929,
      "logits/rejected": -0.8294922113418579,
      "logps/chosen": -0.5916992425918579,
      "logps/rejected": -1.2531249523162842,
      "loss": 0.8725,
      "nll_loss": 0.906054675579071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02959594689309597,
      "rewards/margins": 0.03306121751666069,
      "rewards/rejected": -0.06264648586511612,
      "step": 4640
    },
    {
      "epoch": 0.3387607911703639,
      "grad_norm": 2.0687600065269502,
      "learning_rate": 7.332355751067666e-07,
      "log_odds_chosen": 0.744384765625,
      "log_odds_ratio": -0.542041003704071,
      "logits/chosen": -0.940234363079071,
      "logits/rejected": -0.8218749761581421,
      "logps/chosen": -0.5809570550918579,
      "logps/rejected": -0.9935547113418579,
      "loss": 0.8466,
      "nll_loss": 0.823437511920929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02902832068502903,
      "rewards/margins": 0.0206298828125,
      "rewards/rejected": -0.04965820163488388,
      "step": 4650
    },
    {
      "epoch": 0.3394893090008378,
      "grad_norm": 1.5175908872055206,
      "learning_rate": 7.324484191363096e-07,
      "log_odds_chosen": 1.00750732421875,
      "log_odds_ratio": -0.4970703125,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.8310546875,
      "logps/chosen": -0.5853515863418579,
      "logps/rejected": -1.190820336341858,
      "loss": 0.87,
      "nll_loss": 0.963085949420929,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.029266357421875,
      "rewards/margins": 0.0302734375,
      "rewards/rejected": -0.05955810472369194,
      "step": 4660
    },
    {
      "epoch": 0.3402178268313117,
      "grad_norm": 1.8582847189786569,
      "learning_rate": 7.316637928595242e-07,
      "log_odds_chosen": 0.816760241985321,
      "log_odds_ratio": -0.506298840045929,
      "logits/chosen": -0.9691406488418579,
      "logits/rejected": -0.854687511920929,
      "logps/chosen": -0.55859375,
      "logps/rejected": -1.0302734375,
      "loss": 0.8645,
      "nll_loss": 0.8310546875,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02791748009622097,
      "rewards/margins": 0.02361297607421875,
      "rewards/rejected": -0.05152587965130806,
      "step": 4670
    },
    {
      "epoch": 0.3409463446617856,
      "grad_norm": 1.4771161751343087,
      "learning_rate": 7.308816827558578e-07,
      "log_odds_chosen": 0.8742431402206421,
      "log_odds_ratio": -0.5262695550918579,
      "logits/chosen": -0.9302734136581421,
      "logits/rejected": -0.849609375,
      "logps/chosen": -0.6099609136581421,
      "logps/rejected": -1.1484375,
      "loss": 0.8508,
      "nll_loss": 0.824999988079071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.03048095665872097,
      "rewards/margins": 0.026980210095643997,
      "rewards/rejected": -0.05744628980755806,
      "step": 4680
    },
    {
      "epoch": 0.3416748624922595,
      "grad_norm": 2.0588767838303097,
      "learning_rate": 7.301020754057114e-07,
      "log_odds_chosen": 0.862988293170929,
      "log_odds_ratio": -0.512988269329071,
      "logits/chosen": -0.9232422113418579,
      "logits/rejected": -0.8460937738418579,
      "logps/chosen": -0.5501953363418579,
      "logps/rejected": -1.0558593273162842,
      "loss": 0.8372,
      "nll_loss": 0.7763671875,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02750244177877903,
      "rewards/margins": 0.02526397630572319,
      "rewards/rejected": -0.05278930813074112,
      "step": 4690
    },
    {
      "epoch": 0.3424033803227334,
      "grad_norm": 1.6347705889915451,
      "learning_rate": 7.293249574894729e-07,
      "log_odds_chosen": 0.74951171875,
      "log_odds_ratio": -0.526660144329071,
      "logits/chosen": -0.9341796636581421,
      "logits/rejected": -0.8392578363418579,
      "logps/chosen": -0.6119140386581421,
      "logps/rejected": -1.0617187023162842,
      "loss": 0.87,
      "nll_loss": 0.880664050579071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03058471716940403,
      "rewards/margins": 0.0225067138671875,
      "rewards/rejected": -0.05311279371380806,
      "step": 4700
    },
    {
      "epoch": 0.3431318981532073,
      "grad_norm": 1.5051225860874622,
      "learning_rate": 7.285503157865601e-07,
      "log_odds_chosen": 0.825122058391571,
      "log_odds_ratio": -0.5194336175918579,
      "logits/chosen": -0.9351562261581421,
      "logits/rejected": -0.8231445550918579,
      "logps/chosen": -0.5757812261581421,
      "logps/rejected": -1.045507788658142,
      "loss": 0.845,
      "nll_loss": 0.8861328363418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02879028394818306,
      "rewards/margins": 0.023508453741669655,
      "rewards/rejected": -0.05231323093175888,
      "step": 4710
    },
    {
      "epoch": 0.3438604159836812,
      "grad_norm": 1.4439756107230208,
      "learning_rate": 7.277781371744776e-07,
      "log_odds_chosen": 0.948925793170929,
      "log_odds_ratio": -0.4735351502895355,
      "logits/chosen": -0.962695300579071,
      "logits/rejected": -0.8677734136581421,
      "logps/chosen": -0.574023425579071,
      "logps/rejected": -1.157812476158142,
      "loss": 0.8462,
      "nll_loss": 0.724609375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02868041954934597,
      "rewards/margins": 0.02919921837747097,
      "rewards/rejected": -0.05783691257238388,
      "step": 4720
    },
    {
      "epoch": 0.3445889338141551,
      "grad_norm": 1.5671458813784138,
      "learning_rate": 7.270084086278817e-07,
      "log_odds_chosen": 0.821704089641571,
      "log_odds_ratio": -0.503222644329071,
      "logits/chosen": -0.9732421636581421,
      "logits/rejected": -0.8792968988418579,
      "logps/chosen": -0.598828136920929,
      "logps/rejected": -1.109960913658142,
      "loss": 0.8473,
      "nll_loss": 0.826171875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02994384802877903,
      "rewards/margins": 0.025568390265107155,
      "rewards/rejected": -0.05549316480755806,
      "step": 4730
    },
    {
      "epoch": 0.345317451644629,
      "grad_norm": 1.7407916528606233,
      "learning_rate": 7.262411172176586e-07,
      "log_odds_chosen": 0.83935546875,
      "log_odds_ratio": -0.5289062261581421,
      "logits/chosen": -0.883593738079071,
      "logits/rejected": -0.8333984613418579,
      "logps/chosen": -0.567089855670929,
      "logps/rejected": -1.08349609375,
      "loss": 0.8611,
      "nll_loss": 0.85546875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02837524376809597,
      "rewards/margins": 0.02579040452837944,
      "rewards/rejected": -0.05417480319738388,
      "step": 4740
    },
    {
      "epoch": 0.3460459694751029,
      "grad_norm": 1.6144655758460713,
      "learning_rate": 7.254762501100117e-07,
      "log_odds_chosen": 0.832629382610321,
      "log_odds_ratio": -0.515673816204071,
      "logits/chosen": -0.9535156488418579,
      "logits/rejected": -0.8423827886581421,
      "logps/chosen": -0.579785168170929,
      "logps/rejected": -1.078515648841858,
      "loss": 0.8366,
      "nll_loss": 0.7867187261581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02897949144244194,
      "rewards/margins": 0.02492523193359375,
      "rewards/rejected": -0.05396118015050888,
      "step": 4750
    },
    {
      "epoch": 0.3467744873055768,
      "grad_norm": 1.5922800779978243,
      "learning_rate": 7.247137945655607e-07,
      "log_odds_chosen": 1.110620141029358,
      "log_odds_ratio": -0.44794923067092896,
      "logits/chosen": -0.914843738079071,
      "logits/rejected": -0.835742175579071,
      "logps/chosen": -0.54296875,
      "logps/rejected": -1.2374999523162842,
      "loss": 0.8703,
      "nll_loss": 0.8095703125,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02716674841940403,
      "rewards/margins": 0.03465385362505913,
      "rewards/rejected": -0.06186523288488388,
      "step": 4760
    },
    {
      "epoch": 0.3475030051360507,
      "grad_norm": 1.463581332169827,
      "learning_rate": 7.23953737938449e-07,
      "log_odds_chosen": 0.8287109136581421,
      "log_odds_ratio": -0.5318359136581421,
      "logits/chosen": -0.952929675579071,
      "logits/rejected": -0.857617199420929,
      "logps/chosen": -0.5931640863418579,
      "logps/rejected": -1.0988280773162842,
      "loss": 0.8348,
      "nll_loss": 0.820507824420929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.02968139573931694,
      "rewards/margins": 0.025333404541015625,
      "rewards/rejected": -0.05494995042681694,
      "step": 4770
    },
    {
      "epoch": 0.3482315229665246,
      "grad_norm": 1.66232685083731,
      "learning_rate": 7.231960676754647e-07,
      "log_odds_chosen": 0.955859363079071,
      "log_odds_ratio": -0.4791015684604645,
      "logits/chosen": -0.952929675579071,
      "logits/rejected": -0.845507800579071,
      "logps/chosen": -0.574023425579071,
      "logps/rejected": -1.1328125,
      "loss": 0.8309,
      "nll_loss": 0.7705078125,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02871093712747097,
      "rewards/margins": 0.02792663499712944,
      "rewards/rejected": -0.05662841722369194,
      "step": 4780
    },
    {
      "epoch": 0.3489600407969985,
      "grad_norm": 1.6515254276941034,
      "learning_rate": 7.224407713151682e-07,
      "log_odds_chosen": 0.838183581829071,
      "log_odds_ratio": -0.5010741949081421,
      "logits/chosen": -0.9595702886581421,
      "logits/rejected": -0.8306640386581421,
      "logps/chosen": -0.553906261920929,
      "logps/rejected": -1.0480468273162842,
      "loss": 0.8467,
      "nll_loss": 0.8623046875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.027679443359375,
      "rewards/margins": 0.02471465989947319,
      "rewards/rejected": -0.05238037183880806,
      "step": 4790
    },
    {
      "epoch": 0.3496885586274724,
      "grad_norm": 2.3789311523393786,
      "learning_rate": 7.216878364870323e-07,
      "log_odds_chosen": 0.949414074420929,
      "log_odds_ratio": -0.510937511920929,
      "logits/chosen": -0.904492199420929,
      "logits/rejected": -0.8294922113418579,
      "logps/chosen": -0.5877929925918579,
      "logps/rejected": -1.1570312976837158,
      "loss": 0.845,
      "nll_loss": 0.841015636920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02937011793255806,
      "rewards/margins": 0.02851715125143528,
      "rewards/rejected": -0.05789794772863388,
      "step": 4800
    },
    {
      "epoch": 0.3504170764579463,
      "grad_norm": 1.6646693469948504,
      "learning_rate": 7.209372509105906e-07,
      "log_odds_chosen": 0.897167980670929,
      "log_odds_ratio": -0.483154296875,
      "logits/chosen": -0.8988281488418579,
      "logits/rejected": -0.8609374761581421,
      "logps/chosen": -0.562207043170929,
      "logps/rejected": -1.076757788658142,
      "loss": 0.8377,
      "nll_loss": 0.8216797113418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02809448167681694,
      "rewards/margins": 0.02574310265481472,
      "rewards/rejected": -0.05385742336511612,
      "step": 4810
    },
    {
      "epoch": 0.3511455942884202,
      "grad_norm": 1.7191984745428253,
      "learning_rate": 7.201890023945968e-07,
      "log_odds_chosen": 0.7942870855331421,
      "log_odds_ratio": -0.5576171875,
      "logits/chosen": -0.957812488079071,
      "logits/rejected": -0.867382824420929,
      "logps/chosen": -0.595019519329071,
      "logps/rejected": -1.106054663658142,
      "loss": 0.8626,
      "nll_loss": 0.8226562738418579,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.02976074256002903,
      "rewards/margins": 0.025522232055664062,
      "rewards/rejected": -0.05532226711511612,
      "step": 4820
    },
    {
      "epoch": 0.35187411211889413,
      "grad_norm": 1.638023961776726,
      "learning_rate": 7.194430788361928e-07,
      "log_odds_chosen": 0.93505859375,
      "log_odds_ratio": -0.4844726622104645,
      "logits/chosen": -0.95703125,
      "logits/rejected": -0.8626953363418579,
      "logps/chosen": -0.5677734613418579,
      "logps/rejected": -1.1101562976837158,
      "loss": 0.8437,
      "nll_loss": 0.7900390625,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02839355543255806,
      "rewards/margins": 0.0270843505859375,
      "rewards/rejected": -0.05552978441119194,
      "step": 4830
    },
    {
      "epoch": 0.35260262994936803,
      "grad_norm": 1.70620681923307,
      "learning_rate": 7.186994682200863e-07,
      "log_odds_chosen": 0.984375,
      "log_odds_ratio": -0.4632324278354645,
      "logits/chosen": -0.9876953363418579,
      "logits/rejected": -0.8558593988418579,
      "logps/chosen": -0.5591796636581421,
      "logps/rejected": -1.133203148841858,
      "loss": 0.8577,
      "nll_loss": 0.7943359613418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02792968787252903,
      "rewards/margins": 0.02871093712747097,
      "rewards/rejected": -0.05665893480181694,
      "step": 4840
    },
    {
      "epoch": 0.3533311477798419,
      "grad_norm": 1.7738376756849115,
      "learning_rate": 7.179581586177383e-07,
      "log_odds_chosen": 0.704052746295929,
      "log_odds_ratio": -0.57275390625,
      "logits/chosen": -0.943554699420929,
      "logits/rejected": -0.853710949420929,
      "logps/chosen": -0.604296863079071,
      "logps/rejected": -1.0548827648162842,
      "loss": 0.8807,
      "nll_loss": 0.848828136920929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.03021240234375,
      "rewards/margins": 0.02249450609087944,
      "rewards/rejected": -0.05274047702550888,
      "step": 4850
    },
    {
      "epoch": 0.3540596656103158,
      "grad_norm": 2.0180707977527748,
      "learning_rate": 7.172191381865586e-07,
      "log_odds_chosen": 0.80078125,
      "log_odds_ratio": -0.514453113079071,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.821484386920929,
      "logps/chosen": -0.543652355670929,
      "logps/rejected": -1.037500023841858,
      "loss": 0.8576,
      "nll_loss": 0.817578136920929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02719726599752903,
      "rewards/margins": 0.02463684044778347,
      "rewards/rejected": -0.05180663987994194,
      "step": 4860
    },
    {
      "epoch": 0.3547881834407897,
      "grad_norm": 1.5001032747203953,
      "learning_rate": 7.16482395169113e-07,
      "log_odds_chosen": 0.8203369379043579,
      "log_odds_ratio": -0.49946290254592896,
      "logits/chosen": -0.964062511920929,
      "logits/rejected": -0.838085949420929,
      "logps/chosen": -0.5453125238418579,
      "logps/rejected": -0.998828113079071,
      "loss": 0.8536,
      "nll_loss": 0.817187488079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.027252197265625,
      "rewards/margins": 0.02269744873046875,
      "rewards/rejected": -0.04996337741613388,
      "step": 4870
    },
    {
      "epoch": 0.3555167012712636,
      "grad_norm": 1.682707208629792,
      "learning_rate": 7.157479178923353e-07,
      "log_odds_chosen": 0.7389160394668579,
      "log_odds_ratio": -0.557421863079071,
      "logits/chosen": -0.9371093511581421,
      "logits/rejected": -0.8306640386581421,
      "logps/chosen": -0.583691418170929,
      "logps/rejected": -1.0349609851837158,
      "loss": 0.849,
      "nll_loss": 0.800976574420929,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02915649488568306,
      "rewards/margins": 0.022552490234375,
      "rewards/rejected": -0.05174560472369194,
      "step": 4880
    },
    {
      "epoch": 0.3562452191017375,
      "grad_norm": 1.6634438433203609,
      "learning_rate": 7.150156947667522e-07,
      "log_odds_chosen": 0.829882800579071,
      "log_odds_ratio": -0.49687498807907104,
      "logits/chosen": -0.884765625,
      "logits/rejected": -0.7972656488418579,
      "logps/chosen": -0.5796874761581421,
      "logps/rejected": -1.0832030773162842,
      "loss": 0.8489,
      "nll_loss": 0.8662109375,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02898559533059597,
      "rewards/margins": 0.02519531175494194,
      "rewards/rejected": -0.05420532077550888,
      "step": 4890
    },
    {
      "epoch": 0.3569737369322114,
      "grad_norm": 1.7322728097614415,
      "learning_rate": 7.142857142857143e-07,
      "log_odds_chosen": 1.065759301185608,
      "log_odds_ratio": -0.4556640684604645,
      "logits/chosen": -0.9576171636581421,
      "logits/rejected": -0.8236328363418579,
      "logps/chosen": -0.5269531011581421,
      "logps/rejected": -1.1369140148162842,
      "loss": 0.8387,
      "nll_loss": 0.797656238079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02631836012005806,
      "rewards/margins": 0.03048553504049778,
      "rewards/rejected": -0.05682373046875,
      "step": 4900
    },
    {
      "epoch": 0.3577022547626853,
      "grad_norm": 1.745233948403049,
      "learning_rate": 7.135579650246376e-07,
      "log_odds_chosen": 1.025781273841858,
      "log_odds_ratio": -0.48334962129592896,
      "logits/chosen": -0.9595702886581421,
      "logits/rejected": -0.8451172113418579,
      "logps/chosen": -0.5350586175918579,
      "logps/rejected": -1.14453125,
      "loss": 0.8431,
      "nll_loss": 0.814453125,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02675170823931694,
      "rewards/margins": 0.03049468994140625,
      "rewards/rejected": -0.05722656100988388,
      "step": 4910
    },
    {
      "epoch": 0.3584307725931592,
      "grad_norm": 1.785573103488189,
      "learning_rate": 7.128324356402513e-07,
      "log_odds_chosen": 0.82275390625,
      "log_odds_ratio": -0.5201171636581421,
      "logits/chosen": -0.883984386920929,
      "logits/rejected": -0.824414074420929,
      "logps/chosen": -0.5614258050918579,
      "logps/rejected": -1.0291016101837158,
      "loss": 0.8466,
      "nll_loss": 0.864062488079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02809448167681694,
      "rewards/margins": 0.02335205115377903,
      "rewards/rejected": -0.05145873874425888,
      "step": 4920
    },
    {
      "epoch": 0.3591592904236331,
      "grad_norm": 1.561796687915023,
      "learning_rate": 7.121091148698564e-07,
      "log_odds_chosen": 0.7694091796875,
      "log_odds_ratio": -0.527636706829071,
      "logits/chosen": -0.9378906488418579,
      "logits/rejected": -0.815625011920929,
      "logps/chosen": -0.582714855670929,
      "logps/rejected": -1.0564453601837158,
      "loss": 0.8579,
      "nll_loss": 0.8548828363418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02911376953125,
      "rewards/margins": 0.02373657189309597,
      "rewards/rejected": -0.05286254733800888,
      "step": 4930
    },
    {
      "epoch": 0.35988780825410704,
      "grad_norm": 1.5602611934027426,
      "learning_rate": 7.113879915305904e-07,
      "log_odds_chosen": 0.7474365234375,
      "log_odds_ratio": -0.5313476324081421,
      "logits/chosen": -0.919140636920929,
      "logits/rejected": -0.8140624761581421,
      "logps/chosen": -0.5693359375,
      "logps/rejected": -1.0266602039337158,
      "loss": 0.8495,
      "nll_loss": 0.8091796636581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02849731408059597,
      "rewards/margins": 0.02286987379193306,
      "rewards/rejected": -0.05134887620806694,
      "step": 4940
    },
    {
      "epoch": 0.36061632608458094,
      "grad_norm": 1.8635599462753376,
      "learning_rate": 7.106690545187016e-07,
      "log_odds_chosen": 1.0330078601837158,
      "log_odds_ratio": -0.47529298067092896,
      "logits/chosen": -0.9710937738418579,
      "logits/rejected": -0.8505859375,
      "logps/chosen": -0.53759765625,
      "logps/rejected": -1.1472656726837158,
      "loss": 0.8431,
      "nll_loss": 0.857128918170929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.026885986328125,
      "rewards/margins": 0.03044738806784153,
      "rewards/rejected": -0.05734863132238388,
      "step": 4950
    },
    {
      "epoch": 0.36134484391505484,
      "grad_norm": 1.4597138880187241,
      "learning_rate": 7.09952292808831e-07,
      "log_odds_chosen": 1.03125,
      "log_odds_ratio": -0.47968751192092896,
      "logits/chosen": -0.9388672113418579,
      "logits/rejected": -0.8316406011581421,
      "logps/chosen": -0.549609363079071,
      "logps/rejected": -1.1533203125,
      "loss": 0.8324,
      "nll_loss": 0.820507824420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02744751051068306,
      "rewards/margins": 0.03027801588177681,
      "rewards/rejected": -0.05766601487994194,
      "step": 4960
    },
    {
      "epoch": 0.36207336174552873,
      "grad_norm": 1.7839031063208803,
      "learning_rate": 7.092376954533026e-07,
      "log_odds_chosen": 0.8451172113418579,
      "log_odds_ratio": -0.4913085997104645,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.814648449420929,
      "logps/chosen": -0.576367199420929,
      "logps/rejected": -1.068359375,
      "loss": 0.8607,
      "nll_loss": 0.9150390625,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02882080152630806,
      "rewards/margins": 0.02458343468606472,
      "rewards/rejected": -0.05341796949505806,
      "step": 4970
    },
    {
      "epoch": 0.36280187957600263,
      "grad_norm": 1.5576924179847582,
      "learning_rate": 7.085252515814198e-07,
      "log_odds_chosen": 0.9512695074081421,
      "log_odds_ratio": -0.4912109375,
      "logits/chosen": -0.954296886920929,
      "logits/rejected": -0.8609374761581421,
      "logps/chosen": -0.560742199420929,
      "logps/rejected": -1.1267578601837158,
      "loss": 0.8471,
      "nll_loss": 0.8199218511581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02807006798684597,
      "rewards/margins": 0.02827758714556694,
      "rewards/rejected": -0.05632324144244194,
      "step": 4980
    },
    {
      "epoch": 0.3635303974064765,
      "grad_norm": 1.8775344127244156,
      "learning_rate": 7.07814950398772e-07,
      "log_odds_chosen": 0.771289050579071,
      "log_odds_ratio": -0.52783203125,
      "logits/chosen": -0.931445300579071,
      "logits/rejected": -0.8404296636581421,
      "logps/chosen": -0.5941406488418579,
      "logps/rejected": -1.069726586341858,
      "loss": 0.8557,
      "nll_loss": 0.819531261920929,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02968139573931694,
      "rewards/margins": 0.0237884521484375,
      "rewards/rejected": -0.05347289890050888,
      "step": 4990
    },
    {
      "epoch": 0.3642589152369504,
      "grad_norm": 1.7271174213539158,
      "learning_rate": 7.071067811865476e-07,
      "log_odds_chosen": 0.7044922113418579,
      "log_odds_ratio": -0.549609363079071,
      "logits/chosen": -0.9183593988418579,
      "logits/rejected": -0.8414062261581421,
      "logps/chosen": -0.595898449420929,
      "logps/rejected": -1.0087890625,
      "loss": 0.837,
      "nll_loss": 0.874218761920929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02976684644818306,
      "rewards/margins": 0.02067260816693306,
      "rewards/rejected": -0.05043945461511612,
      "step": 5000
    },
    {
      "epoch": 0.3649874330674243,
      "grad_norm": 1.5840355282168619,
      "learning_rate": 7.06400733300854e-07,
      "log_odds_chosen": 0.8137451410293579,
      "log_odds_ratio": -0.48710936307907104,
      "logits/chosen": -0.942187488079071,
      "logits/rejected": -0.871289074420929,
      "logps/chosen": -0.5712890625,
      "logps/rejected": -1.025976538658142,
      "loss": 0.8435,
      "nll_loss": 0.790332019329071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02857055701315403,
      "rewards/margins": 0.022705841809511185,
      "rewards/rejected": -0.05124511569738388,
      "step": 5010
    },
    {
      "epoch": 0.3657159508978982,
      "grad_norm": 1.7693498039145843,
      "learning_rate": 7.056967961720458e-07,
      "log_odds_chosen": 0.807690441608429,
      "log_odds_ratio": -0.512988269329071,
      "logits/chosen": -0.9087890386581421,
      "logits/rejected": -0.840039074420929,
      "logps/chosen": -0.5367187261581421,
      "logps/rejected": -1.001367211341858,
      "loss": 0.8444,
      "nll_loss": 0.788281261920929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02683105506002903,
      "rewards/margins": 0.02321338653564453,
      "rewards/rejected": -0.05002441257238388,
      "step": 5020
    },
    {
      "epoch": 0.3664444687283721,
      "grad_norm": 2.5420465065427083,
      "learning_rate": 7.049949593040614e-07,
      "log_odds_chosen": 0.98388671875,
      "log_odds_ratio": -0.4896484315395355,
      "logits/chosen": -0.9585937261581421,
      "logits/rejected": -0.866992175579071,
      "logps/chosen": -0.558398425579071,
      "logps/rejected": -1.120703101158142,
      "loss": 0.8195,
      "nll_loss": 0.7607421875,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02791137620806694,
      "rewards/margins": 0.028110504150390625,
      "rewards/rejected": -0.05606689304113388,
      "step": 5030
    },
    {
      "epoch": 0.367172986558846,
      "grad_norm": 2.2035140721097255,
      "learning_rate": 7.042952122737638e-07,
      "log_odds_chosen": 1.0255858898162842,
      "log_odds_ratio": -0.4391113221645355,
      "logits/chosen": -0.956835925579071,
      "logits/rejected": -0.8330078125,
      "logps/chosen": -0.536816418170929,
      "logps/rejected": -1.130273461341858,
      "loss": 0.825,
      "nll_loss": 0.848437488079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02683715894818306,
      "rewards/margins": 0.02966003492474556,
      "rewards/rejected": -0.05646972730755806,
      "step": 5040
    },
    {
      "epoch": 0.36790150438931996,
      "grad_norm": 1.6448945234507004,
      "learning_rate": 7.035975447302919e-07,
      "log_odds_chosen": 1.0900390148162842,
      "log_odds_ratio": -0.43486326932907104,
      "logits/chosen": -0.9369140863418579,
      "logits/rejected": -0.8287109136581421,
      "logps/chosen": -0.515820324420929,
      "logps/rejected": -1.1164062023162842,
      "loss": 0.8497,
      "nll_loss": 0.738085925579071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.02579956129193306,
      "rewards/margins": 0.029994964599609375,
      "rewards/rejected": -0.05577392503619194,
      "step": 5050
    },
    {
      "epoch": 0.36863002221979385,
      "grad_norm": 1.8815218168748673,
      "learning_rate": 7.029019463944166e-07,
      "log_odds_chosen": 0.926464855670929,
      "log_odds_ratio": -0.48540037870407104,
      "logits/chosen": -0.963085949420929,
      "logits/rejected": -0.84375,
      "logps/chosen": -0.56591796875,
      "logps/rejected": -1.112890601158142,
      "loss": 0.8574,
      "nll_loss": 0.8462890386581421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.028289794921875,
      "rewards/margins": 0.02727050706744194,
      "rewards/rejected": -0.05556030198931694,
      "step": 5060
    },
    {
      "epoch": 0.36935854005026775,
      "grad_norm": 1.8291507057475975,
      "learning_rate": 7.022084070579053e-07,
      "log_odds_chosen": 1.086523413658142,
      "log_odds_ratio": -0.45917969942092896,
      "logits/chosen": -0.951953113079071,
      "logits/rejected": -0.8353515863418579,
      "logps/chosen": -0.5762695074081421,
      "logps/rejected": -1.2355468273162842,
      "loss": 0.8353,
      "nll_loss": 0.846875011920929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.028839111328125,
      "rewards/margins": 0.032967377454042435,
      "rewards/rejected": -0.06185302883386612,
      "step": 5070
    },
    {
      "epoch": 0.37008705788074164,
      "grad_norm": 1.9785403807984434,
      "learning_rate": 7.015169165828922e-07,
      "log_odds_chosen": 0.8519531488418579,
      "log_odds_ratio": -0.501904308795929,
      "logits/chosen": -0.9326171875,
      "logits/rejected": -0.8335937261581421,
      "logps/chosen": -0.563183605670929,
      "logps/rejected": -1.0576171875,
      "loss": 0.8511,
      "nll_loss": 0.8492187261581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02813720703125,
      "rewards/margins": 0.02472534216940403,
      "rewards/rejected": -0.05292968824505806,
      "step": 5080
    },
    {
      "epoch": 0.37081557571121554,
      "grad_norm": 1.831540592292387,
      "learning_rate": 7.008274649012563e-07,
      "log_odds_chosen": 0.925830066204071,
      "log_odds_ratio": -0.4844726622104645,
      "logits/chosen": -0.959179699420929,
      "logits/rejected": -0.871289074420929,
      "logps/chosen": -0.562304675579071,
      "logps/rejected": -1.0859375,
      "loss": 0.8624,
      "nll_loss": 0.805468738079071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02808227576315403,
      "rewards/margins": 0.02620086632668972,
      "rewards/rejected": -0.05428466945886612,
      "step": 5090
    },
    {
      "epoch": 0.37154409354168944,
      "grad_norm": 1.8002826988102936,
      "learning_rate": 7.001400420140049e-07,
      "log_odds_chosen": 0.862597644329071,
      "log_odds_ratio": -0.511523425579071,
      "logits/chosen": -0.899609386920929,
      "logits/rejected": -0.8353515863418579,
      "logps/chosen": -0.573925793170929,
      "logps/rejected": -1.0888671875,
      "loss": 0.8396,
      "nll_loss": 0.824999988079071,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0286865234375,
      "rewards/margins": 0.02574462816119194,
      "rewards/rejected": -0.05442504957318306,
      "step": 5100
    },
    {
      "epoch": 0.37227261137216333,
      "grad_norm": 1.8324987957656795,
      "learning_rate": 6.994546379906659e-07,
      "log_odds_chosen": 0.999462902545929,
      "log_odds_ratio": -0.465087890625,
      "logits/chosen": -0.950976550579071,
      "logits/rejected": -0.8257812261581421,
      "logps/chosen": -0.5274413824081421,
      "logps/rejected": -1.1085937023162842,
      "loss": 0.8623,
      "nll_loss": 0.813671886920929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0263824462890625,
      "rewards/margins": 0.029083251953125,
      "rewards/rejected": -0.05540771409869194,
      "step": 5110
    },
    {
      "epoch": 0.3730011292026372,
      "grad_norm": 1.6074509578263498,
      "learning_rate": 6.987712429686844e-07,
      "log_odds_chosen": 0.712890625,
      "log_odds_ratio": -0.52392578125,
      "logits/chosen": -0.936718761920929,
      "logits/rejected": -0.817187488079071,
      "logps/chosen": -0.593457043170929,
      "logps/rejected": -1.0070312023162842,
      "loss": 0.8492,
      "nll_loss": 0.8017578125,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02966918982565403,
      "rewards/margins": 0.020656585693359375,
      "rewards/rejected": -0.05031738430261612,
      "step": 5120
    },
    {
      "epoch": 0.3737296470331111,
      "grad_norm": 1.5460466596043156,
      "learning_rate": 6.98089847152826e-07,
      "log_odds_chosen": 0.8526366949081421,
      "log_odds_ratio": -0.5083984136581421,
      "logits/chosen": -0.9779297113418579,
      "logits/rejected": -0.869921863079071,
      "logps/chosen": -0.5386718511581421,
      "logps/rejected": -1.0382812023162842,
      "loss": 0.8375,
      "nll_loss": 0.8013671636581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02693481370806694,
      "rewards/margins": 0.02500000037252903,
      "rewards/rejected": -0.051910400390625,
      "step": 5130
    },
    {
      "epoch": 0.374458164863585,
      "grad_norm": 1.6484708400984025,
      "learning_rate": 6.974104408145884e-07,
      "log_odds_chosen": 1.236914038658142,
      "log_odds_ratio": -0.39252930879592896,
      "logits/chosen": -0.977343738079071,
      "logits/rejected": -0.8353515863418579,
      "logps/chosen": -0.5375000238418579,
      "logps/rejected": -1.302343726158142,
      "loss": 0.8342,
      "nll_loss": 0.775585949420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02687377855181694,
      "rewards/margins": 0.0382232666015625,
      "rewards/rejected": -0.06510009616613388,
      "step": 5140
    },
    {
      "epoch": 0.3751866826940589,
      "grad_norm": 1.7955943098250744,
      "learning_rate": 6.967330142916177e-07,
      "log_odds_chosen": 1.0591919422149658,
      "log_odds_ratio": -0.44287109375,
      "logits/chosen": -0.938281238079071,
      "logits/rejected": -0.8291015625,
      "logps/chosen": -0.5404297113418579,
      "logps/rejected": -1.1697266101837158,
      "loss": 0.8339,
      "nll_loss": 0.7865234613418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02702026441693306,
      "rewards/margins": 0.031490325927734375,
      "rewards/rejected": -0.05850829929113388,
      "step": 5150
    },
    {
      "epoch": 0.3759152005245328,
      "grad_norm": 1.8125680964952884,
      "learning_rate": 6.960575579871308e-07,
      "log_odds_chosen": 0.797167956829071,
      "log_odds_ratio": -0.527636706829071,
      "logits/chosen": -0.997265636920929,
      "logits/rejected": -0.852343738079071,
      "logps/chosen": -0.5946289300918579,
      "logps/rejected": -1.0568358898162842,
      "loss": 0.853,
      "nll_loss": 0.8121093511581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02971191331744194,
      "rewards/margins": 0.02309570275247097,
      "rewards/rejected": -0.05284423753619194,
      "step": 5160
    },
    {
      "epoch": 0.37664371835500676,
      "grad_norm": 1.771672488578379,
      "learning_rate": 6.953840623693443e-07,
      "log_odds_chosen": 0.9439452886581421,
      "log_odds_ratio": -0.4859375059604645,
      "logits/chosen": -0.939257800579071,
      "logits/rejected": -0.840624988079071,
      "logps/chosen": -0.520800769329071,
      "logps/rejected": -1.0333983898162842,
      "loss": 0.8481,
      "nll_loss": 0.766308605670929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.026031494140625,
      "rewards/margins": 0.02563934400677681,
      "rewards/rejected": -0.05166015774011612,
      "step": 5170
    },
    {
      "epoch": 0.37737223618548066,
      "grad_norm": 1.750137449169017,
      "learning_rate": 6.947125179709106e-07,
      "log_odds_chosen": 0.7857910394668579,
      "log_odds_ratio": -0.5127929449081421,
      "logits/chosen": -0.96875,
      "logits/rejected": -0.839648425579071,
      "logps/chosen": -0.572558581829071,
      "logps/rejected": -1.0558593273162842,
      "loss": 0.8524,
      "nll_loss": 0.8138672113418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.02861938439309597,
      "rewards/margins": 0.02417144738137722,
      "rewards/rejected": -0.05284423753619194,
      "step": 5180
    },
    {
      "epoch": 0.37810075401595455,
      "grad_norm": 1.7196424304969768,
      "learning_rate": 6.940429153883575e-07,
      "log_odds_chosen": 0.9426513910293579,
      "log_odds_ratio": -0.48808592557907104,
      "logits/chosen": -0.9501953125,
      "logits/rejected": -0.842968761920929,
      "logps/chosen": -0.595996081829071,
      "logps/rejected": -1.16015625,
      "loss": 0.8491,
      "nll_loss": 0.8255859613418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02983398362994194,
      "rewards/margins": 0.02822570875287056,
      "rewards/rejected": -0.05800781399011612,
      "step": 5190
    },
    {
      "epoch": 0.37882927184642845,
      "grad_norm": 1.5644032585947727,
      "learning_rate": 6.933752452815365e-07,
      "log_odds_chosen": 1.071923851966858,
      "log_odds_ratio": -0.45478516817092896,
      "logits/chosen": -0.9593750238418579,
      "logits/rejected": -0.8414062261581421,
      "logps/chosen": -0.5892578363418579,
      "logps/rejected": -1.259374976158142,
      "loss": 0.8541,
      "nll_loss": 0.7220703363418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02947387658059597,
      "rewards/margins": 0.03343810886144638,
      "rewards/rejected": -0.06292724609375,
      "step": 5200
    },
    {
      "epoch": 0.37955778967690235,
      "grad_norm": 1.890656238165831,
      "learning_rate": 6.927094983730745e-07,
      "log_odds_chosen": 1.0827147960662842,
      "log_odds_ratio": -0.45087891817092896,
      "logits/chosen": -0.970898449420929,
      "logits/rejected": -0.8671875,
      "logps/chosen": -0.51904296875,
      "logps/rejected": -1.1291015148162842,
      "loss": 0.8472,
      "nll_loss": 0.775585949420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02597656287252903,
      "rewards/margins": 0.03047485277056694,
      "rewards/rejected": -0.05644531175494194,
      "step": 5210
    },
    {
      "epoch": 0.38028630750737624,
      "grad_norm": 1.5388507731145586,
      "learning_rate": 6.920456654478332e-07,
      "log_odds_chosen": 0.823803722858429,
      "log_odds_ratio": -0.53076171875,
      "logits/chosen": -0.946484386920929,
      "logits/rejected": -0.8345702886581421,
      "logps/chosen": -0.579296886920929,
      "logps/rejected": -1.0730469226837158,
      "loss": 0.8307,
      "nll_loss": 0.837207019329071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02897338941693306,
      "rewards/margins": 0.024669265374541283,
      "rewards/rejected": -0.05363769456744194,
      "step": 5220
    },
    {
      "epoch": 0.38101482533785014,
      "grad_norm": 1.716182866048545,
      "learning_rate": 6.913837373523727e-07,
      "log_odds_chosen": 0.861572265625,
      "log_odds_ratio": -0.4849609434604645,
      "logits/chosen": -0.936328113079071,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.534375011920929,
      "logps/rejected": -0.969531238079071,
      "loss": 0.8357,
      "nll_loss": 0.8154296875,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02671508863568306,
      "rewards/margins": 0.02173767052590847,
      "rewards/rejected": -0.04843749850988388,
      "step": 5230
    },
    {
      "epoch": 0.38174334316832403,
      "grad_norm": 1.5279316983277995,
      "learning_rate": 6.907237049944221e-07,
      "log_odds_chosen": 0.9026855230331421,
      "log_odds_ratio": -0.49189454317092896,
      "logits/chosen": -0.921093761920929,
      "logits/rejected": -0.8091796636581421,
      "logps/chosen": -0.554003894329071,
      "logps/rejected": -1.055078148841858,
      "loss": 0.8339,
      "nll_loss": 0.787792980670929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.0277099609375,
      "rewards/margins": 0.02503509446978569,
      "rewards/rejected": -0.05269775539636612,
      "step": 5240
    },
    {
      "epoch": 0.38247186099879793,
      "grad_norm": 1.6917140546351412,
      "learning_rate": 6.900655593423543e-07,
      "log_odds_chosen": 1.0543944835662842,
      "log_odds_ratio": -0.48173826932907104,
      "logits/chosen": -0.9507812261581421,
      "logits/rejected": -0.853320300579071,
      "logps/chosen": -0.564208984375,
      "logps/rejected": -1.179101586341858,
      "loss": 0.8286,
      "nll_loss": 0.791699230670929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0282135009765625,
      "rewards/margins": 0.03074340894818306,
      "rewards/rejected": -0.05897216871380806,
      "step": 5250
    },
    {
      "epoch": 0.3832003788292718,
      "grad_norm": 1.8768548743438875,
      "learning_rate": 6.894092914246672e-07,
      "log_odds_chosen": 1.017968773841858,
      "log_odds_ratio": -0.44853514432907104,
      "logits/chosen": -0.941210925579071,
      "logits/rejected": -0.8384765386581421,
      "logps/chosen": -0.54443359375,
      "logps/rejected": -1.134765625,
      "loss": 0.8544,
      "nll_loss": 0.794726550579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02719726599752903,
      "rewards/margins": 0.029535675421357155,
      "rewards/rejected": -0.05675048753619194,
      "step": 5260
    },
    {
      "epoch": 0.3839288966597457,
      "grad_norm": 1.816165199810762,
      "learning_rate": 6.887548923294701e-07,
      "log_odds_chosen": 1.007470726966858,
      "log_odds_ratio": -0.46142578125,
      "logits/chosen": -0.9185546636581421,
      "logits/rejected": -0.8509765863418579,
      "logps/chosen": -0.5230468511581421,
      "logps/rejected": -1.1091797351837158,
      "loss": 0.8391,
      "nll_loss": 0.7822265625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02618408203125,
      "rewards/margins": 0.0293426513671875,
      "rewards/rejected": -0.05552978441119194,
      "step": 5270
    },
    {
      "epoch": 0.3846574144902197,
      "grad_norm": 1.6289474452338617,
      "learning_rate": 6.881023532039755e-07,
      "log_odds_chosen": 0.733593761920929,
      "log_odds_ratio": -0.5384765863418579,
      "logits/chosen": -0.9169921875,
      "logits/rejected": -0.8423827886581421,
      "logps/chosen": -0.540332019329071,
      "logps/rejected": -0.9564453363418579,
      "loss": 0.838,
      "nll_loss": 0.817089855670929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02701416052877903,
      "rewards/margins": 0.020807648077607155,
      "rewards/rejected": -0.04783935472369194,
      "step": 5280
    },
    {
      "epoch": 0.38538593232069357,
      "grad_norm": 1.717323723534317,
      "learning_rate": 6.874516652539956e-07,
      "log_odds_chosen": 0.752270519733429,
      "log_odds_ratio": -0.550097644329071,
      "logits/chosen": -0.9193359613418579,
      "logits/rejected": -0.8345702886581421,
      "logps/chosen": -0.6146484613418579,
      "logps/rejected": -1.076757788658142,
      "loss": 0.8464,
      "nll_loss": 0.888867199420929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03071289137005806,
      "rewards/margins": 0.023075485602021217,
      "rewards/rejected": -0.05379638820886612,
      "step": 5290
    },
    {
      "epoch": 0.38611445015116747,
      "grad_norm": 1.7874048834058416,
      "learning_rate": 6.868028197434453e-07,
      "log_odds_chosen": 0.8211914300918579,
      "log_odds_ratio": -0.5,
      "logits/chosen": -0.927929699420929,
      "logits/rejected": -0.8544921875,
      "logps/chosen": -0.60205078125,
      "logps/rejected": -1.0968749523162842,
      "loss": 0.8468,
      "nll_loss": 0.883496105670929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.03008422814309597,
      "rewards/margins": 0.024718474596738815,
      "rewards/rejected": -0.05483398586511612,
      "step": 5300
    },
    {
      "epoch": 0.38684296798164136,
      "grad_norm": 1.6539779116842404,
      "learning_rate": 6.861558079938485e-07,
      "log_odds_chosen": 0.6576293706893921,
      "log_odds_ratio": -0.564404308795929,
      "logits/chosen": -0.9244140386581421,
      "logits/rejected": -0.851757824420929,
      "logps/chosen": -0.58447265625,
      "logps/rejected": -0.9808593988418579,
      "loss": 0.86,
      "nll_loss": 0.8505859375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02921753004193306,
      "rewards/margins": 0.01977233961224556,
      "rewards/rejected": -0.04898681491613388,
      "step": 5310
    },
    {
      "epoch": 0.38757148581211526,
      "grad_norm": 2.170717943860033,
      "learning_rate": 6.855106213838522e-07,
      "log_odds_chosen": 0.975476086139679,
      "log_odds_ratio": -0.4892578125,
      "logits/chosen": -0.9457031488418579,
      "logits/rejected": -0.8359375,
      "logps/chosen": -0.5467773675918579,
      "logps/rejected": -1.083593726158142,
      "loss": 0.8371,
      "nll_loss": 0.8101562261581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02733154222369194,
      "rewards/margins": 0.026815032586455345,
      "rewards/rejected": -0.05411987379193306,
      "step": 5320
    },
    {
      "epoch": 0.38830000364258915,
      "grad_norm": 1.5352071539975354,
      "learning_rate": 6.848672513487423e-07,
      "log_odds_chosen": 0.905957043170929,
      "log_odds_ratio": -0.48515623807907104,
      "logits/chosen": -0.9388672113418579,
      "logits/rejected": -0.8402343988418579,
      "logps/chosen": -0.54150390625,
      "logps/rejected": -1.0802733898162842,
      "loss": 0.8263,
      "nll_loss": 0.7837890386581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02710571326315403,
      "rewards/margins": 0.02692718431353569,
      "rewards/rejected": -0.05402832105755806,
      "step": 5330
    },
    {
      "epoch": 0.38902852147306305,
      "grad_norm": 1.6408820902702395,
      "learning_rate": 6.842256893799668e-07,
      "log_odds_chosen": 1.026269555091858,
      "log_odds_ratio": -0.4581542909145355,
      "logits/chosen": -0.9740234613418579,
      "logits/rejected": -0.8349609375,
      "logps/chosen": -0.518750011920929,
      "logps/rejected": -1.103515625,
      "loss": 0.8447,
      "nll_loss": 0.84765625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02592773362994194,
      "rewards/margins": 0.02927246131002903,
      "rewards/rejected": -0.05519409105181694,
      "step": 5340
    },
    {
      "epoch": 0.38975703930353695,
      "grad_norm": 1.8385483568526615,
      "learning_rate": 6.835859270246634e-07,
      "log_odds_chosen": 0.8452392816543579,
      "log_odds_ratio": -0.5206054449081421,
      "logits/chosen": -0.9515625238418579,
      "logits/rejected": -0.857421875,
      "logps/chosen": -0.58447265625,
      "logps/rejected": -1.098242163658142,
      "loss": 0.8228,
      "nll_loss": 0.791210949420929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02923583984375,
      "rewards/margins": 0.02568512037396431,
      "rewards/rejected": -0.054931640625,
      "step": 5350
    },
    {
      "epoch": 0.39048555713401084,
      "grad_norm": 1.7890819977559966,
      "learning_rate": 6.829479558851913e-07,
      "log_odds_chosen": 1.0160644054412842,
      "log_odds_ratio": -0.4800781309604645,
      "logits/chosen": -0.936718761920929,
      "logits/rejected": -0.859570324420929,
      "logps/chosen": -0.5591796636581421,
      "logps/rejected": -1.156835913658142,
      "loss": 0.8304,
      "nll_loss": 0.8228515386581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0279541015625,
      "rewards/margins": 0.029865264892578125,
      "rewards/rejected": -0.05781250074505806,
      "step": 5360
    },
    {
      "epoch": 0.39121407496448474,
      "grad_norm": 1.7164022599264346,
      "learning_rate": 6.82311767618669e-07,
      "log_odds_chosen": 0.8204101324081421,
      "log_odds_ratio": -0.4849609434604645,
      "logits/chosen": -0.9095703363418579,
      "logits/rejected": -0.8326171636581421,
      "logps/chosen": -0.560253918170929,
      "logps/rejected": -1.0398437976837158,
      "loss": 0.8388,
      "nll_loss": 0.837890625,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02802734449505806,
      "rewards/margins": 0.02398834191262722,
      "rewards/rejected": -0.05197753757238388,
      "step": 5370
    },
    {
      "epoch": 0.39194259279495863,
      "grad_norm": 1.6035854307132176,
      "learning_rate": 6.816773539365149e-07,
      "log_odds_chosen": 0.957775890827179,
      "log_odds_ratio": -0.48969727754592896,
      "logits/chosen": -0.973828136920929,
      "logits/rejected": -0.8570312261581421,
      "logps/chosen": -0.5697265863418579,
      "logps/rejected": -1.145117163658142,
      "loss": 0.831,
      "nll_loss": 0.8060547113418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02849731408059597,
      "rewards/margins": 0.028709793463349342,
      "rewards/rejected": -0.05724487453699112,
      "step": 5380
    },
    {
      "epoch": 0.3926711106254326,
      "grad_norm": 2.0233422062651565,
      "learning_rate": 6.810447066039945e-07,
      "log_odds_chosen": 0.923876941204071,
      "log_odds_ratio": -0.530712902545929,
      "logits/chosen": -0.931835949420929,
      "logits/rejected": -0.851757824420929,
      "logps/chosen": -0.5956054925918579,
      "logps/rejected": -1.167578101158142,
      "loss": 0.8259,
      "nll_loss": 0.802734375,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02976684644818306,
      "rewards/margins": 0.02859191969037056,
      "rewards/rejected": -0.058349609375,
      "step": 5390
    },
    {
      "epoch": 0.3933996284559065,
      "grad_norm": 1.8366887510687437,
      "learning_rate": 6.804138174397718e-07,
      "log_odds_chosen": 0.955706775188446,
      "log_odds_ratio": -0.478271484375,
      "logits/chosen": -0.932421863079071,
      "logits/rejected": -0.8480468988418579,
      "logps/chosen": -0.5796874761581421,
      "logps/rejected": -1.1349608898162842,
      "loss": 0.8359,
      "nll_loss": 0.8802734613418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.028961181640625,
      "rewards/margins": 0.02778930589556694,
      "rewards/rejected": -0.05678711086511612,
      "step": 5400
    },
    {
      "epoch": 0.3941281462863804,
      "grad_norm": 1.643975775914322,
      "learning_rate": 6.797846783154637e-07,
      "log_odds_chosen": 1.174902319908142,
      "log_odds_ratio": -0.4124999940395355,
      "logits/chosen": -0.954882800579071,
      "logits/rejected": -0.8349609375,
      "logps/chosen": -0.5166015625,
      "logps/rejected": -1.209375023841858,
      "loss": 0.827,
      "nll_loss": 0.8373047113418579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02581787109375,
      "rewards/margins": 0.03466796875,
      "rewards/rejected": -0.0604248046875,
      "step": 5410
    },
    {
      "epoch": 0.3948566641168543,
      "grad_norm": 1.8045684409369416,
      "learning_rate": 6.791572811552017e-07,
      "log_odds_chosen": 0.8224121332168579,
      "log_odds_ratio": -0.521484375,
      "logits/chosen": -0.917187511920929,
      "logits/rejected": -0.8441406488418579,
      "logps/chosen": -0.599902331829071,
      "logps/rejected": -1.105859398841858,
      "loss": 0.8463,
      "nll_loss": 0.823046863079071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02998046949505806,
      "rewards/margins": 0.02530517615377903,
      "rewards/rejected": -0.05530395358800888,
      "step": 5420
    },
    {
      "epoch": 0.39558518194732817,
      "grad_norm": 2.5798226947488425,
      "learning_rate": 6.785316179351949e-07,
      "log_odds_chosen": 1.022705078125,
      "log_odds_ratio": -0.45527344942092896,
      "logits/chosen": -0.928515613079071,
      "logits/rejected": -0.8656250238418579,
      "logps/chosen": -0.5318359136581421,
      "logps/rejected": -1.106835961341858,
      "loss": 0.86,
      "nll_loss": 0.7744140625,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02658691443502903,
      "rewards/margins": 0.028722381219267845,
      "rewards/rejected": -0.05532226711511612,
      "step": 5430
    },
    {
      "epoch": 0.39631369977780206,
      "grad_norm": 1.8881403593275496,
      "learning_rate": 6.779076806833005e-07,
      "log_odds_chosen": 0.866406261920929,
      "log_odds_ratio": -0.49018555879592896,
      "logits/chosen": -0.8876953125,
      "logits/rejected": -0.812304675579071,
      "logps/chosen": -0.532031238079071,
      "logps/rejected": -1.0302734375,
      "loss": 0.8323,
      "nll_loss": 0.760937511920929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02660522423684597,
      "rewards/margins": 0.024950027465820312,
      "rewards/rejected": -0.051483154296875,
      "step": 5440
    },
    {
      "epoch": 0.39704221760827596,
      "grad_norm": 1.7704294078741691,
      "learning_rate": 6.772854614785964e-07,
      "log_odds_chosen": 0.826489269733429,
      "log_odds_ratio": -0.518359363079071,
      "logits/chosen": -0.8990234136581421,
      "logits/rejected": -0.828125,
      "logps/chosen": -0.550585925579071,
      "logps/rejected": -1.0392577648162842,
      "loss": 0.834,
      "nll_loss": 0.869140625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02750854566693306,
      "rewards/margins": 0.024457549676299095,
      "rewards/rejected": -0.05200805515050888,
      "step": 5450
    },
    {
      "epoch": 0.39777073543874986,
      "grad_norm": 1.7910863803370005,
      "learning_rate": 6.766649524509585e-07,
      "log_odds_chosen": 0.9634765386581421,
      "log_odds_ratio": -0.45610350370407104,
      "logits/chosen": -0.961718738079071,
      "logits/rejected": -0.8486328125,
      "logps/chosen": -0.574023425579071,
      "logps/rejected": -1.1628906726837158,
      "loss": 0.836,
      "nll_loss": 0.8255859613418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02872314490377903,
      "rewards/margins": 0.02945251390337944,
      "rewards/rejected": -0.05817871168255806,
      "step": 5460
    },
    {
      "epoch": 0.39849925326922375,
      "grad_norm": 2.0705212442120966,
      "learning_rate": 6.760461457806433e-07,
      "log_odds_chosen": 1.038549780845642,
      "log_odds_ratio": -0.46484375,
      "logits/chosen": -0.9603515863418579,
      "logits/rejected": -0.8701171875,
      "logps/chosen": -0.530468761920929,
      "logps/rejected": -1.1375000476837158,
      "loss": 0.8195,
      "nll_loss": 0.7535156011581421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02650756761431694,
      "rewards/margins": 0.03033142164349556,
      "rewards/rejected": -0.056884765625,
      "step": 5470
    },
    {
      "epoch": 0.39922777109969765,
      "grad_norm": 1.6381720058811646,
      "learning_rate": 6.75429033697874e-07,
      "log_odds_chosen": 1.166601538658142,
      "log_odds_ratio": -0.4215331971645355,
      "logits/chosen": -0.960156261920929,
      "logits/rejected": -0.8349609375,
      "logps/chosen": -0.554394543170929,
      "logps/rejected": -1.248437523841858,
      "loss": 0.8414,
      "nll_loss": 0.7787109613418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02768554724752903,
      "rewards/margins": 0.03472900390625,
      "rewards/rejected": -0.06241454929113388,
      "step": 5480
    },
    {
      "epoch": 0.39995628893017154,
      "grad_norm": 1.6970690877088648,
      "learning_rate": 6.7481360848243e-07,
      "log_odds_chosen": 1.088281273841858,
      "log_odds_ratio": -0.4375,
      "logits/chosen": -0.952929675579071,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.523730456829071,
      "logps/rejected": -1.130859375,
      "loss": 0.834,
      "nll_loss": 0.865234375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02616577222943306,
      "rewards/margins": 0.03035278245806694,
      "rewards/rejected": -0.05654297024011612,
      "step": 5490
    },
    {
      "epoch": 0.40068480676064544,
      "grad_norm": 1.6384851171993857,
      "learning_rate": 6.741998624632422e-07,
      "log_odds_chosen": 1.2141602039337158,
      "log_odds_ratio": -0.3798828125,
      "logits/chosen": -0.947070300579071,
      "logits/rejected": -0.822460949420929,
      "logps/chosen": -0.556933581829071,
      "logps/rejected": -1.2841796875,
      "loss": 0.8342,
      "nll_loss": 0.792187511920929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.02785644493997097,
      "rewards/margins": 0.036302946507930756,
      "rewards/rejected": -0.06418456882238388,
      "step": 5500
    },
    {
      "epoch": 0.4014133245911194,
      "grad_norm": 1.7210373366464367,
      "learning_rate": 6.735877880179904e-07,
      "log_odds_chosen": 0.9404296875,
      "log_odds_ratio": -0.48798829317092896,
      "logits/chosen": -0.923046886920929,
      "logits/rejected": -0.78759765625,
      "logps/chosen": -0.568652331829071,
      "logps/rejected": -1.1433594226837158,
      "loss": 0.8366,
      "nll_loss": 0.824999988079071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02843017503619194,
      "rewards/margins": 0.02868041954934597,
      "rewards/rejected": -0.05711669847369194,
      "step": 5510
    },
    {
      "epoch": 0.4021418424215933,
      "grad_norm": 1.576690275216497,
      "learning_rate": 6.72977377572707e-07,
      "log_odds_chosen": 0.906982421875,
      "log_odds_ratio": -0.510937511920929,
      "logits/chosen": -0.9404296875,
      "logits/rejected": -0.8539062738418579,
      "logps/chosen": -0.612597644329071,
      "logps/rejected": -1.174218773841858,
      "loss": 0.8427,
      "nll_loss": 0.818359375,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03061523474752903,
      "rewards/margins": 0.02811126783490181,
      "rewards/rejected": -0.05870361253619194,
      "step": 5520
    },
    {
      "epoch": 0.4028703602520672,
      "grad_norm": 1.729185460138946,
      "learning_rate": 6.723686236013819e-07,
      "log_odds_chosen": 0.875048816204071,
      "log_odds_ratio": -0.48139649629592896,
      "logits/chosen": -0.9359375238418579,
      "logits/rejected": -0.815625011920929,
      "logps/chosen": -0.549609363079071,
      "logps/rejected": -1.0771484375,
      "loss": 0.7921,
      "nll_loss": 0.75146484375,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02747192420065403,
      "rewards/margins": 0.02640380896627903,
      "rewards/rejected": -0.05387573316693306,
      "step": 5530
    },
    {
      "epoch": 0.4035988780825411,
      "grad_norm": 1.704790094568569,
      "learning_rate": 6.717615186255738e-07,
      "log_odds_chosen": 0.8372802734375,
      "log_odds_ratio": -0.533398449420929,
      "logits/chosen": -0.8873046636581421,
      "logits/rejected": -0.7796875238418579,
      "logps/chosen": -0.5829101800918579,
      "logps/rejected": -1.0671875476837158,
      "loss": 0.8172,
      "nll_loss": 0.8017578125,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02912597730755806,
      "rewards/margins": 0.02420654334127903,
      "rewards/rejected": -0.05335693433880806,
      "step": 5540
    },
    {
      "epoch": 0.404327395913015,
      "grad_norm": 1.9726962018006575,
      "learning_rate": 6.711560552140243e-07,
      "log_odds_chosen": 1.142187476158142,
      "log_odds_ratio": -0.439453125,
      "logits/chosen": -0.951367199420929,
      "logits/rejected": -0.8570312261581421,
      "logps/chosen": -0.574023425579071,
      "logps/rejected": -1.2763671875,
      "loss": 0.822,
      "nll_loss": 0.7845703363418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02869262732565403,
      "rewards/margins": 0.03505248948931694,
      "rewards/rejected": -0.06379394233226776,
      "step": 5550
    },
    {
      "epoch": 0.40505591374348887,
      "grad_norm": 2.0129039813076206,
      "learning_rate": 6.705522259822751e-07,
      "log_odds_chosen": 1.14398193359375,
      "log_odds_ratio": -0.4281249940395355,
      "logits/chosen": -0.924609363079071,
      "logits/rejected": -0.7919921875,
      "logps/chosen": -0.542675793170929,
      "logps/rejected": -1.2365233898162842,
      "loss": 0.8378,
      "nll_loss": 0.8412109613418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02711791917681694,
      "rewards/margins": 0.034723661839962006,
      "rewards/rejected": -0.06184082105755806,
      "step": 5560
    },
    {
      "epoch": 0.40578443157396277,
      "grad_norm": 1.858665225380592,
      "learning_rate": 6.699500235922906e-07,
      "log_odds_chosen": 0.92388916015625,
      "log_odds_ratio": -0.503466784954071,
      "logits/chosen": -0.9662109613418579,
      "logits/rejected": -0.8306640386581421,
      "logps/chosen": -0.578417956829071,
      "logps/rejected": -1.1482422351837158,
      "loss": 0.8279,
      "nll_loss": 0.803906261920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02891845628619194,
      "rewards/margins": 0.02851257286965847,
      "rewards/rejected": -0.05743408203125,
      "step": 5570
    },
    {
      "epoch": 0.40651294940443666,
      "grad_norm": 1.5965365782740903,
      "learning_rate": 6.693494407520824e-07,
      "log_odds_chosen": 0.84814453125,
      "log_odds_ratio": -0.521679699420929,
      "logits/chosen": -0.9267578125,
      "logits/rejected": -0.8218749761581421,
      "logps/chosen": -0.6015625,
      "logps/rejected": -1.1085937023162842,
      "loss": 0.8533,
      "nll_loss": 0.835742175579071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.030120849609375,
      "rewards/margins": 0.025298308581113815,
      "rewards/rejected": -0.05540771409869194,
      "step": 5580
    },
    {
      "epoch": 0.40724146723491056,
      "grad_norm": 1.668969762644284,
      "learning_rate": 6.687504702153398e-07,
      "log_odds_chosen": 0.90478515625,
      "log_odds_ratio": -0.49882811307907104,
      "logits/chosen": -0.939648449420929,
      "logits/rejected": -0.81640625,
      "logps/chosen": -0.53173828125,
      "logps/rejected": -1.0466797351837158,
      "loss": 0.8247,
      "nll_loss": 0.8140624761581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02656860277056694,
      "rewards/margins": 0.02578430250287056,
      "rewards/rejected": -0.05235595628619194,
      "step": 5590
    },
    {
      "epoch": 0.40796998506538446,
      "grad_norm": 1.946009015439919,
      "learning_rate": 6.681531047810611e-07,
      "log_odds_chosen": 0.925000011920929,
      "log_odds_ratio": -0.4800781309604645,
      "logits/chosen": -0.9742187261581421,
      "logits/rejected": -0.8414062261581421,
      "logps/chosen": -0.594531238079071,
      "logps/rejected": -1.1443359851837158,
      "loss": 0.8536,
      "nll_loss": 0.837695300579071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02971191331744194,
      "rewards/margins": 0.02745513990521431,
      "rewards/rejected": -0.05720214918255806,
      "step": 5600
    },
    {
      "epoch": 0.40869850289585835,
      "grad_norm": 2.1819011078223713,
      "learning_rate": 6.675573372931909e-07,
      "log_odds_chosen": 0.840527355670929,
      "log_odds_ratio": -0.5225585699081421,
      "logits/chosen": -0.9222656488418579,
      "logits/rejected": -0.830859363079071,
      "logps/chosen": -0.541796863079071,
      "logps/rejected": -1.0167968273162842,
      "loss": 0.8427,
      "nll_loss": 0.793749988079071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02709350548684597,
      "rewards/margins": 0.02374877966940403,
      "rewards/rejected": -0.05084228515625,
      "step": 5610
    },
    {
      "epoch": 0.4094270207263323,
      "grad_norm": 1.6775703886550937,
      "learning_rate": 6.669631606402604e-07,
      "log_odds_chosen": 0.992626965045929,
      "log_odds_ratio": -0.4562011659145355,
      "logits/chosen": -0.9291015863418579,
      "logits/rejected": -0.8316406011581421,
      "logps/chosen": -0.5658203363418579,
      "logps/rejected": -1.1257812976837158,
      "loss": 0.8279,
      "nll_loss": 0.8167968988418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02828369103372097,
      "rewards/margins": 0.02798309363424778,
      "rewards/rejected": -0.05626220628619194,
      "step": 5620
    },
    {
      "epoch": 0.4101555385568062,
      "grad_norm": 1.5692413443616162,
      "learning_rate": 6.663705677550291e-07,
      "log_odds_chosen": 0.786669909954071,
      "log_odds_ratio": -0.5274413824081421,
      "logits/chosen": -0.9818359613418579,
      "logits/rejected": -0.8617187738418579,
      "logps/chosen": -0.5821288824081421,
      "logps/rejected": -1.076171875,
      "loss": 0.825,
      "nll_loss": 0.765429675579071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02912597730755806,
      "rewards/margins": 0.02471008338034153,
      "rewards/rejected": -0.05381469801068306,
      "step": 5630
    },
    {
      "epoch": 0.4108840563872801,
      "grad_norm": 1.7750043507107167,
      "learning_rate": 6.657795516141343e-07,
      "log_odds_chosen": 0.861035168170929,
      "log_odds_ratio": -0.500781238079071,
      "logits/chosen": -0.923828125,
      "logits/rejected": -0.833984375,
      "logps/chosen": -0.5918945074081421,
      "logps/rejected": -1.099218726158142,
      "loss": 0.8524,
      "nll_loss": 0.890429675579071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02956542931497097,
      "rewards/margins": 0.0253753662109375,
      "rewards/rejected": -0.05496826022863388,
      "step": 5640
    },
    {
      "epoch": 0.411612574217754,
      "grad_norm": 2.2006717292374187,
      "learning_rate": 6.651901052377394e-07,
      "log_odds_chosen": 0.941113293170929,
      "log_odds_ratio": -0.50341796875,
      "logits/chosen": -0.9345703125,
      "logits/rejected": -0.8466796875,
      "logps/chosen": -0.555371105670929,
      "logps/rejected": -1.133203148841858,
      "loss": 0.8401,
      "nll_loss": 0.8238281011581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02775878831744194,
      "rewards/margins": 0.0288848876953125,
      "rewards/rejected": -0.05659789964556694,
      "step": 5650
    },
    {
      "epoch": 0.4123410920482279,
      "grad_norm": 1.6336244164265625,
      "learning_rate": 6.646022216891883e-07,
      "log_odds_chosen": 1.1787109375,
      "log_odds_ratio": -0.39643555879592896,
      "logits/chosen": -0.988085925579071,
      "logits/rejected": -0.849414050579071,
      "logps/chosen": -0.5224609375,
      "logps/rejected": -1.2080078125,
      "loss": 0.8287,
      "nll_loss": 0.8070312738418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.02612915076315403,
      "rewards/margins": 0.0343170166015625,
      "rewards/rejected": -0.06046142429113388,
      "step": 5660
    },
    {
      "epoch": 0.4130696098787018,
      "grad_norm": 1.749688060217104,
      "learning_rate": 6.640158940746632e-07,
      "log_odds_chosen": 0.878710925579071,
      "log_odds_ratio": -0.4888671934604645,
      "logits/chosen": -0.9501953125,
      "logits/rejected": -0.8291015625,
      "logps/chosen": -0.5732421875,
      "logps/rejected": -1.117773413658142,
      "loss": 0.8144,
      "nll_loss": 0.798046886920929,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.028656005859375,
      "rewards/margins": 0.02726135216653347,
      "rewards/rejected": -0.05592041090130806,
      "step": 5670
    },
    {
      "epoch": 0.4137981277091757,
      "grad_norm": 1.6033308932466823,
      "learning_rate": 6.634311155428441e-07,
      "log_odds_chosen": 0.8003174066543579,
      "log_odds_ratio": -0.535839855670929,
      "logits/chosen": -0.9251953363418579,
      "logits/rejected": -0.830078125,
      "logps/chosen": -0.6005859375,
      "logps/rejected": -1.106835961341858,
      "loss": 0.8238,
      "nll_loss": 0.850390613079071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03004150465130806,
      "rewards/margins": 0.025305557996034622,
      "rewards/rejected": -0.05535888671875,
      "step": 5680
    },
    {
      "epoch": 0.4145266455396496,
      "grad_norm": 1.8311004924328644,
      "learning_rate": 6.628478792845734e-07,
      "log_odds_chosen": 0.775927722454071,
      "log_odds_ratio": -0.5245116949081421,
      "logits/chosen": -0.9173828363418579,
      "logits/rejected": -0.787890613079071,
      "logps/chosen": -0.55517578125,
      "logps/rejected": -1.014062523841858,
      "loss": 0.8406,
      "nll_loss": 0.795117199420929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02777099609375,
      "rewards/margins": 0.022966766729950905,
      "rewards/rejected": -0.05073242262005806,
      "step": 5690
    },
    {
      "epoch": 0.41525516337012347,
      "grad_norm": 1.7948927962697536,
      "learning_rate": 6.62266178532522e-07,
      "log_odds_chosen": 0.9814453125,
      "log_odds_ratio": -0.477294921875,
      "logits/chosen": -0.9537109136581421,
      "logits/rejected": -0.854296863079071,
      "logps/chosen": -0.5606445074081421,
      "logps/rejected": -1.1076171398162842,
      "loss": 0.8434,
      "nll_loss": 0.7818359136581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02802734449505806,
      "rewards/margins": 0.02734680101275444,
      "rewards/rejected": -0.05537109449505806,
      "step": 5700
    },
    {
      "epoch": 0.41598368120059737,
      "grad_norm": 1.7084675435966337,
      "learning_rate": 6.616860065608603e-07,
      "log_odds_chosen": 0.887158215045929,
      "log_odds_ratio": -0.48828125,
      "logits/chosen": -0.9052734375,
      "logits/rejected": -0.806445300579071,
      "logps/chosen": -0.5503906011581421,
      "logps/rejected": -1.086328148841858,
      "loss": 0.8436,
      "nll_loss": 0.809374988079071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.027557373046875,
      "rewards/margins": 0.026795197278261185,
      "rewards/rejected": -0.05433349683880806,
      "step": 5710
    },
    {
      "epoch": 0.41671219903107126,
      "grad_norm": 1.6000348438969838,
      "learning_rate": 6.611073566849313e-07,
      "log_odds_chosen": 0.810742199420929,
      "log_odds_ratio": -0.5040038824081421,
      "logits/chosen": -0.910351574420929,
      "logits/rejected": -0.828320324420929,
      "logps/chosen": -0.5726562738418579,
      "logps/rejected": -1.045507788658142,
      "loss": 0.8308,
      "nll_loss": 0.824023425579071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02862548828125,
      "rewards/margins": 0.02362060546875,
      "rewards/rejected": -0.05223388597369194,
      "step": 5720
    },
    {
      "epoch": 0.4174407168615452,
      "grad_norm": 1.6586900652026977,
      "learning_rate": 6.605302222609272e-07,
      "log_odds_chosen": 0.860156238079071,
      "log_odds_ratio": -0.5359863042831421,
      "logits/chosen": -0.942578136920929,
      "logits/rejected": -0.848437488079071,
      "logps/chosen": -0.563183605670929,
      "logps/rejected": -1.092187523841858,
      "loss": 0.8397,
      "nll_loss": 0.787402331829071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02814941480755806,
      "rewards/margins": 0.02644500695168972,
      "rewards/rejected": -0.05460815504193306,
      "step": 5730
    },
    {
      "epoch": 0.4181692346920191,
      "grad_norm": 2.618038944322766,
      "learning_rate": 6.599545966855683e-07,
      "log_odds_chosen": 1.0208008289337158,
      "log_odds_ratio": -0.47236329317092896,
      "logits/chosen": -0.922656238079071,
      "logits/rejected": -0.826367199420929,
      "logps/chosen": -0.582226574420929,
      "logps/rejected": -1.2048828601837158,
      "loss": 0.8329,
      "nll_loss": 0.799121081829071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02911376953125,
      "rewards/margins": 0.03113555908203125,
      "rewards/rejected": -0.06024169921875,
      "step": 5740
    },
    {
      "epoch": 0.418897752522493,
      "grad_norm": 2.0025749167822293,
      "learning_rate": 6.59380473395787e-07,
      "log_odds_chosen": 0.884692370891571,
      "log_odds_ratio": -0.50244140625,
      "logits/chosen": -0.9986327886581421,
      "logits/rejected": -0.8570312261581421,
      "logps/chosen": -0.59228515625,
      "logps/rejected": -1.173828125,
      "loss": 0.8498,
      "nll_loss": 0.845898449420929,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02961425855755806,
      "rewards/margins": 0.029103850945830345,
      "rewards/rejected": -0.05869140475988388,
      "step": 5750
    },
    {
      "epoch": 0.4196262703529669,
      "grad_norm": 1.6482633711618822,
      "learning_rate": 6.588078458684124e-07,
      "log_odds_chosen": 0.8570556640625,
      "log_odds_ratio": -0.5419921875,
      "logits/chosen": -0.92578125,
      "logits/rejected": -0.8167968988418579,
      "logps/chosen": -0.522656261920929,
      "logps/rejected": -1.0070312023162842,
      "loss": 0.8398,
      "nll_loss": 0.795117199420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02614746056497097,
      "rewards/margins": 0.024217605590820312,
      "rewards/rejected": -0.05036010593175888,
      "step": 5760
    },
    {
      "epoch": 0.4203547881834408,
      "grad_norm": 2.638784786451909,
      "learning_rate": 6.582367076198594e-07,
      "log_odds_chosen": 0.9320312738418579,
      "log_odds_ratio": -0.47705078125,
      "logits/chosen": -0.9375,
      "logits/rejected": -0.8480468988418579,
      "logps/chosen": -0.553906261920929,
      "logps/rejected": -1.0919921398162842,
      "loss": 0.8175,
      "nll_loss": 0.8140624761581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.027679443359375,
      "rewards/margins": 0.026971053332090378,
      "rewards/rejected": -0.05463256686925888,
      "step": 5770
    },
    {
      "epoch": 0.4210833060139147,
      "grad_norm": 2.0426499835323586,
      "learning_rate": 6.576670522058205e-07,
      "log_odds_chosen": 0.898193359375,
      "log_odds_ratio": -0.520703136920929,
      "logits/chosen": -0.9292968511581421,
      "logits/rejected": -0.8421875238418579,
      "logps/chosen": -0.5777343511581421,
      "logps/rejected": -1.1550781726837158,
      "loss": 0.835,
      "nll_loss": 0.846875011920929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02888793870806694,
      "rewards/margins": 0.02889099158346653,
      "rewards/rejected": -0.05778808519244194,
      "step": 5780
    },
    {
      "epoch": 0.4218118238443886,
      "grad_norm": 1.6862241801953783,
      "learning_rate": 6.570988732209603e-07,
      "log_odds_chosen": 0.8391357660293579,
      "log_odds_ratio": -0.5140625238418579,
      "logits/chosen": -0.973437488079071,
      "logits/rejected": -0.866406261920929,
      "logps/chosen": -0.576855480670929,
      "logps/rejected": -1.111718773841858,
      "loss": 0.8608,
      "nll_loss": 0.839648425579071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02886352501809597,
      "rewards/margins": 0.02675781212747097,
      "rewards/rejected": -0.05560302734375,
      "step": 5790
    },
    {
      "epoch": 0.4225403416748625,
      "grad_norm": 1.6106568926035498,
      "learning_rate": 6.565321642986128e-07,
      "log_odds_chosen": 0.8546386957168579,
      "log_odds_ratio": -0.5091308355331421,
      "logits/chosen": -0.896484375,
      "logits/rejected": -0.8306640386581421,
      "logps/chosen": -0.525390625,
      "logps/rejected": -1.024999976158142,
      "loss": 0.792,
      "nll_loss": 0.7681640386581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02626342698931694,
      "rewards/margins": 0.02499694749712944,
      "rewards/rejected": -0.05125732347369194,
      "step": 5800
    },
    {
      "epoch": 0.4232688595053364,
      "grad_norm": 1.691379245444721,
      "learning_rate": 6.559669191104821e-07,
      "log_odds_chosen": 0.997363269329071,
      "log_odds_ratio": -0.46435546875,
      "logits/chosen": -0.9566406011581421,
      "logits/rejected": -0.83203125,
      "logps/chosen": -0.53369140625,
      "logps/rejected": -1.113671898841858,
      "loss": 0.8296,
      "nll_loss": 0.8052734136581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02667846716940403,
      "rewards/margins": 0.02901916578412056,
      "rewards/rejected": -0.05568847805261612,
      "step": 5810
    },
    {
      "epoch": 0.4239973773358103,
      "grad_norm": 1.772638402741094,
      "learning_rate": 6.554031313663455e-07,
      "log_odds_chosen": 1.093408226966858,
      "log_odds_ratio": -0.484375,
      "logits/chosen": -0.959179699420929,
      "logits/rejected": -0.8345702886581421,
      "logps/chosen": -0.5503906011581421,
      "logps/rejected": -1.225195288658142,
      "loss": 0.8313,
      "nll_loss": 0.802734375,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02750854566693306,
      "rewards/margins": 0.03374939039349556,
      "rewards/rejected": -0.06123046949505806,
      "step": 5820
    },
    {
      "epoch": 0.4247258951662842,
      "grad_norm": 1.7729542883060383,
      "learning_rate": 6.548407948137591e-07,
      "log_odds_chosen": 0.872021496295929,
      "log_odds_ratio": -0.537060558795929,
      "logits/chosen": -0.933789074420929,
      "logits/rejected": -0.8326171636581421,
      "logps/chosen": -0.531542956829071,
      "logps/rejected": -1.0505859851837158,
      "loss": 0.8063,
      "nll_loss": 0.7837890386581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02656860277056694,
      "rewards/margins": 0.025946807116270065,
      "rewards/rejected": -0.05247802659869194,
      "step": 5830
    },
    {
      "epoch": 0.42545441299675807,
      "grad_norm": 1.78994182484539,
      "learning_rate": 6.542799032377671e-07,
      "log_odds_chosen": 0.9050048589706421,
      "log_odds_ratio": -0.5150390863418579,
      "logits/chosen": -0.947265625,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -0.581250011920929,
      "logps/rejected": -1.1110351085662842,
      "loss": 0.8421,
      "nll_loss": 0.8548828363418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.029052734375,
      "rewards/margins": 0.02648162841796875,
      "rewards/rejected": -0.05549316480755806,
      "step": 5840
    },
    {
      "epoch": 0.426182930827232,
      "grad_norm": 1.6195348878498872,
      "learning_rate": 6.537204504606134e-07,
      "log_odds_chosen": 0.888293445110321,
      "log_odds_ratio": -0.5001465082168579,
      "logits/chosen": -0.9771484136581421,
      "logits/rejected": -0.8695312738418579,
      "logps/chosen": -0.5936523675918579,
      "logps/rejected": -1.125390648841858,
      "loss": 0.8512,
      "nll_loss": 0.8388671875,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02966918982565403,
      "rewards/margins": 0.026613617315888405,
      "rewards/rejected": -0.05626831203699112,
      "step": 5850
    },
    {
      "epoch": 0.4269114486577059,
      "grad_norm": 1.684495949796905,
      "learning_rate": 6.531624303414552e-07,
      "log_odds_chosen": 0.9286133050918579,
      "log_odds_ratio": -0.4803710877895355,
      "logits/chosen": -0.9400390386581421,
      "logits/rejected": -0.8597656488418579,
      "logps/chosen": -0.5082031488418579,
      "logps/rejected": -1.060546875,
      "loss": 0.8171,
      "nll_loss": 0.745898425579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02543335035443306,
      "rewards/margins": 0.02758483961224556,
      "rewards/rejected": -0.05307617038488388,
      "step": 5860
    },
    {
      "epoch": 0.4276399664881798,
      "grad_norm": 2.3447141809884195,
      "learning_rate": 6.526058367760803e-07,
      "log_odds_chosen": 0.926440417766571,
      "log_odds_ratio": -0.486083984375,
      "logits/chosen": -0.970703125,
      "logits/rejected": -0.871289074420929,
      "logps/chosen": -0.5474609136581421,
      "logps/rejected": -1.0925781726837158,
      "loss": 0.8299,
      "nll_loss": 0.721875011920929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02739868126809597,
      "rewards/margins": 0.02726135216653347,
      "rewards/rejected": -0.05460815504193306,
      "step": 5870
    },
    {
      "epoch": 0.4283684843186537,
      "grad_norm": 1.857381752704321,
      "learning_rate": 6.520506636966264e-07,
      "log_odds_chosen": 0.9110473394393921,
      "log_odds_ratio": -0.534863293170929,
      "logits/chosen": -0.935742199420929,
      "logits/rejected": -0.8548828363418579,
      "logps/chosen": -0.5267578363418579,
      "logps/rejected": -1.027734398841858,
      "loss": 0.8257,
      "nll_loss": 0.8017578125,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.02634887769818306,
      "rewards/margins": 0.02505188062787056,
      "rewards/rejected": -0.05138549953699112,
      "step": 5880
    },
    {
      "epoch": 0.4290970021491276,
      "grad_norm": 1.7098471638073984,
      "learning_rate": 6.514969050713038e-07,
      "log_odds_chosen": 0.853515625,
      "log_odds_ratio": -0.52734375,
      "logits/chosen": -0.9537109136581421,
      "logits/rejected": -0.8207031488418579,
      "logps/chosen": -0.616992175579071,
      "logps/rejected": -1.119726538658142,
      "loss": 0.8373,
      "nll_loss": 0.8111327886581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.030853271484375,
      "rewards/margins": 0.02509002760052681,
      "rewards/rejected": -0.05598754808306694,
      "step": 5890
    },
    {
      "epoch": 0.4298255199796015,
      "grad_norm": 1.7884946521606935,
      "learning_rate": 6.509445549041195e-07,
      "log_odds_chosen": 0.913867175579071,
      "log_odds_ratio": -0.5160156488418579,
      "logits/chosen": -0.922656238079071,
      "logits/rejected": -0.8099609613418579,
      "logps/chosen": -0.526171863079071,
      "logps/rejected": -1.059960961341858,
      "loss": 0.8258,
      "nll_loss": 0.7767578363418579,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02629394456744194,
      "rewards/margins": 0.02671051025390625,
      "rewards/rejected": -0.05301513522863388,
      "step": 5900
    },
    {
      "epoch": 0.4305540378100754,
      "grad_norm": 1.7102703401977146,
      "learning_rate": 6.503936072346047e-07,
      "log_odds_chosen": 0.9638671875,
      "log_odds_ratio": -0.4961914122104645,
      "logits/chosen": -0.9624999761581421,
      "logits/rejected": -0.8765624761581421,
      "logps/chosen": -0.5863281488418579,
      "logps/rejected": -1.1960937976837158,
      "loss": 0.826,
      "nll_loss": 0.768359363079071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02930908277630806,
      "rewards/margins": 0.03050537034869194,
      "rewards/rejected": -0.05980224534869194,
      "step": 5910
    },
    {
      "epoch": 0.4312825556405493,
      "grad_norm": 1.7692913863511515,
      "learning_rate": 6.498440561375455e-07,
      "log_odds_chosen": 1.2000000476837158,
      "log_odds_ratio": -0.4000000059604645,
      "logits/chosen": -0.973828136920929,
      "logits/rejected": -0.8705078363418579,
      "logps/chosen": -0.54296875,
      "logps/rejected": -1.280859351158142,
      "loss": 0.8336,
      "nll_loss": 0.8072265386581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02715454064309597,
      "rewards/margins": 0.03692321851849556,
      "rewards/rejected": -0.06406249850988388,
      "step": 5920
    },
    {
      "epoch": 0.4320110734710232,
      "grad_norm": 1.8331179522178587,
      "learning_rate": 6.492958957227136e-07,
      "log_odds_chosen": 1.04296875,
      "log_odds_ratio": -0.47407227754592896,
      "logits/chosen": -0.945117175579071,
      "logits/rejected": -0.8443359136581421,
      "logps/chosen": -0.525683581829071,
      "logps/rejected": -1.150390625,
      "loss": 0.8314,
      "nll_loss": 0.80078125,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02629394456744194,
      "rewards/margins": 0.03125,
      "rewards/rejected": -0.05753173679113388,
      "step": 5930
    },
    {
      "epoch": 0.4327395913014971,
      "grad_norm": 1.7504404142070562,
      "learning_rate": 6.487491201346026e-07,
      "log_odds_chosen": 0.908825695514679,
      "log_odds_ratio": -0.5210937261581421,
      "logits/chosen": -0.9320312738418579,
      "logits/rejected": -0.815234363079071,
      "logps/chosen": -0.57568359375,
      "logps/rejected": -1.1171875,
      "loss": 0.8473,
      "nll_loss": 0.8355468511581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02873535081744194,
      "rewards/margins": 0.027101706713438034,
      "rewards/rejected": -0.055877685546875,
      "step": 5940
    },
    {
      "epoch": 0.433468109131971,
      "grad_norm": 2.186616829537579,
      "learning_rate": 6.482037235521644e-07,
      "log_odds_chosen": 1.2980468273162842,
      "log_odds_ratio": -0.38066405057907104,
      "logits/chosen": -0.943554699420929,
      "logits/rejected": -0.808398425579071,
      "logps/chosen": -0.563671886920929,
      "logps/rejected": -1.345312476158142,
      "loss": 0.8175,
      "nll_loss": 0.7925781011581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.02817993238568306,
      "rewards/margins": 0.03909606859087944,
      "rewards/rejected": -0.06722412258386612,
      "step": 5950
    },
    {
      "epoch": 0.43419662696244493,
      "grad_norm": 2.229008638989092,
      "learning_rate": 6.476597001885497e-07,
      "log_odds_chosen": 0.977490246295929,
      "log_odds_ratio": -0.47050780057907104,
      "logits/chosen": -0.897265613079071,
      "logits/rejected": -0.796679675579071,
      "logps/chosen": -0.548828125,
      "logps/rejected": -1.119531273841858,
      "loss": 0.8165,
      "nll_loss": 0.8148437738418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02741699293255806,
      "rewards/margins": 0.028590012341737747,
      "rewards/rejected": -0.05599365383386612,
      "step": 5960
    },
    {
      "epoch": 0.4349251447929188,
      "grad_norm": 1.8453078530267972,
      "learning_rate": 6.471170442908495e-07,
      "log_odds_chosen": 0.881591796875,
      "log_odds_ratio": -0.515917956829071,
      "logits/chosen": -0.9085937738418579,
      "logits/rejected": -0.81640625,
      "logps/chosen": -0.548632800579071,
      "logps/rejected": -1.076171875,
      "loss": 0.8104,
      "nll_loss": 0.778124988079071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02742309495806694,
      "rewards/margins": 0.026383209973573685,
      "rewards/rejected": -0.05381469801068306,
      "step": 5970
    },
    {
      "epoch": 0.4356536626233927,
      "grad_norm": 1.702572284403242,
      "learning_rate": 6.465757501398396e-07,
      "log_odds_chosen": 1.0,
      "log_odds_ratio": -0.46259766817092896,
      "logits/chosen": -0.965039074420929,
      "logits/rejected": -0.8291015625,
      "logps/chosen": -0.58251953125,
      "logps/rejected": -1.1806640625,
      "loss": 0.8218,
      "nll_loss": 0.7660156488418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02909545972943306,
      "rewards/margins": 0.02991943433880806,
      "rewards/rejected": -0.05903320387005806,
      "step": 5980
    },
    {
      "epoch": 0.4363821804538666,
      "grad_norm": 1.7659560885739558,
      "learning_rate": 6.460358120497279e-07,
      "log_odds_chosen": 0.963818371295929,
      "log_odds_ratio": -0.501269519329071,
      "logits/chosen": -0.9039062261581421,
      "logits/rejected": -0.760546863079071,
      "logps/chosen": -0.555957019329071,
      "logps/rejected": -1.1453125476837158,
      "loss": 0.8227,
      "nll_loss": 0.823437511920929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02782592736184597,
      "rewards/margins": 0.029435347765684128,
      "rewards/rejected": -0.05731201171875,
      "step": 5990
    },
    {
      "epoch": 0.4371106982843405,
      "grad_norm": 2.3991334959658874,
      "learning_rate": 6.454972243679028e-07,
      "log_odds_chosen": 1.2761719226837158,
      "log_odds_ratio": -0.3694824278354645,
      "logits/chosen": -0.943359375,
      "logits/rejected": -0.797070324420929,
      "logps/chosen": -0.545703113079071,
      "logps/rejected": -1.3292968273162842,
      "loss": 0.8202,
      "nll_loss": 0.8326171636581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.02728271484375,
      "rewards/margins": 0.0391387939453125,
      "rewards/rejected": -0.06646728515625,
      "step": 6000
    },
    {
      "epoch": 0.4378392161148144,
      "grad_norm": 2.0584952547489004,
      "learning_rate": 6.449599814746858e-07,
      "log_odds_chosen": 0.99261474609375,
      "log_odds_ratio": -0.4684081971645355,
      "logits/chosen": -0.9544922113418579,
      "logits/rejected": -0.826367199420929,
      "logps/chosen": -0.538769543170929,
      "logps/rejected": -1.119726538658142,
      "loss": 0.8375,
      "nll_loss": 0.7943359613418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02691650390625,
      "rewards/margins": 0.02900390699505806,
      "rewards/rejected": -0.05595703050494194,
      "step": 6010
    },
    {
      "epoch": 0.4385677339452883,
      "grad_norm": 1.7362461616928082,
      "learning_rate": 6.444240777830839e-07,
      "log_odds_chosen": 0.878125011920929,
      "log_odds_ratio": -0.521777331829071,
      "logits/chosen": -0.927929699420929,
      "logits/rejected": -0.825976550579071,
      "logps/chosen": -0.5430663824081421,
      "logps/rejected": -1.065039038658142,
      "loss": 0.8279,
      "nll_loss": 0.7783203125,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02714233472943306,
      "rewards/margins": 0.02610168419778347,
      "rewards/rejected": -0.05328369140625,
      "step": 6020
    },
    {
      "epoch": 0.4392962517757622,
      "grad_norm": 1.9325276147274169,
      "learning_rate": 6.438895077385467e-07,
      "log_odds_chosen": 1.0196411609649658,
      "log_odds_ratio": -0.4793945252895355,
      "logits/chosen": -0.9458984136581421,
      "logits/rejected": -0.8589843511581421,
      "logps/chosen": -0.538867175579071,
      "logps/rejected": -1.1533203125,
      "loss": 0.8284,
      "nll_loss": 0.8003906011581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.026947021484375,
      "rewards/margins": 0.03068237379193306,
      "rewards/rejected": -0.05767822265625,
      "step": 6030
    },
    {
      "epoch": 0.4400247696062361,
      "grad_norm": 1.9135325219981534,
      "learning_rate": 6.433562658187235e-07,
      "log_odds_chosen": 1.0986328125,
      "log_odds_ratio": -0.4532226622104645,
      "logits/chosen": -0.9632812738418579,
      "logits/rejected": -0.8544921875,
      "logps/chosen": -0.561328113079071,
      "logps/rejected": -1.2107422351837158,
      "loss": 0.8378,
      "nll_loss": 0.777636706829071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02805786207318306,
      "rewards/margins": 0.03253784030675888,
      "rewards/rejected": -0.06057129055261612,
      "step": 6040
    },
    {
      "epoch": 0.44075328743671,
      "grad_norm": 2.2411363624102747,
      "learning_rate": 6.428243465332252e-07,
      "log_odds_chosen": 0.7564452886581421,
      "log_odds_ratio": -0.55908203125,
      "logits/chosen": -0.9371093511581421,
      "logits/rejected": -0.834765613079071,
      "logps/chosen": -0.628710925579071,
      "logps/rejected": -1.0908203125,
      "loss": 0.8386,
      "nll_loss": 0.8792968988418579,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.031463623046875,
      "rewards/margins": 0.02310028113424778,
      "rewards/rejected": -0.054534912109375,
      "step": 6050
    },
    {
      "epoch": 0.4414818052671839,
      "grad_norm": 2.0291181811344705,
      "learning_rate": 6.422937444233849e-07,
      "log_odds_chosen": 0.974804699420929,
      "log_odds_ratio": -0.486328125,
      "logits/chosen": -0.9281250238418579,
      "logits/rejected": -0.84765625,
      "logps/chosen": -0.592089831829071,
      "logps/rejected": -1.1906249523162842,
      "loss": 0.8231,
      "nll_loss": 0.80859375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02962036058306694,
      "rewards/margins": 0.02990875206887722,
      "rewards/rejected": -0.0595703125,
      "step": 6060
    },
    {
      "epoch": 0.44221032309765784,
      "grad_norm": 1.934403306927631,
      "learning_rate": 6.417644540620249e-07,
      "log_odds_chosen": 1.1471679210662842,
      "log_odds_ratio": -0.42333984375,
      "logits/chosen": -0.927929699420929,
      "logits/rejected": -0.8519531488418579,
      "logps/chosen": -0.5264648199081421,
      "logps/rejected": -1.1798827648162842,
      "loss": 0.8317,
      "nll_loss": 0.800000011920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02633056603372097,
      "rewards/margins": 0.03265685960650444,
      "rewards/rejected": -0.05900878831744194,
      "step": 6070
    },
    {
      "epoch": 0.44293884092813174,
      "grad_norm": 1.7385599395530955,
      "learning_rate": 6.412364700532215e-07,
      "log_odds_chosen": 0.9161132574081421,
      "log_odds_ratio": -0.46064454317092896,
      "logits/chosen": -0.968554675579071,
      "logits/rejected": -0.862500011920929,
      "logps/chosen": -0.54736328125,
      "logps/rejected": -1.0554687976837158,
      "loss": 0.8301,
      "nll_loss": 0.7989257574081421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02735595777630806,
      "rewards/margins": 0.02540283277630806,
      "rewards/rejected": -0.05274658277630806,
      "step": 6080
    },
    {
      "epoch": 0.44366735875860563,
      "grad_norm": 1.7538486950132275,
      "learning_rate": 6.407097870320747e-07,
      "log_odds_chosen": 0.805859386920929,
      "log_odds_ratio": -0.5220702886581421,
      "logits/chosen": -0.9312499761581421,
      "logits/rejected": -0.838085949420929,
      "logps/chosen": -0.5611327886581421,
      "logps/rejected": -1.053320288658142,
      "loss": 0.8492,
      "nll_loss": 0.787792980670929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02805175818502903,
      "rewards/margins": 0.02460327185690403,
      "rewards/rejected": -0.05265503004193306,
      "step": 6090
    },
    {
      "epoch": 0.44439587658907953,
      "grad_norm": 1.7900057441287156,
      "learning_rate": 6.4018439966448e-07,
      "log_odds_chosen": 1.026269555091858,
      "log_odds_ratio": -0.4630371034145355,
      "logits/chosen": -0.9296875,
      "logits/rejected": -0.8505859375,
      "logps/chosen": -0.5040038824081421,
      "logps/rejected": -1.113671898841858,
      "loss": 0.8185,
      "nll_loss": 0.7093750238418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02519531175494194,
      "rewards/margins": 0.03051147423684597,
      "rewards/rejected": -0.05570068210363388,
      "step": 6100
    },
    {
      "epoch": 0.4451243944195534,
      "grad_norm": 1.7492878723855092,
      "learning_rate": 6.396603026469003e-07,
      "log_odds_chosen": 1.177148461341858,
      "log_odds_ratio": -0.4356445372104645,
      "logits/chosen": -0.982421875,
      "logits/rejected": -0.892578125,
      "logps/chosen": -0.5546875,
      "logps/rejected": -1.2531249523162842,
      "loss": 0.8065,
      "nll_loss": 0.799609363079071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0277099609375,
      "rewards/margins": 0.034912109375,
      "rewards/rejected": -0.06265868991613388,
      "step": 6110
    },
    {
      "epoch": 0.4458529122500273,
      "grad_norm": 2.10744380327657,
      "learning_rate": 6.39137490706142e-07,
      "log_odds_chosen": 0.9112304449081421,
      "log_odds_ratio": -0.49560546875,
      "logits/chosen": -0.962890625,
      "logits/rejected": -0.842968761920929,
      "logps/chosen": -0.58984375,
      "logps/rejected": -1.118554711341858,
      "loss": 0.8431,
      "nll_loss": 0.879687488079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02948608435690403,
      "rewards/margins": 0.026457976549863815,
      "rewards/rejected": -0.05592041090130806,
      "step": 6120
    },
    {
      "epoch": 0.4465814300805012,
      "grad_norm": 2.0167643478884925,
      "learning_rate": 6.386159585991317e-07,
      "log_odds_chosen": 1.010839819908142,
      "log_odds_ratio": -0.47236329317092896,
      "logits/chosen": -0.927929699420929,
      "logits/rejected": -0.8330078125,
      "logps/chosen": -0.550097644329071,
      "logps/rejected": -1.141210913658142,
      "loss": 0.8234,
      "nll_loss": 0.8121093511581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02752075158059597,
      "rewards/margins": 0.029502104967832565,
      "rewards/rejected": -0.05704345554113388,
      "step": 6130
    },
    {
      "epoch": 0.4473099479109751,
      "grad_norm": 2.188500363305756,
      "learning_rate": 6.380957011126949e-07,
      "log_odds_chosen": 0.8973633050918579,
      "log_odds_ratio": -0.48369139432907104,
      "logits/chosen": -0.9253906011581421,
      "logits/rejected": -0.843554675579071,
      "logps/chosen": -0.542187511920929,
      "logps/rejected": -1.0498046875,
      "loss": 0.8203,
      "nll_loss": 0.801562488079071,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02710571326315403,
      "rewards/margins": 0.02537841722369194,
      "rewards/rejected": -0.05245361477136612,
      "step": 6140
    },
    {
      "epoch": 0.448038465741449,
      "grad_norm": 1.637568717260621,
      "learning_rate": 6.375767130633382e-07,
      "log_odds_chosen": 0.878955066204071,
      "log_odds_ratio": -0.5179687738418579,
      "logits/chosen": -0.9566406011581421,
      "logits/rejected": -0.856640636920929,
      "logps/chosen": -0.593066394329071,
      "logps/rejected": -1.1462891101837158,
      "loss": 0.8286,
      "nll_loss": 0.8330078125,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02963867224752903,
      "rewards/margins": 0.027678679674863815,
      "rewards/rejected": -0.05731201171875,
      "step": 6150
    },
    {
      "epoch": 0.4487669835719229,
      "grad_norm": 1.751177302191039,
      "learning_rate": 6.370589892970319e-07,
      "log_odds_chosen": 1.16162109375,
      "log_odds_ratio": -0.44145506620407104,
      "logits/chosen": -0.9755859375,
      "logits/rejected": -0.8480468988418579,
      "logps/chosen": -0.523242175579071,
      "logps/rejected": -1.2228515148162842,
      "loss": 0.8008,
      "nll_loss": 0.7705078125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02617187425494194,
      "rewards/margins": 0.03500061109662056,
      "rewards/rejected": -0.06113281100988388,
      "step": 6160
    },
    {
      "epoch": 0.4494955014023968,
      "grad_norm": 2.7835724045568826,
      "learning_rate": 6.365425246889947e-07,
      "log_odds_chosen": 0.881152331829071,
      "log_odds_ratio": -0.4986328184604645,
      "logits/chosen": -0.9146484136581421,
      "logits/rejected": -0.848828136920929,
      "logps/chosen": -0.5503906011581421,
      "logps/rejected": -1.0421874523162842,
      "loss": 0.8264,
      "nll_loss": 0.8011718988418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02751464769244194,
      "rewards/margins": 0.02458953857421875,
      "rewards/rejected": -0.05205688625574112,
      "step": 6170
    },
    {
      "epoch": 0.4502240192328707,
      "grad_norm": 1.5077224635060322,
      "learning_rate": 6.360273141434806e-07,
      "log_odds_chosen": 0.8873046636581421,
      "log_odds_ratio": -0.47880858182907104,
      "logits/chosen": -0.951367199420929,
      "logits/rejected": -0.8550781011581421,
      "logps/chosen": -0.5611327886581421,
      "logps/rejected": -1.1171875,
      "loss": 0.8032,
      "nll_loss": 0.7197265625,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02803955040872097,
      "rewards/margins": 0.02778320387005806,
      "rewards/rejected": -0.05582275241613388,
      "step": 6180
    },
    {
      "epoch": 0.45095253706334465,
      "grad_norm": 1.7331350743617022,
      "learning_rate": 6.355133525935684e-07,
      "log_odds_chosen": 0.84466552734375,
      "log_odds_ratio": -0.549853503704071,
      "logits/chosen": -0.9449218511581421,
      "logits/rejected": -0.839062511920929,
      "logps/chosen": -0.583300769329071,
      "logps/rejected": -1.0916016101837158,
      "loss": 0.8165,
      "nll_loss": 0.7701171636581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02915649488568306,
      "rewards/margins": 0.02542266808450222,
      "rewards/rejected": -0.0545654296875,
      "step": 6190
    },
    {
      "epoch": 0.45168105489381855,
      "grad_norm": 2.0891687501608436,
      "learning_rate": 6.350006350009525e-07,
      "log_odds_chosen": 1.0161621570587158,
      "log_odds_ratio": -0.4615234434604645,
      "logits/chosen": -0.945507824420929,
      "logits/rejected": -0.852734386920929,
      "logps/chosen": -0.5707031488418579,
      "logps/rejected": -1.193750023841858,
      "loss": 0.8146,
      "nll_loss": 0.7601562738418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.028533935546875,
      "rewards/margins": 0.0311431884765625,
      "rewards/rejected": -0.05969848483800888,
      "step": 6200
    },
    {
      "epoch": 0.45240957272429244,
      "grad_norm": 1.9432831974957376,
      "learning_rate": 6.344891563557342e-07,
      "log_odds_chosen": 1.0848388671875,
      "log_odds_ratio": -0.44853514432907104,
      "logits/chosen": -0.9310547113418579,
      "logits/rejected": -0.832226574420929,
      "logps/chosen": -0.517773449420929,
      "logps/rejected": -1.1808593273162842,
      "loss": 0.8451,
      "nll_loss": 0.8179687261581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02590331993997097,
      "rewards/margins": 0.03309173509478569,
      "rewards/rejected": -0.05897216871380806,
      "step": 6210
    },
    {
      "epoch": 0.45313809055476634,
      "grad_norm": 1.4968890088118079,
      "learning_rate": 6.339789116762172e-07,
      "log_odds_chosen": 1.027929663658142,
      "log_odds_ratio": -0.4618164002895355,
      "logits/chosen": -0.9125000238418579,
      "logits/rejected": -0.813671886920929,
      "logps/chosen": -0.5526367425918579,
      "logps/rejected": -1.1746094226837158,
      "loss": 0.8119,
      "nll_loss": 0.806835949420929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02764282189309597,
      "rewards/margins": 0.0310211181640625,
      "rewards/rejected": -0.05872802808880806,
      "step": 6220
    },
    {
      "epoch": 0.45386660838524023,
      "grad_norm": 1.7089767689342783,
      "learning_rate": 6.334698960087037e-07,
      "log_odds_chosen": 0.842089831829071,
      "log_odds_ratio": -0.49638670682907104,
      "logits/chosen": -0.990429699420929,
      "logits/rejected": -0.8843749761581421,
      "logps/chosen": -0.5459960699081421,
      "logps/rejected": -1.0164062976837158,
      "loss": 0.8139,
      "nll_loss": 0.7994140386581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02730712853372097,
      "rewards/margins": 0.02351837232708931,
      "rewards/rejected": -0.05083007737994194,
      "step": 6230
    },
    {
      "epoch": 0.45459512621571413,
      "grad_norm": 1.9185389955171805,
      "learning_rate": 6.329621044272917e-07,
      "log_odds_chosen": 1.13330078125,
      "log_odds_ratio": -0.4349609315395355,
      "logits/chosen": -0.958789050579071,
      "logits/rejected": -0.8326171636581421,
      "logps/chosen": -0.530468761920929,
      "logps/rejected": -1.1904296875,
      "loss": 0.817,
      "nll_loss": 0.8414062261581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02652587927877903,
      "rewards/margins": 0.032978057861328125,
      "rewards/rejected": -0.05951537936925888,
      "step": 6240
    },
    {
      "epoch": 0.455323644046188,
      "grad_norm": 2.275129725292274,
      "learning_rate": 6.324555320336758e-07,
      "log_odds_chosen": 0.997998058795929,
      "log_odds_ratio": -0.48945313692092896,
      "logits/chosen": -0.9378906488418579,
      "logits/rejected": -0.8568359613418579,
      "logps/chosen": -0.569628894329071,
      "logps/rejected": -1.1541016101837158,
      "loss": 0.8083,
      "nll_loss": 0.752734363079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02847900427877903,
      "rewards/margins": 0.02923278883099556,
      "rewards/rejected": -0.05769043043255806,
      "step": 6250
    },
    {
      "epoch": 0.4560521618766619,
      "grad_norm": 2.3188964933281566,
      "learning_rate": 6.319501739569483e-07,
      "log_odds_chosen": 0.9769531488418579,
      "log_odds_ratio": -0.48554688692092896,
      "logits/chosen": -0.9449218511581421,
      "logits/rejected": -0.893750011920929,
      "logps/chosen": -0.53515625,
      "logps/rejected": -1.1003906726837158,
      "loss": 0.8233,
      "nll_loss": 0.759570300579071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02677001990377903,
      "rewards/margins": 0.02823944017291069,
      "rewards/rejected": -0.05499267578125,
      "step": 6260
    },
    {
      "epoch": 0.4567806797071358,
      "grad_norm": 1.9992934258409152,
      "learning_rate": 6.31446025353402e-07,
      "log_odds_chosen": 0.8736327886581421,
      "log_odds_ratio": -0.49687498807907104,
      "logits/chosen": -0.9517577886581421,
      "logits/rejected": -0.8636718988418579,
      "logps/chosen": -0.5453125238418579,
      "logps/rejected": -1.0535156726837158,
      "loss": 0.8219,
      "nll_loss": 0.803906261920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02724609337747097,
      "rewards/margins": 0.02537689171731472,
      "rewards/rejected": -0.05263672024011612,
      "step": 6270
    },
    {
      "epoch": 0.4575091975376097,
      "grad_norm": 1.8388950520765956,
      "learning_rate": 6.30943081406336e-07,
      "log_odds_chosen": 0.966064453125,
      "log_odds_ratio": -0.48193359375,
      "logits/chosen": -0.9486328363418579,
      "logits/rejected": -0.868945300579071,
      "logps/chosen": -0.5672851800918579,
      "logps/rejected": -1.109765648841858,
      "loss": 0.8195,
      "nll_loss": 0.762988269329071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02837524376809597,
      "rewards/margins": 0.027135467156767845,
      "rewards/rejected": -0.05552368238568306,
      "step": 6280
    },
    {
      "epoch": 0.4582377153680836,
      "grad_norm": 1.9967415345404995,
      "learning_rate": 6.304413373258618e-07,
      "log_odds_chosen": 0.802978515625,
      "log_odds_ratio": -0.5341796875,
      "logits/chosen": -0.965039074420929,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.556640625,
      "logps/rejected": -1.031640648841858,
      "loss": 0.8033,
      "nll_loss": 0.794921875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02782592736184597,
      "rewards/margins": 0.02370910719037056,
      "rewards/rejected": -0.05152587965130806,
      "step": 6290
    },
    {
      "epoch": 0.45896623319855756,
      "grad_norm": 1.823206768460473,
      "learning_rate": 6.29940788348712e-07,
      "log_odds_chosen": 0.761279284954071,
      "log_odds_ratio": -0.5467773675918579,
      "logits/chosen": -0.926953136920929,
      "logits/rejected": -0.843945324420929,
      "logps/chosen": -0.568066418170929,
      "logps/rejected": -1.015039086341858,
      "loss": 0.8058,
      "nll_loss": 0.7744140625,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.02843017503619194,
      "rewards/margins": 0.022346878424286842,
      "rewards/rejected": -0.05076904222369194,
      "step": 6300
    },
    {
      "epoch": 0.45969475102903146,
      "grad_norm": 1.7976668073636033,
      "learning_rate": 6.294414297380508e-07,
      "log_odds_chosen": 0.9678710699081421,
      "log_odds_ratio": -0.49951171875,
      "logits/chosen": -0.986328125,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -0.568066418170929,
      "logps/rejected": -1.1115233898162842,
      "loss": 0.8139,
      "nll_loss": 0.7586914300918579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02836913987994194,
      "rewards/margins": 0.02717437781393528,
      "rewards/rejected": -0.05555419996380806,
      "step": 6310
    },
    {
      "epoch": 0.46042326885950535,
      "grad_norm": 3.7709670731342726,
      "learning_rate": 6.289432567832845e-07,
      "log_odds_chosen": 0.9629882574081421,
      "log_odds_ratio": -0.44140625,
      "logits/chosen": -0.944140613079071,
      "logits/rejected": -0.8412109613418579,
      "logps/chosen": -0.5238281488418579,
      "logps/rejected": -1.0419921875,
      "loss": 0.8421,
      "nll_loss": 0.7835937738418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02622070349752903,
      "rewards/margins": 0.025909041985869408,
      "rewards/rejected": -0.05210571363568306,
      "step": 6320
    },
    {
      "epoch": 0.46115178668997925,
      "grad_norm": 1.829848505033585,
      "learning_rate": 6.284462647998764e-07,
      "log_odds_chosen": 1.0010254383087158,
      "log_odds_ratio": -0.42524415254592896,
      "logits/chosen": -1.0197265148162842,
      "logits/rejected": -0.8783203363418579,
      "logps/chosen": -0.537402331829071,
      "logps/rejected": -1.115820288658142,
      "loss": 0.8186,
      "nll_loss": 0.791015625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02686157263815403,
      "rewards/margins": 0.02889404259622097,
      "rewards/rejected": -0.0557861328125,
      "step": 6330
    },
    {
      "epoch": 0.46188030452045314,
      "grad_norm": 1.9041409034288237,
      "learning_rate": 6.279504491291604e-07,
      "log_odds_chosen": 1.0001952648162842,
      "log_odds_ratio": -0.4781250059604645,
      "logits/chosen": -0.9769531488418579,
      "logits/rejected": -0.869335949420929,
      "logps/chosen": -0.542773425579071,
      "logps/rejected": -1.1814453601837158,
      "loss": 0.8093,
      "nll_loss": 0.793749988079071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02712402306497097,
      "rewards/margins": 0.03195514529943466,
      "rewards/rejected": -0.05903930589556694,
      "step": 6340
    },
    {
      "epoch": 0.46260882235092704,
      "grad_norm": 1.9184115970773503,
      "learning_rate": 6.274558051381586e-07,
      "log_odds_chosen": 0.753466784954071,
      "log_odds_ratio": -0.532470703125,
      "logits/chosen": -0.9429687261581421,
      "logits/rejected": -0.848828136920929,
      "logps/chosen": -0.563281238079071,
      "logps/rejected": -1.020117163658142,
      "loss": 0.8279,
      "nll_loss": 0.8080078363418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02815551683306694,
      "rewards/margins": 0.02286529541015625,
      "rewards/rejected": -0.05098877102136612,
      "step": 6350
    },
    {
      "epoch": 0.46333734018140094,
      "grad_norm": 1.735584835089299,
      "learning_rate": 6.26962328219399e-07,
      "log_odds_chosen": 1.037109375,
      "log_odds_ratio": -0.4539550840854645,
      "logits/chosen": -0.9365234375,
      "logits/rejected": -0.840039074420929,
      "logps/chosen": -0.5425781011581421,
      "logps/rejected": -1.129296898841858,
      "loss": 0.8221,
      "nll_loss": 0.7685546875,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02709350548684597,
      "rewards/margins": 0.02935180626809597,
      "rewards/rejected": -0.05645751953125,
      "step": 6360
    },
    {
      "epoch": 0.46406585801187483,
      "grad_norm": 1.8510736762911082,
      "learning_rate": 6.264700137907352e-07,
      "log_odds_chosen": 1.054785132408142,
      "log_odds_ratio": -0.4278320372104645,
      "logits/chosen": -0.9482421875,
      "logits/rejected": -0.8646484613418579,
      "logps/chosen": -0.5230468511581421,
      "logps/rejected": -1.1062500476837158,
      "loss": 0.8169,
      "nll_loss": 0.81298828125,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02614135667681694,
      "rewards/margins": 0.029144287109375,
      "rewards/rejected": -0.05527343600988388,
      "step": 6370
    },
    {
      "epoch": 0.46479437584234873,
      "grad_norm": 2.2196550324968656,
      "learning_rate": 6.259788572951681e-07,
      "log_odds_chosen": 0.916748046875,
      "log_odds_ratio": -0.48613280057907104,
      "logits/chosen": -0.9439452886581421,
      "logits/rejected": -0.8363281488418579,
      "logps/chosen": -0.56298828125,
      "logps/rejected": -1.095703125,
      "loss": 0.8219,
      "nll_loss": 0.752734363079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02817382849752903,
      "rewards/margins": 0.026592254638671875,
      "rewards/rejected": -0.05474853515625,
      "step": 6380
    },
    {
      "epoch": 0.4655228936728226,
      "grad_norm": 1.5979877120420798,
      "learning_rate": 6.25488854200668e-07,
      "log_odds_chosen": 0.7856689691543579,
      "log_odds_ratio": -0.504101574420929,
      "logits/chosen": -0.9466797113418579,
      "logits/rejected": -0.8392578363418579,
      "logps/chosen": -0.56103515625,
      "logps/rejected": -1.007421851158142,
      "loss": 0.8161,
      "nll_loss": 0.8160156011581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02807006798684597,
      "rewards/margins": 0.02231292799115181,
      "rewards/rejected": -0.05039062350988388,
      "step": 6390
    },
    {
      "epoch": 0.4662514115032965,
      "grad_norm": 1.840552516967059,
      "learning_rate": 6.25e-07,
      "log_odds_chosen": 0.967041015625,
      "log_odds_ratio": -0.4812988340854645,
      "logits/chosen": -0.982617199420929,
      "logits/rejected": -0.8828125,
      "logps/chosen": -0.5562499761581421,
      "logps/rejected": -1.1349608898162842,
      "loss": 0.7953,
      "nll_loss": 0.7718750238418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02778930589556694,
      "rewards/margins": 0.028980636969208717,
      "rewards/rejected": -0.05673827975988388,
      "step": 6400
    },
    {
      "epoch": 0.46697992933377047,
      "grad_norm": 1.7443822908759388,
      "learning_rate": 6.24512290210549e-07,
      "log_odds_chosen": 0.9513183832168579,
      "log_odds_ratio": -0.47968751192092896,
      "logits/chosen": -0.967578113079071,
      "logits/rejected": -0.885546863079071,
      "logps/chosen": -0.558398425579071,
      "logps/rejected": -1.085546851158142,
      "loss": 0.8288,
      "nll_loss": 0.848437488079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02792968787252903,
      "rewards/margins": 0.02636413648724556,
      "rewards/rejected": -0.05425415188074112,
      "step": 6410
    },
    {
      "epoch": 0.46770844716424437,
      "grad_norm": 1.8387076530147108,
      "learning_rate": 6.240257203741474e-07,
      "log_odds_chosen": 0.8374999761581421,
      "log_odds_ratio": -0.518261730670929,
      "logits/chosen": -0.9673827886581421,
      "logits/rejected": -0.8675781488418579,
      "logps/chosen": -0.529296875,
      "logps/rejected": -1.0203125476837158,
      "loss": 0.8074,
      "nll_loss": 0.780468761920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02648315392434597,
      "rewards/margins": 0.0245208740234375,
      "rewards/rejected": -0.05098877102136612,
      "step": 6420
    },
    {
      "epoch": 0.46843696499471826,
      "grad_norm": 1.8224365182084539,
      "learning_rate": 6.235402860569038e-07,
      "log_odds_chosen": 0.96728515625,
      "log_odds_ratio": -0.468994140625,
      "logits/chosen": -0.947265625,
      "logits/rejected": -0.8218749761581421,
      "logps/chosen": -0.550097644329071,
      "logps/rejected": -1.1398437023162842,
      "loss": 0.8241,
      "nll_loss": 0.778515636920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02752685546875,
      "rewards/margins": 0.02942657470703125,
      "rewards/rejected": -0.05695800855755806,
      "step": 6430
    },
    {
      "epoch": 0.46916548282519216,
      "grad_norm": 1.601316189941807,
      "learning_rate": 6.230559828490335e-07,
      "log_odds_chosen": 1.0169188976287842,
      "log_odds_ratio": -0.4646972715854645,
      "logits/chosen": -0.909960925579071,
      "logits/rejected": -0.798046886920929,
      "logps/chosen": -0.564453125,
      "logps/rejected": -1.136132836341858,
      "loss": 0.8065,
      "nll_loss": 0.7876952886581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02819213829934597,
      "rewards/margins": 0.02861480787396431,
      "rewards/rejected": -0.05679931491613388,
      "step": 6440
    },
    {
      "epoch": 0.46989400065566606,
      "grad_norm": 1.8467886584489281,
      "learning_rate": 6.225728063646904e-07,
      "log_odds_chosen": 1.136621117591858,
      "log_odds_ratio": -0.4351562559604645,
      "logits/chosen": -0.943359375,
      "logits/rejected": -0.830859363079071,
      "logps/chosen": -0.560546875,
      "logps/rejected": -1.2781250476837158,
      "loss": 0.8226,
      "nll_loss": 0.889843761920929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02802734449505806,
      "rewards/margins": 0.03583373874425888,
      "rewards/rejected": -0.06386718899011612,
      "step": 6450
    },
    {
      "epoch": 0.47062251848613995,
      "grad_norm": 2.306170231927496,
      "learning_rate": 6.220907522417995e-07,
      "log_odds_chosen": 1.0652344226837158,
      "log_odds_ratio": -0.4518066346645355,
      "logits/chosen": -0.969531238079071,
      "logits/rejected": -0.858203113079071,
      "logps/chosen": -0.5455077886581421,
      "logps/rejected": -1.1853516101837158,
      "loss": 0.8056,
      "nll_loss": 0.8267577886581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02728271484375,
      "rewards/margins": 0.03199920803308487,
      "rewards/rejected": -0.059326171875,
      "step": 6460
    },
    {
      "epoch": 0.47135103631661385,
      "grad_norm": 1.9622084558743247,
      "learning_rate": 6.216098161418919e-07,
      "log_odds_chosen": 1.0615234375,
      "log_odds_ratio": -0.46577149629592896,
      "logits/chosen": -0.9468749761581421,
      "logits/rejected": -0.890429675579071,
      "logps/chosen": -0.56689453125,
      "logps/rejected": -1.230859398841858,
      "loss": 0.8281,
      "nll_loss": 0.7796875238418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0283203125,
      "rewards/margins": 0.033203125,
      "rewards/rejected": -0.06156005710363388,
      "step": 6470
    },
    {
      "epoch": 0.47207955414708774,
      "grad_norm": 1.7903366022760525,
      "learning_rate": 6.211299937499417e-07,
      "log_odds_chosen": 1.078369140625,
      "log_odds_ratio": -0.4476074278354645,
      "logits/chosen": -0.977734386920929,
      "logits/rejected": -0.889453113079071,
      "logps/chosen": -0.519238293170929,
      "logps/rejected": -1.176367163658142,
      "loss": 0.8071,
      "nll_loss": 0.737500011920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02594604529440403,
      "rewards/margins": 0.032907865941524506,
      "rewards/rejected": -0.05888061597943306,
      "step": 6480
    },
    {
      "epoch": 0.47280807197756164,
      "grad_norm": 3.72406476669939,
      "learning_rate": 6.206512807742013e-07,
      "log_odds_chosen": 0.9952148199081421,
      "log_odds_ratio": -0.45625001192092896,
      "logits/chosen": -0.978320300579071,
      "logits/rejected": -0.859179675579071,
      "logps/chosen": -0.55078125,
      "logps/rejected": -1.156835913658142,
      "loss": 0.8106,
      "nll_loss": 0.7154296636581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02754516527056694,
      "rewards/margins": 0.03028106689453125,
      "rewards/rejected": -0.05778808519244194,
      "step": 6490
    },
    {
      "epoch": 0.47353658980803554,
      "grad_norm": 2.882817390879355,
      "learning_rate": 6.201736729460424e-07,
      "log_odds_chosen": 0.8067626953125,
      "log_odds_ratio": -0.543261706829071,
      "logits/chosen": -0.9691406488418579,
      "logits/rejected": -0.864062488079071,
      "logps/chosen": -0.591601550579071,
      "logps/rejected": -1.0734374523162842,
      "loss": 0.7705,
      "nll_loss": 0.804882824420929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02958984300494194,
      "rewards/margins": 0.02407226525247097,
      "rewards/rejected": -0.05364990234375,
      "step": 6500
    },
    {
      "epoch": 0.47426510763850943,
      "grad_norm": 1.77260823273428,
      "learning_rate": 6.196971660197946e-07,
      "log_odds_chosen": 0.8583129644393921,
      "log_odds_ratio": -0.4981445372104645,
      "logits/chosen": -0.958203136920929,
      "logits/rejected": -0.867382824420929,
      "logps/chosen": -0.63330078125,
      "logps/rejected": -1.1533203125,
      "loss": 0.8136,
      "nll_loss": 0.892773449420929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.03167724609375,
      "rewards/margins": 0.026024628430604935,
      "rewards/rejected": -0.05770263820886612,
      "step": 6510
    },
    {
      "epoch": 0.4749936254689833,
      "grad_norm": 1.961757398324331,
      "learning_rate": 6.192217557725875e-07,
      "log_odds_chosen": 0.9103027582168579,
      "log_odds_ratio": -0.48681640625,
      "logits/chosen": -0.96875,
      "logits/rejected": -0.8373047113418579,
      "logps/chosen": -0.566699206829071,
      "logps/rejected": -1.110742211341858,
      "loss": 0.8194,
      "nll_loss": 0.7822265625,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02830810472369194,
      "rewards/margins": 0.027206802740693092,
      "rewards/rejected": -0.05552978441119194,
      "step": 6520
    },
    {
      "epoch": 0.4757221432994573,
      "grad_norm": 1.821649796776313,
      "learning_rate": 6.187474380041937e-07,
      "log_odds_chosen": 1.199804663658142,
      "log_odds_ratio": -0.42431640625,
      "logits/chosen": -0.954882800579071,
      "logits/rejected": -0.8218749761581421,
      "logps/chosen": -0.5673828125,
      "logps/rejected": -1.284570336341858,
      "loss": 0.8061,
      "nll_loss": 0.77734375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02836303785443306,
      "rewards/margins": 0.03593902662396431,
      "rewards/rejected": -0.06424560397863388,
      "step": 6530
    },
    {
      "epoch": 0.4764506611299312,
      "grad_norm": 1.7947652404817085,
      "learning_rate": 6.182742085368725e-07,
      "log_odds_chosen": 1.0986328125,
      "log_odds_ratio": -0.4496093690395355,
      "logits/chosen": -0.9400390386581421,
      "logits/rejected": -0.8285156488418579,
      "logps/chosen": -0.540234386920929,
      "logps/rejected": -1.216406226158142,
      "loss": 0.8181,
      "nll_loss": 0.7762695550918579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02701110765337944,
      "rewards/margins": 0.03385009616613388,
      "rewards/rejected": -0.06083984300494194,
      "step": 6540
    },
    {
      "epoch": 0.47717917896040507,
      "grad_norm": 2.054392486077471,
      "learning_rate": 6.178020632152155e-07,
      "log_odds_chosen": 0.897705078125,
      "log_odds_ratio": -0.512402355670929,
      "logits/chosen": -0.9117187261581421,
      "logits/rejected": -0.8373047113418579,
      "logps/chosen": -0.529003918170929,
      "logps/rejected": -1.0498046875,
      "loss": 0.8227,
      "nll_loss": 0.800976574420929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02644653245806694,
      "rewards/margins": 0.026018524542450905,
      "rewards/rejected": -0.05247802659869194,
      "step": 6550
    },
    {
      "epoch": 0.47790769679087897,
      "grad_norm": 1.9453261205243393,
      "learning_rate": 6.173309979059936e-07,
      "log_odds_chosen": 1.05419921875,
      "log_odds_ratio": -0.45732420682907104,
      "logits/chosen": -0.9244140386581421,
      "logits/rejected": -0.8158203363418579,
      "logps/chosen": -0.513671875,
      "logps/rejected": -1.1144530773162842,
      "loss": 0.7911,
      "nll_loss": 0.805468738079071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02568359300494194,
      "rewards/margins": 0.03004150465130806,
      "rewards/rejected": -0.05568237230181694,
      "step": 6560
    },
    {
      "epoch": 0.47863621462135286,
      "grad_norm": 1.8807287970405302,
      "learning_rate": 6.168610084980044e-07,
      "log_odds_chosen": 0.9461914300918579,
      "log_odds_ratio": -0.4574218690395355,
      "logits/chosen": -0.9330078363418579,
      "logits/rejected": -0.8451172113418579,
      "logps/chosen": -0.529492199420929,
      "logps/rejected": -1.084570288658142,
      "loss": 0.7892,
      "nll_loss": 0.6939452886581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.026458740234375,
      "rewards/margins": 0.02775268629193306,
      "rewards/rejected": -0.05421142652630806,
      "step": 6570
    },
    {
      "epoch": 0.47936473245182676,
      "grad_norm": 2.855025720987765,
      "learning_rate": 6.163920909019224e-07,
      "log_odds_chosen": 0.998608410358429,
      "log_odds_ratio": -0.490966796875,
      "logits/chosen": -0.9429687261581421,
      "logits/rejected": -0.8257812261581421,
      "logps/chosen": -0.54541015625,
      "logps/rejected": -1.163476586341858,
      "loss": 0.8002,
      "nll_loss": 0.720507800579071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02728881873190403,
      "rewards/margins": 0.03092346154153347,
      "rewards/rejected": -0.05820312350988388,
      "step": 6580
    },
    {
      "epoch": 0.48009325028230065,
      "grad_norm": 1.6504225351741613,
      "learning_rate": 6.159242410501489e-07,
      "log_odds_chosen": 0.956347644329071,
      "log_odds_ratio": -0.49101561307907104,
      "logits/chosen": -0.9583984613418579,
      "logits/rejected": -0.8519531488418579,
      "logps/chosen": -0.5443359613418579,
      "logps/rejected": -1.064453125,
      "loss": 0.8084,
      "nll_loss": 0.82421875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02722778357565403,
      "rewards/margins": 0.0260009765625,
      "rewards/rejected": -0.05324096605181694,
      "step": 6590
    },
    {
      "epoch": 0.48082176811277455,
      "grad_norm": 1.7833533799829493,
      "learning_rate": 6.154574548966637e-07,
      "log_odds_chosen": 1.0413818359375,
      "log_odds_ratio": -0.4811035096645355,
      "logits/chosen": -0.935351550579071,
      "logits/rejected": -0.8310546875,
      "logps/chosen": -0.5718749761581421,
      "logps/rejected": -1.196679711341858,
      "loss": 0.7996,
      "nll_loss": 0.76953125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02860107459127903,
      "rewards/margins": 0.031223297119140625,
      "rewards/rejected": -0.05980224534869194,
      "step": 6600
    },
    {
      "epoch": 0.48155028594324845,
      "grad_norm": 1.8197645892079333,
      "learning_rate": 6.149917284168789e-07,
      "log_odds_chosen": 0.9093261957168579,
      "log_odds_ratio": -0.4716796875,
      "logits/chosen": -0.9144531488418579,
      "logits/rejected": -0.8515625,
      "logps/chosen": -0.5166015625,
      "logps/rejected": -1.005468726158142,
      "loss": 0.8185,
      "nll_loss": 0.740429699420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02581787109375,
      "rewards/margins": 0.02448425255715847,
      "rewards/rejected": -0.05029296875,
      "step": 6610
    },
    {
      "epoch": 0.48227880377372234,
      "grad_norm": 1.9040420129737095,
      "learning_rate": 6.145270576074922e-07,
      "log_odds_chosen": 0.9396728277206421,
      "log_odds_ratio": -0.46577149629592896,
      "logits/chosen": -0.940234363079071,
      "logits/rejected": -0.836718738079071,
      "logps/chosen": -0.555957019329071,
      "logps/rejected": -1.1083984375,
      "loss": 0.8016,
      "nll_loss": 0.800976574420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02780761756002903,
      "rewards/margins": 0.027660369873046875,
      "rewards/rejected": -0.05549316480755806,
      "step": 6620
    },
    {
      "epoch": 0.48300732160419624,
      "grad_norm": 1.770919641663557,
      "learning_rate": 6.140634384863435e-07,
      "log_odds_chosen": 1.2160155773162842,
      "log_odds_ratio": -0.4169921875,
      "logits/chosen": -0.939648449420929,
      "logits/rejected": -0.8451172113418579,
      "logps/chosen": -0.523242175579071,
      "logps/rejected": -1.2736327648162842,
      "loss": 0.8004,
      "nll_loss": 0.7557617425918579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02616577222943306,
      "rewards/margins": 0.03752746433019638,
      "rewards/rejected": -0.06369628757238388,
      "step": 6630
    },
    {
      "epoch": 0.4837358394346702,
      "grad_norm": 1.768679538730458,
      "learning_rate": 6.136008670922701e-07,
      "log_odds_chosen": 0.929028332233429,
      "log_odds_ratio": -0.5140625238418579,
      "logits/chosen": -0.9576171636581421,
      "logits/rejected": -0.828125,
      "logps/chosen": -0.577832043170929,
      "logps/rejected": -1.1640625,
      "loss": 0.8117,
      "nll_loss": 0.8453124761581421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02889404259622097,
      "rewards/margins": 0.02933349646627903,
      "rewards/rejected": -0.05819091945886612,
      "step": 6640
    },
    {
      "epoch": 0.4844643572651441,
      "grad_norm": 1.845567828924711,
      "learning_rate": 6.131393394849658e-07,
      "log_odds_chosen": 1.0359008312225342,
      "log_odds_ratio": -0.48896485567092896,
      "logits/chosen": -0.951953113079071,
      "logits/rejected": -0.889843761920929,
      "logps/chosen": -0.5352538824081421,
      "logps/rejected": -1.123632788658142,
      "loss": 0.8056,
      "nll_loss": 0.783984363079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02678832970559597,
      "rewards/margins": 0.02943267859518528,
      "rewards/rejected": -0.05620727688074112,
      "step": 6650
    },
    {
      "epoch": 0.485192875095618,
      "grad_norm": 1.9060590466283178,
      "learning_rate": 6.126788517448398e-07,
      "log_odds_chosen": 1.287500023841858,
      "log_odds_ratio": -0.36279296875,
      "logits/chosen": -0.9580078125,
      "logits/rejected": -0.848828136920929,
      "logps/chosen": -0.539257824420929,
      "logps/rejected": -1.3181641101837158,
      "loss": 0.799,
      "nll_loss": 0.763867199420929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.02695922926068306,
      "rewards/margins": 0.03899536281824112,
      "rewards/rejected": -0.06600341945886612,
      "step": 6660
    },
    {
      "epoch": 0.4859213929260919,
      "grad_norm": 2.5386615984533556,
      "learning_rate": 6.122193999728761e-07,
      "log_odds_chosen": 1.0808594226837158,
      "log_odds_ratio": -0.47319334745407104,
      "logits/chosen": -0.97265625,
      "logits/rejected": -0.8726562261581421,
      "logps/chosen": -0.572460949420929,
      "logps/rejected": -1.244140625,
      "loss": 0.8005,
      "nll_loss": 0.7710937261581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.028594970703125,
      "rewards/margins": 0.033599853515625,
      "rewards/rejected": -0.06214599683880806,
      "step": 6670
    },
    {
      "epoch": 0.4866499107565658,
      "grad_norm": 1.7462957273390787,
      "learning_rate": 6.117609802904955e-07,
      "log_odds_chosen": 0.937255859375,
      "log_odds_ratio": -0.5205078125,
      "logits/chosen": -0.9408203363418579,
      "logits/rejected": -0.818164050579071,
      "logps/chosen": -0.607128918170929,
      "logps/rejected": -1.190039038658142,
      "loss": 0.8084,
      "nll_loss": 0.814453125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.03033447265625,
      "rewards/margins": 0.029205322265625,
      "rewards/rejected": -0.05954589694738388,
      "step": 6680
    },
    {
      "epoch": 0.48737842858703967,
      "grad_norm": 2.046763783294418,
      "learning_rate": 6.11303588839418e-07,
      "log_odds_chosen": 1.0314452648162842,
      "log_odds_ratio": -0.482666015625,
      "logits/chosen": -0.9525390863418579,
      "logits/rejected": -0.839648425579071,
      "logps/chosen": -0.539111316204071,
      "logps/rejected": -1.1218750476837158,
      "loss": 0.8056,
      "nll_loss": 0.7635742425918579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02695312537252903,
      "rewards/margins": 0.02914123609662056,
      "rewards/rejected": -0.05609130859375,
      "step": 6690
    },
    {
      "epoch": 0.48810694641751357,
      "grad_norm": 1.850897370099511,
      "learning_rate": 6.108472217815262e-07,
      "log_odds_chosen": 1.1369140148162842,
      "log_odds_ratio": -0.42573243379592896,
      "logits/chosen": -0.9566406011581421,
      "logits/rejected": -0.837890625,
      "logps/chosen": -0.55712890625,
      "logps/rejected": -1.21484375,
      "loss": 0.8159,
      "nll_loss": 0.7777343988418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02785644493997097,
      "rewards/margins": 0.03287506103515625,
      "rewards/rejected": -0.06070556491613388,
      "step": 6700
    },
    {
      "epoch": 0.48883546424798746,
      "grad_norm": 1.9379083770775454,
      "learning_rate": 6.103918752987302e-07,
      "log_odds_chosen": 1.007604956626892,
      "log_odds_ratio": -0.4493164122104645,
      "logits/chosen": -0.9677734375,
      "logits/rejected": -0.8404296636581421,
      "logps/chosen": -0.5879882574081421,
      "logps/rejected": -1.164453148841858,
      "loss": 0.8004,
      "nll_loss": 0.8306640386581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02939453162252903,
      "rewards/margins": 0.028807450085878372,
      "rewards/rejected": -0.05820312350988388,
      "step": 6710
    },
    {
      "epoch": 0.48956398207846136,
      "grad_norm": 2.6357780853135875,
      "learning_rate": 6.099375455928332e-07,
      "log_odds_chosen": 0.93701171875,
      "log_odds_ratio": -0.4925781190395355,
      "logits/chosen": -0.968945324420929,
      "logits/rejected": -0.8267577886581421,
      "logps/chosen": -0.567578136920929,
      "logps/rejected": -1.1349608898162842,
      "loss": 0.7991,
      "nll_loss": 0.8062499761581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02836303785443306,
      "rewards/margins": 0.028400421142578125,
      "rewards/rejected": -0.0567626953125,
      "step": 6720
    },
    {
      "epoch": 0.49029249990893525,
      "grad_norm": 2.2020837797051263,
      "learning_rate": 6.094842288853993e-07,
      "log_odds_chosen": 0.84765625,
      "log_odds_ratio": -0.51171875,
      "logits/chosen": -0.922070324420929,
      "logits/rejected": -0.83203125,
      "logps/chosen": -0.606738269329071,
      "logps/rejected": -1.0871093273162842,
      "loss": 0.7978,
      "nll_loss": 0.806445300579071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.03034668043255806,
      "rewards/margins": 0.02398071251809597,
      "rewards/rejected": -0.05430908128619194,
      "step": 6730
    },
    {
      "epoch": 0.49102101773940915,
      "grad_norm": 1.6219369863042739,
      "learning_rate": 6.090319214176199e-07,
      "log_odds_chosen": 0.9721435308456421,
      "log_odds_ratio": -0.4649414122104645,
      "logits/chosen": -0.9302734136581421,
      "logits/rejected": -0.8128906488418579,
      "logps/chosen": -0.5125976800918579,
      "logps/rejected": -1.068359375,
      "loss": 0.8162,
      "nll_loss": 0.7323242425918579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02563171461224556,
      "rewards/margins": 0.027783965691924095,
      "rewards/rejected": -0.05346069484949112,
      "step": 6740
    },
    {
      "epoch": 0.4917495355698831,
      "grad_norm": 1.7879626246956457,
      "learning_rate": 6.085806194501847e-07,
      "log_odds_chosen": 1.024658203125,
      "log_odds_ratio": -0.47607421875,
      "logits/chosen": -0.9859374761581421,
      "logits/rejected": -0.8695312738418579,
      "logps/chosen": -0.546875,
      "logps/rejected": -1.1708984375,
      "loss": 0.7995,
      "nll_loss": 0.7486327886581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02734985388815403,
      "rewards/margins": 0.0312042236328125,
      "rewards/rejected": -0.05850829929113388,
      "step": 6750
    },
    {
      "epoch": 0.492478053400357,
      "grad_norm": 2.0943125836539873,
      "learning_rate": 6.081303192631499e-07,
      "log_odds_chosen": 1.050390601158142,
      "log_odds_ratio": -0.47509765625,
      "logits/chosen": -0.9917968511581421,
      "logits/rejected": -0.871874988079071,
      "logps/chosen": -0.521191418170929,
      "logps/rejected": -1.1589844226837158,
      "loss": 0.8028,
      "nll_loss": 0.757031261920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.026031494140625,
      "rewards/margins": 0.03191222995519638,
      "rewards/rejected": -0.05795898288488388,
      "step": 6760
    },
    {
      "epoch": 0.4932065712308309,
      "grad_norm": 2.1757025441877227,
      "learning_rate": 6.076810171558113e-07,
      "log_odds_chosen": 0.9897216558456421,
      "log_odds_ratio": -0.4546875059604645,
      "logits/chosen": -0.9779297113418579,
      "logits/rejected": -0.885937511920929,
      "logps/chosen": -0.5609375238418579,
      "logps/rejected": -1.1320312023162842,
      "loss": 0.8013,
      "nll_loss": 0.816210925579071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02805786207318306,
      "rewards/margins": 0.02855682373046875,
      "rewards/rejected": -0.05659789964556694,
      "step": 6770
    },
    {
      "epoch": 0.4939350890613048,
      "grad_norm": 2.26396116577513,
      "learning_rate": 6.072327094465754e-07,
      "log_odds_chosen": 0.987597644329071,
      "log_odds_ratio": -0.508105456829071,
      "logits/chosen": -0.9095703363418579,
      "logits/rejected": -0.8226562738418579,
      "logps/chosen": -0.589550793170929,
      "logps/rejected": -1.2273437976837158,
      "loss": 0.8088,
      "nll_loss": 0.822949230670929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02948608435690403,
      "rewards/margins": 0.03191375732421875,
      "rewards/rejected": -0.06136474758386612,
      "step": 6780
    },
    {
      "epoch": 0.4946636068917787,
      "grad_norm": 2.124154827969808,
      "learning_rate": 6.067853924728325e-07,
      "log_odds_chosen": 1.006445288658142,
      "log_odds_ratio": -0.4684081971645355,
      "logits/chosen": -0.9375,
      "logits/rejected": -0.828125,
      "logps/chosen": -0.5609375238418579,
      "logps/rejected": -1.1408202648162842,
      "loss": 0.8319,
      "nll_loss": 0.8221679925918579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.028045654296875,
      "rewards/margins": 0.02897338941693306,
      "rewards/rejected": -0.05701904371380806,
      "step": 6790
    },
    {
      "epoch": 0.4953921247222526,
      "grad_norm": 1.5755326485218888,
      "learning_rate": 6.063390625908324e-07,
      "log_odds_chosen": 0.9500976800918579,
      "log_odds_ratio": -0.50732421875,
      "logits/chosen": -0.9117187261581421,
      "logits/rejected": -0.807421863079071,
      "logps/chosen": -0.553417980670929,
      "logps/rejected": -1.0968749523162842,
      "loss": 0.8017,
      "nll_loss": 0.7833007574081421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02767333947122097,
      "rewards/margins": 0.02719421312212944,
      "rewards/rejected": -0.05485229566693306,
      "step": 6800
    },
    {
      "epoch": 0.4961206425527265,
      "grad_norm": 1.8204391678253782,
      "learning_rate": 6.058937161755583e-07,
      "log_odds_chosen": 1.081945776939392,
      "log_odds_ratio": -0.4905761778354645,
      "logits/chosen": -0.967968761920929,
      "logits/rejected": -0.884765625,
      "logps/chosen": -0.547656238079071,
      "logps/rejected": -1.1865234375,
      "loss": 0.8195,
      "nll_loss": 0.8207031488418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02738647535443306,
      "rewards/margins": 0.03194427490234375,
      "rewards/rejected": -0.05933837965130806,
      "step": 6810
    },
    {
      "epoch": 0.49684916038320037,
      "grad_norm": 1.8637583276986454,
      "learning_rate": 6.054493496206035e-07,
      "log_odds_chosen": 1.008203148841858,
      "log_odds_ratio": -0.4569335877895355,
      "logits/chosen": -0.9652343988418579,
      "logits/rejected": -0.8658202886581421,
      "logps/chosen": -0.52587890625,
      "logps/rejected": -1.080468773841858,
      "loss": 0.7889,
      "nll_loss": 0.789746105670929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02630004845559597,
      "rewards/margins": 0.02765960618853569,
      "rewards/rejected": -0.05400390550494194,
      "step": 6820
    },
    {
      "epoch": 0.49757767821367427,
      "grad_norm": 2.9257054868607675,
      "learning_rate": 6.050059593380494e-07,
      "log_odds_chosen": 0.85986328125,
      "log_odds_ratio": -0.492431640625,
      "logits/chosen": -0.995898425579071,
      "logits/rejected": -0.9039062261581421,
      "logps/chosen": -0.5482422113418579,
      "logps/rejected": -1.0388672351837158,
      "loss": 0.7861,
      "nll_loss": 0.759960949420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02741088904440403,
      "rewards/margins": 0.02458343468606472,
      "rewards/rejected": -0.05198364332318306,
      "step": 6830
    },
    {
      "epoch": 0.49830619604414816,
      "grad_norm": 1.9507709976769918,
      "learning_rate": 6.045635417583431e-07,
      "log_odds_chosen": 1.222558617591858,
      "log_odds_ratio": -0.41230469942092896,
      "logits/chosen": -1.0027344226837158,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.578125,
      "logps/rejected": -1.309179663658142,
      "loss": 0.8039,
      "nll_loss": 0.76953125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02888793870806694,
      "rewards/margins": 0.036569975316524506,
      "rewards/rejected": -0.06546630710363388,
      "step": 6840
    },
    {
      "epoch": 0.49903471387462206,
      "grad_norm": 2.654385159372612,
      "learning_rate": 6.04122093330177e-07,
      "log_odds_chosen": 0.9979492425918579,
      "log_odds_ratio": -0.468017578125,
      "logits/chosen": -0.9892578125,
      "logits/rejected": -0.871289074420929,
      "logps/chosen": -0.547558605670929,
      "logps/rejected": -1.1083984375,
      "loss": 0.7908,
      "nll_loss": 0.785351574420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02736816368997097,
      "rewards/margins": 0.02805175818502903,
      "rewards/rejected": -0.05544433742761612,
      "step": 6850
    },
    {
      "epoch": 0.49976323170509596,
      "grad_norm": 2.240186970516036,
      "learning_rate": 6.03681610520369e-07,
      "log_odds_chosen": 0.81494140625,
      "log_odds_ratio": -0.5096679925918579,
      "logits/chosen": -0.953906238079071,
      "logits/rejected": -0.853515625,
      "logps/chosen": -0.5459960699081421,
      "logps/rejected": -0.9888671636581421,
      "loss": 0.7905,
      "nll_loss": 0.8052734136581421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02728271484375,
      "rewards/margins": 0.022125244140625,
      "rewards/rejected": -0.04944457858800888,
      "step": 6860
    },
    {
      "epoch": 0.5004917495355699,
      "grad_norm": 2.6019864343733965,
      "learning_rate": 6.032420898137439e-07,
      "log_odds_chosen": 0.890087902545929,
      "log_odds_ratio": -0.513476550579071,
      "logits/chosen": -0.948046863079071,
      "logits/rejected": -0.8314453363418579,
      "logps/chosen": -0.587597668170929,
      "logps/rejected": -1.1455078125,
      "loss": 0.8145,
      "nll_loss": 0.8500000238418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02937011793255806,
      "rewards/margins": 0.0279083251953125,
      "rewards/rejected": -0.05727539211511612,
      "step": 6870
    },
    {
      "epoch": 0.5012202673660437,
      "grad_norm": 1.7433561796899268,
      "learning_rate": 6.028035277130152e-07,
      "log_odds_chosen": 1.192773461341858,
      "log_odds_ratio": -0.43583983182907104,
      "logits/chosen": -0.985156238079071,
      "logits/rejected": -0.884570300579071,
      "logps/chosen": -0.55908203125,
      "logps/rejected": -1.2716796398162842,
      "loss": 0.8004,
      "nll_loss": 0.7669922113418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0279541015625,
      "rewards/margins": 0.03565063327550888,
      "rewards/rejected": -0.06356201320886612,
      "step": 6880
    },
    {
      "epoch": 0.5019487851965176,
      "grad_norm": 1.792632476193455,
      "learning_rate": 6.023659207386686e-07,
      "log_odds_chosen": 0.89794921875,
      "log_odds_ratio": -0.49101561307907104,
      "logits/chosen": -0.920117199420929,
      "logits/rejected": -0.8187500238418579,
      "logps/chosen": -0.534960925579071,
      "logps/rejected": -1.0556640625,
      "loss": 0.804,
      "nll_loss": 0.8197265863418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02673950232565403,
      "rewards/margins": 0.02604370191693306,
      "rewards/rejected": -0.05278320237994194,
      "step": 6890
    },
    {
      "epoch": 0.5026773030269915,
      "grad_norm": 2.1413671182988487,
      "learning_rate": 6.019292654288461e-07,
      "log_odds_chosen": 0.926074206829071,
      "log_odds_ratio": -0.47041016817092896,
      "logits/chosen": -0.980273425579071,
      "logits/rejected": -0.8648437261581421,
      "logps/chosen": -0.533007800579071,
      "logps/rejected": -1.0568358898162842,
      "loss": 0.7992,
      "nll_loss": 0.746386706829071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02665405347943306,
      "rewards/margins": 0.02617950364947319,
      "rewards/rejected": -0.05281982570886612,
      "step": 6900
    },
    {
      "epoch": 0.5034058208574654,
      "grad_norm": 1.924386656686497,
      "learning_rate": 6.014935583392304e-07,
      "log_odds_chosen": 0.9996337890625,
      "log_odds_ratio": -0.4512695372104645,
      "logits/chosen": -0.935742199420929,
      "logits/rejected": -0.8226562738418579,
      "logps/chosen": -0.52392578125,
      "logps/rejected": -1.128515601158142,
      "loss": 0.8021,
      "nll_loss": 0.749218761920929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02623290941119194,
      "rewards/margins": 0.03021087683737278,
      "rewards/rejected": -0.05638427659869194,
      "step": 6910
    },
    {
      "epoch": 0.5041343386879394,
      "grad_norm": 1.9160178066978766,
      "learning_rate": 6.010587960429313e-07,
      "log_odds_chosen": 1.048925757408142,
      "log_odds_ratio": -0.4564453065395355,
      "logits/chosen": -0.936718761920929,
      "logits/rejected": -0.794726550579071,
      "logps/chosen": -0.557324230670929,
      "logps/rejected": -1.1964843273162842,
      "loss": 0.8108,
      "nll_loss": 0.8167968988418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02786865271627903,
      "rewards/margins": 0.03194427490234375,
      "rewards/rejected": -0.05979003757238388,
      "step": 6920
    },
    {
      "epoch": 0.5048628565184133,
      "grad_norm": 1.9989763023831946,
      "learning_rate": 6.006249751303728e-07,
      "log_odds_chosen": 1.0026366710662842,
      "log_odds_ratio": -0.4686035215854645,
      "logits/chosen": -0.9462890625,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.538867175579071,
      "logps/rejected": -1.1609375476837158,
      "loss": 0.7932,
      "nll_loss": 0.750195324420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02693481370806694,
      "rewards/margins": 0.03107452392578125,
      "rewards/rejected": -0.05802612379193306,
      "step": 6930
    },
    {
      "epoch": 0.5055913743488872,
      "grad_norm": 2.234716486206121,
      "learning_rate": 6.001920922091795e-07,
      "log_odds_chosen": 0.842333972454071,
      "log_odds_ratio": -0.51318359375,
      "logits/chosen": -0.9410156011581421,
      "logits/rejected": -0.845703125,
      "logps/chosen": -0.5342773199081421,
      "logps/rejected": -1.021093726158142,
      "loss": 0.7985,
      "nll_loss": 0.780078113079071,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0267333984375,
      "rewards/margins": 0.024327849969267845,
      "rewards/rejected": -0.05103759840130806,
      "step": 6940
    },
    {
      "epoch": 0.5063198921793611,
      "grad_norm": 2.043243963655475,
      "learning_rate": 5.997601439040673e-07,
      "log_odds_chosen": 0.8753906488418579,
      "log_odds_ratio": -0.518750011920929,
      "logits/chosen": -0.9427734613418579,
      "logits/rejected": -0.837890625,
      "logps/chosen": -0.56787109375,
      "logps/rejected": -1.1124999523162842,
      "loss": 0.7937,
      "nll_loss": 0.788867175579071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02841796912252903,
      "rewards/margins": 0.02719421312212944,
      "rewards/rejected": -0.05562744289636612,
      "step": 6950
    },
    {
      "epoch": 0.507048410009835,
      "grad_norm": 2.31779793120845,
      "learning_rate": 5.993291268567303e-07,
      "log_odds_chosen": 1.038818359375,
      "log_odds_ratio": -0.45849609375,
      "logits/chosen": -0.9888671636581421,
      "logits/rejected": -0.8558593988418579,
      "logps/chosen": -0.5264648199081421,
      "logps/rejected": -1.1580078601837158,
      "loss": 0.7976,
      "nll_loss": 0.762011706829071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02631836012005806,
      "rewards/margins": 0.031609345227479935,
      "rewards/rejected": -0.05794677883386612,
      "step": 6960
    },
    {
      "epoch": 0.5077769278403089,
      "grad_norm": 1.8256949963962348,
      "learning_rate": 5.988990377257333e-07,
      "log_odds_chosen": 1.07177734375,
      "log_odds_ratio": -0.42626953125,
      "logits/chosen": -0.973828136920929,
      "logits/rejected": -0.864062488079071,
      "logps/chosen": -0.5528320074081421,
      "logps/rejected": -1.203710913658142,
      "loss": 0.8107,
      "nll_loss": 0.75390625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02762451209127903,
      "rewards/margins": 0.03253173828125,
      "rewards/rejected": -0.06020507961511612,
      "step": 6970
    },
    {
      "epoch": 0.5085054456707828,
      "grad_norm": 1.870409727404531,
      "learning_rate": 5.984698731864018e-07,
      "log_odds_chosen": 0.9418579339981079,
      "log_odds_ratio": -0.5218750238418579,
      "logits/chosen": -0.931445300579071,
      "logits/rejected": -0.8232421875,
      "logps/chosen": -0.546582043170929,
      "logps/rejected": -1.1033203601837158,
      "loss": 0.8067,
      "nll_loss": 0.834179699420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02734069898724556,
      "rewards/margins": 0.027863312512636185,
      "rewards/rejected": -0.05522460862994194,
      "step": 6980
    },
    {
      "epoch": 0.5092339635012567,
      "grad_norm": 1.7901316421611024,
      "learning_rate": 5.980416299307149e-07,
      "log_odds_chosen": 0.9957031011581421,
      "log_odds_ratio": -0.4693847596645355,
      "logits/chosen": -0.9144531488418579,
      "logits/rejected": -0.8255859613418579,
      "logps/chosen": -0.5425781011581421,
      "logps/rejected": -1.115234375,
      "loss": 0.7983,
      "nll_loss": 0.828906238079071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02710571326315403,
      "rewards/margins": 0.02863769605755806,
      "rewards/rejected": -0.05576171725988388,
      "step": 6990
    },
    {
      "epoch": 0.5099624813317306,
      "grad_norm": 1.8715053775285748,
      "learning_rate": 5.976143046671969e-07,
      "log_odds_chosen": 1.205957055091858,
      "log_odds_ratio": -0.42536622285842896,
      "logits/chosen": -0.9525390863418579,
      "logits/rejected": -0.8460937738418579,
      "logps/chosen": -0.509472668170929,
      "logps/rejected": -1.2156250476837158,
      "loss": 0.7944,
      "nll_loss": 0.7528320550918579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.02545776404440403,
      "rewards/margins": 0.03529968112707138,
      "rewards/rejected": -0.06076660007238388,
      "step": 7000
    },
    {
      "epoch": 0.5106909991622045,
      "grad_norm": 1.6541417041640032,
      "learning_rate": 5.971878941208125e-07,
      "log_odds_chosen": 1.142236351966858,
      "log_odds_ratio": -0.43891602754592896,
      "logits/chosen": -0.962890625,
      "logits/rejected": -0.878125011920929,
      "logps/chosen": -0.523730456829071,
      "logps/rejected": -1.1867187023162842,
      "loss": 0.8011,
      "nll_loss": 0.682910144329071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02619018591940403,
      "rewards/margins": 0.03318481519818306,
      "rewards/rejected": -0.05936279147863388,
      "step": 7010
    },
    {
      "epoch": 0.5114195169926784,
      "grad_norm": 1.9041322507075782,
      "learning_rate": 5.967623950328609e-07,
      "log_odds_chosen": 1.254638671875,
      "log_odds_ratio": -0.4111328125,
      "logits/chosen": -0.9283202886581421,
      "logits/rejected": -0.7916015386581421,
      "logps/chosen": -0.5386718511581421,
      "logps/rejected": -1.3019530773162842,
      "loss": 0.7937,
      "nll_loss": 0.7398437261581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02695312537252903,
      "rewards/margins": 0.0381011962890625,
      "rewards/rejected": -0.06503906100988388,
      "step": 7020
    },
    {
      "epoch": 0.5121480348231523,
      "grad_norm": 1.8391843496434273,
      "learning_rate": 5.963378041608701e-07,
      "log_odds_chosen": 1.003381371498108,
      "log_odds_ratio": -0.48310548067092896,
      "logits/chosen": -0.959179699420929,
      "logits/rejected": -0.830859363079071,
      "logps/chosen": -0.5654296875,
      "logps/rejected": -1.1533203125,
      "loss": 0.7955,
      "nll_loss": 0.7347656488418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02827148512005806,
      "rewards/margins": 0.02939300611615181,
      "rewards/rejected": -0.0576171875,
      "step": 7030
    },
    {
      "epoch": 0.5128765526536262,
      "grad_norm": 1.774493791321294,
      "learning_rate": 5.959141182784952e-07,
      "log_odds_chosen": 0.8209472894668579,
      "log_odds_ratio": -0.5185546875,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.8580077886581421,
      "logps/chosen": -0.5731445550918579,
      "logps/rejected": -1.047460913658142,
      "loss": 0.8133,
      "nll_loss": 0.7945312261581421,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.028656005859375,
      "rewards/margins": 0.023703765124082565,
      "rewards/rejected": -0.052398681640625,
      "step": 7040
    },
    {
      "epoch": 0.5136050704841001,
      "grad_norm": 2.163028034194422,
      "learning_rate": 5.954913341754137e-07,
      "log_odds_chosen": 1.0040283203125,
      "log_odds_ratio": -0.46044921875,
      "logits/chosen": -0.943554699420929,
      "logits/rejected": -0.8382812738418579,
      "logps/chosen": -0.54736328125,
      "logps/rejected": -1.1554687023162842,
      "loss": 0.8034,
      "nll_loss": 0.780078113079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.027374267578125,
      "rewards/margins": 0.03037414513528347,
      "rewards/rejected": -0.05772705003619194,
      "step": 7050
    },
    {
      "epoch": 0.514333588314574,
      "grad_norm": 2.0868779991525592,
      "learning_rate": 5.950694486572239e-07,
      "log_odds_chosen": 0.98779296875,
      "log_odds_ratio": -0.4803710877895355,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.8740234375,
      "logps/chosen": -0.562695324420929,
      "logps/rejected": -1.1599609851837158,
      "loss": 0.7912,
      "nll_loss": 0.779492199420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02814941480755806,
      "rewards/margins": 0.02990112267434597,
      "rewards/rejected": -0.05803222581744194,
      "step": 7060
    },
    {
      "epoch": 0.5150621061450479,
      "grad_norm": 1.7702066463008341,
      "learning_rate": 5.94648458545344e-07,
      "log_odds_chosen": 1.0829589366912842,
      "log_odds_ratio": -0.46293944120407104,
      "logits/chosen": -0.9173828363418579,
      "logits/rejected": -0.843945324420929,
      "logps/chosen": -0.5684570074081421,
      "logps/rejected": -1.2068359851837158,
      "loss": 0.7908,
      "nll_loss": 0.734375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02846069261431694,
      "rewards/margins": 0.03191222995519638,
      "rewards/rejected": -0.06036376953125,
      "step": 7070
    },
    {
      "epoch": 0.5157906239755218,
      "grad_norm": 2.0912300112288116,
      "learning_rate": 5.942283606769104e-07,
      "log_odds_chosen": 1.109594702720642,
      "log_odds_ratio": -0.46293944120407104,
      "logits/chosen": -0.959765613079071,
      "logits/rejected": -0.866992175579071,
      "logps/chosen": -0.545605480670929,
      "logps/rejected": -1.1544921398162842,
      "loss": 0.7891,
      "nll_loss": 0.8212890625,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0272674560546875,
      "rewards/margins": 0.03049774095416069,
      "rewards/rejected": -0.05776367336511612,
      "step": 7080
    },
    {
      "epoch": 0.5165191418059957,
      "grad_norm": 1.8609386058902846,
      "learning_rate": 5.938091519046793e-07,
      "log_odds_chosen": 0.993457019329071,
      "log_odds_ratio": -0.46904295682907104,
      "logits/chosen": -0.9878906011581421,
      "logits/rejected": -0.8642578125,
      "logps/chosen": -0.53662109375,
      "logps/rejected": -1.1238281726837158,
      "loss": 0.8094,
      "nll_loss": 0.7422851324081421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02684326097369194,
      "rewards/margins": 0.02936096116900444,
      "rewards/rejected": -0.05621337890625,
      "step": 7090
    },
    {
      "epoch": 0.5172476596364696,
      "grad_norm": 1.9469447493969667,
      "learning_rate": 5.933908290969267e-07,
      "log_odds_chosen": 1.087304711341858,
      "log_odds_ratio": -0.4334960877895355,
      "logits/chosen": -1.0031249523162842,
      "logits/rejected": -0.8785156011581421,
      "logps/chosen": -0.49921876192092896,
      "logps/rejected": -1.119531273841858,
      "loss": 0.7857,
      "nll_loss": 0.736328125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02495727501809597,
      "rewards/margins": 0.03099365159869194,
      "rewards/rejected": -0.05594482272863388,
      "step": 7100
    },
    {
      "epoch": 0.5179761774669435,
      "grad_norm": 1.8623954005026504,
      "learning_rate": 5.929733891373501e-07,
      "log_odds_chosen": 1.2216796875,
      "log_odds_ratio": -0.3966308534145355,
      "logits/chosen": -0.9730468988418579,
      "logits/rejected": -0.841601550579071,
      "logps/chosen": -0.511523425579071,
      "logps/rejected": -1.183203101158142,
      "loss": 0.7893,
      "nll_loss": 0.723828136920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02556152269244194,
      "rewards/margins": 0.03354492038488388,
      "rewards/rejected": -0.05909423902630806,
      "step": 7110
    },
    {
      "epoch": 0.5187046952974174,
      "grad_norm": 2.2223700137795293,
      "learning_rate": 5.925568289249717e-07,
      "log_odds_chosen": 0.971484363079071,
      "log_odds_ratio": -0.4784179627895355,
      "logits/chosen": -0.9576171636581421,
      "logits/rejected": -0.8707031011581421,
      "logps/chosen": -0.6009765863418579,
      "logps/rejected": -1.2048828601837158,
      "loss": 0.7889,
      "nll_loss": 0.778515636920929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.03007812425494194,
      "rewards/margins": 0.030207062140107155,
      "rewards/rejected": -0.060302734375,
      "step": 7120
    },
    {
      "epoch": 0.5194332131278913,
      "grad_norm": 2.1198762097500117,
      "learning_rate": 5.921411453740407e-07,
      "log_odds_chosen": 1.0038573741912842,
      "log_odds_ratio": -0.46879881620407104,
      "logits/chosen": -0.9390624761581421,
      "logits/rejected": -0.8486328125,
      "logps/chosen": -0.5435546636581421,
      "logps/rejected": -1.141210913658142,
      "loss": 0.7897,
      "nll_loss": 0.761425793170929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02717895433306694,
      "rewards/margins": 0.02991027757525444,
      "rewards/rejected": -0.05708007887005806,
      "step": 7130
    },
    {
      "epoch": 0.5201617309583652,
      "grad_norm": 1.7765891886659566,
      "learning_rate": 5.917263354139386e-07,
      "log_odds_chosen": 0.9723144769668579,
      "log_odds_ratio": -0.4759765565395355,
      "logits/chosen": -0.9203125238418579,
      "logits/rejected": -0.828125,
      "logps/chosen": -0.5206054449081421,
      "logps/rejected": -1.112890601158142,
      "loss": 0.7987,
      "nll_loss": 0.7015625238418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02605590783059597,
      "rewards/margins": 0.02962493896484375,
      "rewards/rejected": -0.05562744289636612,
      "step": 7140
    },
    {
      "epoch": 0.5208902487888392,
      "grad_norm": 2.1597955253262224,
      "learning_rate": 5.913123959890826e-07,
      "log_odds_chosen": 1.009619116783142,
      "log_odds_ratio": -0.46684569120407104,
      "logits/chosen": -0.928906261920929,
      "logits/rejected": -0.824023425579071,
      "logps/chosen": -0.539746105670929,
      "logps/rejected": -1.111914038658142,
      "loss": 0.7955,
      "nll_loss": 0.785937488079071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0269775390625,
      "rewards/margins": 0.0286102294921875,
      "rewards/rejected": -0.05554809421300888,
      "step": 7150
    },
    {
      "epoch": 0.5216187666193131,
      "grad_norm": 2.2621310125431293,
      "learning_rate": 5.90899324058832e-07,
      "log_odds_chosen": 1.260107398033142,
      "log_odds_ratio": -0.390625,
      "logits/chosen": -0.982421875,
      "logits/rejected": -0.844921886920929,
      "logps/chosen": -0.503125011920929,
      "logps/rejected": -1.249609351158142,
      "loss": 0.7754,
      "nll_loss": 0.728515625,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.02517089806497097,
      "rewards/margins": 0.03728637844324112,
      "rewards/rejected": -0.06248779222369194,
      "step": 7160
    },
    {
      "epoch": 0.522347284449787,
      "grad_norm": 1.8882443555491908,
      "learning_rate": 5.904871165973932e-07,
      "log_odds_chosen": 1.084570288658142,
      "log_odds_ratio": -0.44829100370407104,
      "logits/chosen": -0.984179675579071,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.511425793170929,
      "logps/rejected": -1.1677734851837158,
      "loss": 0.7817,
      "nll_loss": 0.741894543170929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02557983435690403,
      "rewards/margins": 0.03287048265337944,
      "rewards/rejected": -0.05842285230755806,
      "step": 7170
    },
    {
      "epoch": 0.5230758022802608,
      "grad_norm": 1.5723817533089601,
      "learning_rate": 5.900757705937288e-07,
      "log_odds_chosen": 0.926562488079071,
      "log_odds_ratio": -0.4803710877895355,
      "logits/chosen": -0.9849609136581421,
      "logits/rejected": -0.8863281011581421,
      "logps/chosen": -0.555371105670929,
      "logps/rejected": -1.069921851158142,
      "loss": 0.7914,
      "nll_loss": 0.766406238079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02777099609375,
      "rewards/margins": 0.02570495568215847,
      "rewards/rejected": -0.05351562425494194,
      "step": 7180
    },
    {
      "epoch": 0.5238043201107347,
      "grad_norm": 1.640809597160558,
      "learning_rate": 5.896652830514625e-07,
      "log_odds_chosen": 1.1474120616912842,
      "log_odds_ratio": -0.429931640625,
      "logits/chosen": -0.9302734136581421,
      "logits/rejected": -0.8246093988418579,
      "logps/chosen": -0.562792956829071,
      "logps/rejected": -1.2931640148162842,
      "loss": 0.7998,
      "nll_loss": 0.7837890386581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02813720703125,
      "rewards/margins": 0.03653831407427788,
      "rewards/rejected": -0.06462402641773224,
      "step": 7190
    },
    {
      "epoch": 0.5245328379412086,
      "grad_norm": 1.7408936858110828,
      "learning_rate": 5.892556509887897e-07,
      "log_odds_chosen": 1.0598633289337158,
      "log_odds_ratio": -0.48491209745407104,
      "logits/chosen": -0.9794921875,
      "logits/rejected": -0.8623046875,
      "logps/chosen": -0.59814453125,
      "logps/rejected": -1.193359375,
      "loss": 0.7951,
      "nll_loss": 0.799023449420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0299072265625,
      "rewards/margins": 0.02981262281537056,
      "rewards/rejected": -0.0596923828125,
      "step": 7200
    },
    {
      "epoch": 0.5252613557716825,
      "grad_norm": 2.513243021034279,
      "learning_rate": 5.88846871438385e-07,
      "log_odds_chosen": 1.2138671875,
      "log_odds_ratio": -0.416259765625,
      "logits/chosen": -0.999804675579071,
      "logits/rejected": -0.8755859136581421,
      "logps/chosen": -0.559374988079071,
      "logps/rejected": -1.2980468273162842,
      "loss": 0.7948,
      "nll_loss": 0.7655273675918579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02794799767434597,
      "rewards/margins": 0.03692016750574112,
      "rewards/rejected": -0.06489257514476776,
      "step": 7210
    },
    {
      "epoch": 0.5259898736021564,
      "grad_norm": 2.7884989085315652,
      "learning_rate": 5.884389414473131e-07,
      "log_odds_chosen": 1.0146484375,
      "log_odds_ratio": -0.45722657442092896,
      "logits/chosen": -0.967578113079071,
      "logits/rejected": -0.847460925579071,
      "logps/chosen": -0.507519543170929,
      "logps/rejected": -1.12109375,
      "loss": 0.7862,
      "nll_loss": 0.71337890625,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02537231519818306,
      "rewards/margins": 0.03064575232565403,
      "rewards/rejected": -0.0560302734375,
      "step": 7220
    },
    {
      "epoch": 0.5267183914326303,
      "grad_norm": 2.0688272708869477,
      "learning_rate": 5.880318580769381e-07,
      "log_odds_chosen": 0.915966808795929,
      "log_odds_ratio": -0.5230468511581421,
      "logits/chosen": -0.9638671875,
      "logits/rejected": -0.8863281011581421,
      "logps/chosen": -0.605273425579071,
      "logps/rejected": -1.147070288658142,
      "loss": 0.7913,
      "nll_loss": 0.824414074420929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.030242919921875,
      "rewards/margins": 0.027133941650390625,
      "rewards/rejected": -0.05733642727136612,
      "step": 7230
    },
    {
      "epoch": 0.5274469092631042,
      "grad_norm": 1.9994681054776564,
      "learning_rate": 5.876256184028356e-07,
      "log_odds_chosen": 1.010644555091858,
      "log_odds_ratio": -0.4944824278354645,
      "logits/chosen": -0.9554687738418579,
      "logits/rejected": -0.880664050579071,
      "logps/chosen": -0.5782226324081421,
      "logps/rejected": -1.1533203125,
      "loss": 0.8087,
      "nll_loss": 0.814453125,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02891235426068306,
      "rewards/margins": 0.02875060960650444,
      "rewards/rejected": -0.05764160305261612,
      "step": 7240
    },
    {
      "epoch": 0.5281754270935781,
      "grad_norm": 2.050098610952569,
      "learning_rate": 5.872202195147035e-07,
      "log_odds_chosen": 1.033837914466858,
      "log_odds_ratio": -0.46171873807907104,
      "logits/chosen": -1.010156273841858,
      "logits/rejected": -0.899609386920929,
      "logps/chosen": -0.55078125,
      "logps/rejected": -1.1806640625,
      "loss": 0.7898,
      "nll_loss": 0.795117199420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02756347693502903,
      "rewards/margins": 0.03146209567785263,
      "rewards/rejected": -0.05900878831744194,
      "step": 7250
    },
    {
      "epoch": 0.528903944924052,
      "grad_norm": 2.164175299433039,
      "learning_rate": 5.868156585162754e-07,
      "log_odds_chosen": 0.948803722858429,
      "log_odds_ratio": -0.5127929449081421,
      "logits/chosen": -0.992382824420929,
      "logits/rejected": -0.888867199420929,
      "logps/chosen": -0.5428711175918579,
      "logps/rejected": -1.1287109851837158,
      "loss": 0.7661,
      "nll_loss": 0.6875,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02714233472943306,
      "rewards/margins": 0.02929992601275444,
      "rewards/rejected": -0.05643920972943306,
      "step": 7260
    },
    {
      "epoch": 0.5296324627545259,
      "grad_norm": 1.6797930460062116,
      "learning_rate": 5.864119325252326e-07,
      "log_odds_chosen": 0.974316418170929,
      "log_odds_ratio": -0.4775390625,
      "logits/chosen": -1.0011718273162842,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.5577148199081421,
      "logps/rejected": -1.1658203601837158,
      "loss": 0.7917,
      "nll_loss": 0.7490234375,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02789306640625,
      "rewards/margins": 0.03042297437787056,
      "rewards/rejected": -0.05832519382238388,
      "step": 7270
    },
    {
      "epoch": 0.5303609805849998,
      "grad_norm": 4.162380663473278,
      "learning_rate": 5.860090386731193e-07,
      "log_odds_chosen": 0.9098876714706421,
      "log_odds_ratio": -0.51171875,
      "logits/chosen": -0.9664062261581421,
      "logits/rejected": -0.830859363079071,
      "logps/chosen": -0.5653320550918579,
      "logps/rejected": -1.1150391101837158,
      "loss": 0.7794,
      "nll_loss": 0.7724609375,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02825317345559597,
      "rewards/margins": 0.02748260460793972,
      "rewards/rejected": -0.05572509765625,
      "step": 7280
    },
    {
      "epoch": 0.5310894984154737,
      "grad_norm": 1.842735130139255,
      "learning_rate": 5.856069741052554e-07,
      "log_odds_chosen": 1.107812523841858,
      "log_odds_ratio": -0.4559082090854645,
      "logits/chosen": -0.941210925579071,
      "logits/rejected": -0.793749988079071,
      "logps/chosen": -0.535937488079071,
      "logps/rejected": -1.2423827648162842,
      "loss": 0.8038,
      "nll_loss": 0.795703113079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02681274339556694,
      "rewards/margins": 0.03532104566693306,
      "rewards/rejected": -0.06217651441693306,
      "step": 7290
    },
    {
      "epoch": 0.5318180162459476,
      "grad_norm": 1.966339530874054,
      "learning_rate": 5.852057359806529e-07,
      "log_odds_chosen": 1.004492163658142,
      "log_odds_ratio": -0.43681639432907104,
      "logits/chosen": -0.942578136920929,
      "logits/rejected": -0.8179687261581421,
      "logps/chosen": -0.48320311307907104,
      "logps/rejected": -1.05078125,
      "loss": 0.7848,
      "nll_loss": 0.783886730670929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02416076697409153,
      "rewards/margins": 0.02836913987994194,
      "rewards/rejected": -0.05255126953125,
      "step": 7300
    },
    {
      "epoch": 0.5325465340764215,
      "grad_norm": 2.128493924269245,
      "learning_rate": 5.848053214719305e-07,
      "log_odds_chosen": 1.1364257335662842,
      "log_odds_ratio": -0.4093261659145355,
      "logits/chosen": -0.9720703363418579,
      "logits/rejected": -0.8462890386581421,
      "logps/chosen": -0.535937488079071,
      "logps/rejected": -1.220703125,
      "loss": 0.776,
      "nll_loss": 0.75390625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02680053748190403,
      "rewards/margins": 0.03419799730181694,
      "rewards/rejected": -0.06102294847369194,
      "step": 7310
    },
    {
      "epoch": 0.5332750519068954,
      "grad_norm": 1.8652574908789763,
      "learning_rate": 5.844057277652305e-07,
      "log_odds_chosen": 1.1713378429412842,
      "log_odds_ratio": -0.4159179627895355,
      "logits/chosen": -0.935351550579071,
      "logits/rejected": -0.8355468511581421,
      "logps/chosen": -0.5218750238418579,
      "logps/rejected": -1.1990234851837158,
      "loss": 0.8125,
      "nll_loss": 0.793164074420929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.02608642540872097,
      "rewards/margins": 0.03386993333697319,
      "rewards/rejected": -0.06000976637005806,
      "step": 7320
    },
    {
      "epoch": 0.5340035697373693,
      "grad_norm": 2.0682621091435025,
      "learning_rate": 5.840069520601363e-07,
      "log_odds_chosen": 1.073095679283142,
      "log_odds_ratio": -0.4571289122104645,
      "logits/chosen": -0.958203136920929,
      "logits/rejected": -0.871874988079071,
      "logps/chosen": -0.5362304449081421,
      "logps/rejected": -1.172460913658142,
      "loss": 0.7939,
      "nll_loss": 0.776171863079071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02682800218462944,
      "rewards/margins": 0.03178100660443306,
      "rewards/rejected": -0.058624267578125,
      "step": 7330
    },
    {
      "epoch": 0.5347320875678432,
      "grad_norm": 1.848024638285276,
      "learning_rate": 5.836089915695886e-07,
      "log_odds_chosen": 1.1921875476837158,
      "log_odds_ratio": -0.4237304627895355,
      "logits/chosen": -0.9810546636581421,
      "logits/rejected": -0.8138672113418579,
      "logps/chosen": -0.5264648199081421,
      "logps/rejected": -1.270117163658142,
      "loss": 0.7914,
      "nll_loss": 0.759960949420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02634887769818306,
      "rewards/margins": 0.03711547702550888,
      "rewards/rejected": -0.0634765625,
      "step": 7340
    },
    {
      "epoch": 0.5354606053983171,
      "grad_norm": 1.8655184012198824,
      "learning_rate": 5.832118435198044e-07,
      "log_odds_chosen": 0.9458984136581421,
      "log_odds_ratio": -0.5040527582168579,
      "logits/chosen": -0.959765613079071,
      "logits/rejected": -0.8587890863418579,
      "logps/chosen": -0.541210949420929,
      "logps/rejected": -1.082617163658142,
      "loss": 0.7975,
      "nll_loss": 0.801953136920929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02705078199505806,
      "rewards/margins": 0.027069091796875,
      "rewards/rejected": -0.05413818359375,
      "step": 7350
    },
    {
      "epoch": 0.536189123228791,
      "grad_norm": 1.8133058592770108,
      "learning_rate": 5.828155051501962e-07,
      "log_odds_chosen": 1.135009765625,
      "log_odds_ratio": -0.44062501192092896,
      "logits/chosen": -0.9720703363418579,
      "logits/rejected": -0.8773437738418579,
      "logps/chosen": -0.534375011920929,
      "logps/rejected": -1.201171875,
      "loss": 0.7803,
      "nll_loss": 0.7494140863418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02670898474752903,
      "rewards/margins": 0.03336486965417862,
      "rewards/rejected": -0.06009521335363388,
      "step": 7360
    },
    {
      "epoch": 0.5369176410592649,
      "grad_norm": 1.9251597268867797,
      "learning_rate": 5.824199737132906e-07,
      "log_odds_chosen": 1.0089843273162842,
      "log_odds_ratio": -0.47685545682907104,
      "logits/chosen": -0.9951171875,
      "logits/rejected": -0.873828113079071,
      "logps/chosen": -0.5494140386581421,
      "logps/rejected": -1.1623046398162842,
      "loss": 0.772,
      "nll_loss": 0.7646484375,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02749023400247097,
      "rewards/margins": 0.03061828576028347,
      "rewards/rejected": -0.05809326097369194,
      "step": 7370
    },
    {
      "epoch": 0.5376461588897389,
      "grad_norm": 1.820846613949449,
      "learning_rate": 5.820252464746485e-07,
      "log_odds_chosen": 1.0087890625,
      "log_odds_ratio": -0.454833984375,
      "logits/chosen": -0.9853515625,
      "logits/rejected": -0.8578125238418579,
      "logps/chosen": -0.546093761920929,
      "logps/rejected": -1.1697266101837158,
      "loss": 0.7939,
      "nll_loss": 0.725390613079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02729492262005806,
      "rewards/margins": 0.03117675706744194,
      "rewards/rejected": -0.05849609524011612,
      "step": 7380
    },
    {
      "epoch": 0.5383746767202128,
      "grad_norm": 2.120783746307397,
      "learning_rate": 5.816313207127863e-07,
      "log_odds_chosen": 1.113061547279358,
      "log_odds_ratio": -0.42949217557907104,
      "logits/chosen": -0.986132800579071,
      "logits/rejected": -0.857617199420929,
      "logps/chosen": -0.5194336175918579,
      "logps/rejected": -1.181054711341858,
      "loss": 0.788,
      "nll_loss": 0.7841796875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02595214918255806,
      "rewards/margins": 0.03309784084558487,
      "rewards/rejected": -0.05903930589556694,
      "step": 7390
    },
    {
      "epoch": 0.5391031945506867,
      "grad_norm": 1.9377884253526751,
      "learning_rate": 5.812381937190964e-07,
      "log_odds_chosen": 1.0455443859100342,
      "log_odds_ratio": -0.48222655057907104,
      "logits/chosen": -1.014062523841858,
      "logits/rejected": -0.8837890625,
      "logps/chosen": -0.576171875,
      "logps/rejected": -1.201171875,
      "loss": 0.7754,
      "nll_loss": 0.762402355670929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02882690355181694,
      "rewards/margins": 0.0312768928706646,
      "rewards/rejected": -0.06010131910443306,
      "step": 7400
    },
    {
      "epoch": 0.5398317123811606,
      "grad_norm": 1.8469938442827678,
      "learning_rate": 5.808458627977691e-07,
      "log_odds_chosen": 1.036035180091858,
      "log_odds_ratio": -0.47490233182907104,
      "logits/chosen": -0.9603515863418579,
      "logits/rejected": -0.861523449420929,
      "logps/chosen": -0.552734375,
      "logps/rejected": -1.171875,
      "loss": 0.7883,
      "nll_loss": 0.765917956829071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02765502966940403,
      "rewards/margins": 0.03096771240234375,
      "rewards/rejected": -0.05858154222369194,
      "step": 7410
    },
    {
      "epoch": 0.5405602302116345,
      "grad_norm": 2.0084741776485813,
      "learning_rate": 5.804543252657151e-07,
      "log_odds_chosen": 1.1291015148162842,
      "log_odds_ratio": -0.4454101622104645,
      "logits/chosen": -0.944531261920929,
      "logits/rejected": -0.8246093988418579,
      "logps/chosen": -0.5538085699081421,
      "logps/rejected": -1.2742187976837158,
      "loss": 0.7855,
      "nll_loss": 0.808789074420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02764892578125,
      "rewards/margins": 0.03608093410730362,
      "rewards/rejected": -0.06373290717601776,
      "step": 7420
    },
    {
      "epoch": 0.5412887480421084,
      "grad_norm": 1.9307915081792715,
      "learning_rate": 5.800635784524886e-07,
      "log_odds_chosen": 0.9886718988418579,
      "log_odds_ratio": -0.4754882752895355,
      "logits/chosen": -0.9974609613418579,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.5748046636581421,
      "logps/rejected": -1.178320288658142,
      "loss": 0.7965,
      "nll_loss": 0.7666015625,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02875366248190403,
      "rewards/margins": 0.030199432745575905,
      "rewards/rejected": -0.05892334133386612,
      "step": 7430
    },
    {
      "epoch": 0.5420172658725823,
      "grad_norm": 1.8198284360130392,
      "learning_rate": 5.796736197002103e-07,
      "log_odds_chosen": 1.267217993736267,
      "log_odds_ratio": -0.38666993379592896,
      "logits/chosen": -0.941210925579071,
      "logits/rejected": -0.802539050579071,
      "logps/chosen": -0.565625011920929,
      "logps/rejected": -1.377539038658142,
      "loss": 0.8008,
      "nll_loss": 0.83984375,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02824096754193306,
      "rewards/margins": 0.0406036376953125,
      "rewards/rejected": -0.06883545219898224,
      "step": 7440
    },
    {
      "epoch": 0.5427457837030562,
      "grad_norm": 2.8505881521833882,
      "learning_rate": 5.792844463634923e-07,
      "log_odds_chosen": 0.92724609375,
      "log_odds_ratio": -0.48554688692092896,
      "logits/chosen": -0.946484386920929,
      "logits/rejected": -0.846875011920929,
      "logps/chosen": -0.556933581829071,
      "logps/rejected": -1.11328125,
      "loss": 0.7857,
      "nll_loss": 0.719531238079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02785644493997097,
      "rewards/margins": 0.0278167724609375,
      "rewards/rejected": -0.05565796047449112,
      "step": 7450
    },
    {
      "epoch": 0.54347430153353,
      "grad_norm": 1.9372847768614077,
      "learning_rate": 5.788960558093616e-07,
      "log_odds_chosen": 0.9361327886581421,
      "log_odds_ratio": -0.46611326932907104,
      "logits/chosen": -0.950976550579071,
      "logits/rejected": -0.846484363079071,
      "logps/chosen": -0.5282226800918579,
      "logps/rejected": -1.0617187023162842,
      "loss": 0.7867,
      "nll_loss": 0.7865234613418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02641601487994194,
      "rewards/margins": 0.02667846716940403,
      "rewards/rejected": -0.05307617038488388,
      "step": 7460
    },
    {
      "epoch": 0.5442028193640039,
      "grad_norm": 1.863468646794446,
      "learning_rate": 5.785084454171862e-07,
      "log_odds_chosen": 0.896472156047821,
      "log_odds_ratio": -0.510693371295929,
      "logits/chosen": -1.004492163658142,
      "logits/rejected": -0.8832031488418579,
      "logps/chosen": -0.6070312261581421,
      "logps/rejected": -1.171875,
      "loss": 0.7967,
      "nll_loss": 0.738964855670929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.030364990234375,
      "rewards/margins": 0.02821197547018528,
      "rewards/rejected": -0.05855713039636612,
      "step": 7470
    },
    {
      "epoch": 0.5449313371944778,
      "grad_norm": 1.9634497514702216,
      "learning_rate": 5.781216125786003e-07,
      "log_odds_chosen": 1.010400414466858,
      "log_odds_ratio": -0.4515136778354645,
      "logits/chosen": -0.986523449420929,
      "logits/rejected": -0.8896484375,
      "logps/chosen": -0.5059570074081421,
      "logps/rejected": -1.065039038658142,
      "loss": 0.8119,
      "nll_loss": 0.741406261920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.025299072265625,
      "rewards/margins": 0.02798919752240181,
      "rewards/rejected": -0.05329589918255806,
      "step": 7480
    },
    {
      "epoch": 0.5456598550249517,
      "grad_norm": 2.5650116790387956,
      "learning_rate": 5.777355546974315e-07,
      "log_odds_chosen": 1.040307641029358,
      "log_odds_ratio": -0.50341796875,
      "logits/chosen": -0.919726550579071,
      "logits/rejected": -0.846484363079071,
      "logps/chosen": -0.562695324420929,
      "logps/rejected": -1.1552734375,
      "loss": 0.7937,
      "nll_loss": 0.796875,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02814331091940403,
      "rewards/margins": 0.02965698204934597,
      "rewards/rejected": -0.05775146558880806,
      "step": 7490
    },
    {
      "epoch": 0.5463883728554256,
      "grad_norm": 2.292762659612414,
      "learning_rate": 5.773502691896257e-07,
      "log_odds_chosen": 1.194091796875,
      "log_odds_ratio": -0.41484373807907104,
      "logits/chosen": -0.990429699420929,
      "logits/rejected": -0.8720703125,
      "logps/chosen": -0.542285144329071,
      "logps/rejected": -1.254492163658142,
      "loss": 0.7958,
      "nll_loss": 0.744140625,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.027130126953125,
      "rewards/margins": 0.03562622144818306,
      "rewards/rejected": -0.062744140625,
      "step": 7500
    },
    {
      "epoch": 0.5471168906858995,
      "grad_norm": 2.154679411893149,
      "learning_rate": 5.76965753483177e-07,
      "log_odds_chosen": 0.962451159954071,
      "log_odds_ratio": -0.5255371332168579,
      "logits/chosen": -0.990234375,
      "logits/rejected": -0.885937511920929,
      "logps/chosen": -0.60888671875,
      "logps/rejected": -1.2146484851837158,
      "loss": 0.7923,
      "nll_loss": 0.747265636920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0304412841796875,
      "rewards/margins": 0.03031463548541069,
      "rewards/rejected": -0.06072998046875,
      "step": 7510
    },
    {
      "epoch": 0.5478454085163734,
      "grad_norm": 1.6682171447605216,
      "learning_rate": 5.765820050180532e-07,
      "log_odds_chosen": 1.0886719226837158,
      "log_odds_ratio": -0.44355469942092896,
      "logits/chosen": -0.970507800579071,
      "logits/rejected": -0.865234375,
      "logps/chosen": -0.575390636920929,
      "logps/rejected": -1.2492187023162842,
      "loss": 0.7827,
      "nll_loss": 0.7591797113418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02877197228372097,
      "rewards/margins": 0.03369445726275444,
      "rewards/rejected": -0.0625,
      "step": 7520
    },
    {
      "epoch": 0.5485739263468473,
      "grad_norm": 1.879942648943626,
      "learning_rate": 5.761990212461259e-07,
      "log_odds_chosen": 1.0066406726837158,
      "log_odds_ratio": -0.4664062559604645,
      "logits/chosen": -0.989453136920929,
      "logits/rejected": -0.8500000238418579,
      "logps/chosen": -0.520800769329071,
      "logps/rejected": -1.0769531726837158,
      "loss": 0.7805,
      "nll_loss": 0.7646484375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02603759802877903,
      "rewards/margins": 0.02779388427734375,
      "rewards/rejected": -0.05384521558880806,
      "step": 7530
    },
    {
      "epoch": 0.5493024441773212,
      "grad_norm": 1.8986192602081087,
      "learning_rate": 5.758167996310984e-07,
      "log_odds_chosen": 1.0534179210662842,
      "log_odds_ratio": -0.43754881620407104,
      "logits/chosen": -0.9623047113418579,
      "logits/rejected": -0.8296874761581421,
      "logps/chosen": -0.529101550579071,
      "logps/rejected": -1.1232421398162842,
      "loss": 0.7757,
      "nll_loss": 0.7535156011581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02645263634622097,
      "rewards/margins": 0.02971496619284153,
      "rewards/rejected": -0.05615234375,
      "step": 7540
    },
    {
      "epoch": 0.5500309620077951,
      "grad_norm": 2.9461736332984594,
      "learning_rate": 5.75435337648436e-07,
      "log_odds_chosen": 1.2599608898162842,
      "log_odds_ratio": -0.40351563692092896,
      "logits/chosen": -0.9599609375,
      "logits/rejected": -0.821484386920929,
      "logps/chosen": -0.5254882574081421,
      "logps/rejected": -1.259765625,
      "loss": 0.7783,
      "nll_loss": 0.742968738079071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.026275634765625,
      "rewards/margins": 0.03672485426068306,
      "rewards/rejected": -0.06300048530101776,
      "step": 7550
    },
    {
      "epoch": 0.550759479838269,
      "grad_norm": 1.8644510799433285,
      "learning_rate": 5.750546327852952e-07,
      "log_odds_chosen": 1.185644507408142,
      "log_odds_ratio": -0.44189453125,
      "logits/chosen": -0.960742175579071,
      "logits/rejected": -0.8373047113418579,
      "logps/chosen": -0.50927734375,
      "logps/rejected": -1.169921875,
      "loss": 0.8062,
      "nll_loss": 0.7579101324081421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02545776404440403,
      "rewards/margins": 0.0330047607421875,
      "rewards/rejected": -0.05848388746380806,
      "step": 7560
    },
    {
      "epoch": 0.5514879976687429,
      "grad_norm": 2.3682837916339277,
      "learning_rate": 5.746746825404547e-07,
      "log_odds_chosen": 1.0602538585662842,
      "log_odds_ratio": -0.510546863079071,
      "logits/chosen": -0.910937488079071,
      "logits/rejected": -0.812304675579071,
      "logps/chosen": -0.5986328125,
      "logps/rejected": -1.2921874523162842,
      "loss": 0.7812,
      "nll_loss": 0.8060547113418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02994384802877903,
      "rewards/margins": 0.03471221774816513,
      "rewards/rejected": -0.06456299126148224,
      "step": 7570
    },
    {
      "epoch": 0.5522165154992168,
      "grad_norm": 2.019621191839815,
      "learning_rate": 5.742954844242457e-07,
      "log_odds_chosen": 1.0164062976837158,
      "log_odds_ratio": -0.47001951932907104,
      "logits/chosen": -0.9794921875,
      "logits/rejected": -0.888867199420929,
      "logps/chosen": -0.5298827886581421,
      "logps/rejected": -1.122460961341858,
      "loss": 0.793,
      "nll_loss": 0.772656261920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.0264892578125,
      "rewards/margins": 0.029608916491270065,
      "rewards/rejected": -0.05609741061925888,
      "step": 7580
    },
    {
      "epoch": 0.5529450333296907,
      "grad_norm": 2.026085986618019,
      "learning_rate": 5.739170359584849e-07,
      "log_odds_chosen": 0.931445300579071,
      "log_odds_ratio": -0.490234375,
      "logits/chosen": -0.9585937261581421,
      "logits/rejected": -0.875781238079071,
      "logps/chosen": -0.5423828363418579,
      "logps/rejected": -1.095312476158142,
      "loss": 0.7954,
      "nll_loss": 0.742968738079071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02712402306497097,
      "rewards/margins": 0.02761688269674778,
      "rewards/rejected": -0.05479126051068306,
      "step": 7590
    },
    {
      "epoch": 0.5536735511601646,
      "grad_norm": 1.7244984383055249,
      "learning_rate": 5.735393346764044e-07,
      "log_odds_chosen": 0.855542004108429,
      "log_odds_ratio": -0.5003906488418579,
      "logits/chosen": -0.964062511920929,
      "logits/rejected": -0.866015613079071,
      "logps/chosen": -0.563769519329071,
      "logps/rejected": -1.03515625,
      "loss": 0.7908,
      "nll_loss": 0.705761730670929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02817382849752903,
      "rewards/margins": 0.02355041541159153,
      "rewards/rejected": -0.05171508714556694,
      "step": 7600
    },
    {
      "epoch": 0.5544020689906386,
      "grad_norm": 1.8664219420533879,
      "learning_rate": 5.731623781225858e-07,
      "log_odds_chosen": 0.983203113079071,
      "log_odds_ratio": -0.4625488221645355,
      "logits/chosen": -0.9208984375,
      "logits/rejected": -0.8505859375,
      "logps/chosen": -0.52099609375,
      "logps/rejected": -1.090429663658142,
      "loss": 0.7959,
      "nll_loss": 0.814648449420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02606201171875,
      "rewards/margins": 0.028449248522520065,
      "rewards/rejected": -0.05454101413488388,
      "step": 7610
    },
    {
      "epoch": 0.5551305868211125,
      "grad_norm": 2.0451680246693615,
      "learning_rate": 5.727861638528923e-07,
      "log_odds_chosen": 0.981249988079071,
      "log_odds_ratio": -0.47871094942092896,
      "logits/chosen": -0.9058593511581421,
      "logits/rejected": -0.8121093511581421,
      "logps/chosen": -0.531445324420929,
      "logps/rejected": -1.099023461341858,
      "loss": 0.7843,
      "nll_loss": 0.7171875238418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02656250074505806,
      "rewards/margins": 0.02834777906537056,
      "rewards/rejected": -0.05490722507238388,
      "step": 7620
    },
    {
      "epoch": 0.5558591046515864,
      "grad_norm": 2.2015106531179804,
      "learning_rate": 5.724106894344026e-07,
      "log_odds_chosen": 0.972851574420929,
      "log_odds_ratio": -0.48833006620407104,
      "logits/chosen": -0.9798828363418579,
      "logits/rejected": -0.862109363079071,
      "logps/chosen": -0.567187488079071,
      "logps/rejected": -1.163671851158142,
      "loss": 0.7982,
      "nll_loss": 0.8021484613418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02838134765625,
      "rewards/margins": 0.02979736402630806,
      "rewards/rejected": -0.05816650390625,
      "step": 7630
    },
    {
      "epoch": 0.5565876224820603,
      "grad_norm": 1.7856622902005155,
      "learning_rate": 5.720359524453445e-07,
      "log_odds_chosen": 1.167089819908142,
      "log_odds_ratio": -0.4473632872104645,
      "logits/chosen": -0.913281261920929,
      "logits/rejected": -0.8441406488418579,
      "logps/chosen": -0.547656238079071,
      "logps/rejected": -1.2361328601837158,
      "loss": 0.7853,
      "nll_loss": 0.7640625238418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.027374267578125,
      "rewards/margins": 0.03441772609949112,
      "rewards/rejected": -0.06181640550494194,
      "step": 7640
    },
    {
      "epoch": 0.5573161403125342,
      "grad_norm": 2.1830833844230013,
      "learning_rate": 5.716619504750295e-07,
      "log_odds_chosen": 1.1283690929412842,
      "log_odds_ratio": -0.44414061307907104,
      "logits/chosen": -0.958203136920929,
      "logits/rejected": -0.837890625,
      "logps/chosen": -0.5401366949081421,
      "logps/rejected": -1.2451171875,
      "loss": 0.7908,
      "nll_loss": 0.7943359613418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.027008056640625,
      "rewards/margins": 0.03515167161822319,
      "rewards/rejected": -0.06221923977136612,
      "step": 7650
    },
    {
      "epoch": 0.5580446581430081,
      "grad_norm": 2.043019693345901,
      "learning_rate": 5.712886811237877e-07,
      "log_odds_chosen": 0.951904296875,
      "log_odds_ratio": -0.49658203125,
      "logits/chosen": -0.931835949420929,
      "logits/rejected": -0.8138672113418579,
      "logps/chosen": -0.5672851800918579,
      "logps/rejected": -1.138085961341858,
      "loss": 0.7776,
      "nll_loss": 0.8158203363418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02834472618997097,
      "rewards/margins": 0.02859954908490181,
      "rewards/rejected": -0.05694580078125,
      "step": 7660
    },
    {
      "epoch": 0.558773175973482,
      "grad_norm": 2.3051605920760667,
      "learning_rate": 5.709161420029027e-07,
      "log_odds_chosen": 0.964550793170929,
      "log_odds_ratio": -0.511523425579071,
      "logits/chosen": -0.9449218511581421,
      "logits/rejected": -0.8427734375,
      "logps/chosen": -0.6009765863418579,
      "logps/rejected": -1.190820336341858,
      "loss": 0.7862,
      "nll_loss": 0.813671886920929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03005371056497097,
      "rewards/margins": 0.029510498046875,
      "rewards/rejected": -0.05950927734375,
      "step": 7670
    },
    {
      "epoch": 0.5595016938039559,
      "grad_norm": 2.1880958870812317,
      "learning_rate": 5.705443307345481e-07,
      "log_odds_chosen": 1.103417992591858,
      "log_odds_ratio": -0.456787109375,
      "logits/chosen": -0.9808593988418579,
      "logits/rejected": -0.865234375,
      "logps/chosen": -0.51806640625,
      "logps/rejected": -1.165429711341858,
      "loss": 0.7776,
      "nll_loss": 0.7105468511581421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02588501013815403,
      "rewards/margins": 0.03236999362707138,
      "rewards/rejected": -0.05827636644244194,
      "step": 7680
    },
    {
      "epoch": 0.5602302116344298,
      "grad_norm": 1.9232845225219823,
      "learning_rate": 5.701732449517231e-07,
      "log_odds_chosen": 0.855419933795929,
      "log_odds_ratio": -0.508984386920929,
      "logits/chosen": -0.9296875,
      "logits/rejected": -0.860156238079071,
      "logps/chosen": -0.5362304449081421,
      "logps/rejected": -1.020898461341858,
      "loss": 0.7704,
      "nll_loss": 0.718945324420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02680664137005806,
      "rewards/margins": 0.024222563952207565,
      "rewards/rejected": -0.05103149265050888,
      "step": 7690
    },
    {
      "epoch": 0.5609587294649037,
      "grad_norm": 2.0806140326948945,
      "learning_rate": 5.698028822981898e-07,
      "log_odds_chosen": 0.993945300579071,
      "log_odds_ratio": -0.4671874940395355,
      "logits/chosen": -0.979296863079071,
      "logits/rejected": -0.8374999761581421,
      "logps/chosen": -0.5806640386581421,
      "logps/rejected": -1.1876952648162842,
      "loss": 0.7943,
      "nll_loss": 0.786328136920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02907104417681694,
      "rewards/margins": 0.0302886962890625,
      "rewards/rejected": -0.05935058742761612,
      "step": 7700
    },
    {
      "epoch": 0.5616872472953776,
      "grad_norm": 2.128386322940307,
      "learning_rate": 5.694332404284097e-07,
      "log_odds_chosen": 0.901611328125,
      "log_odds_ratio": -0.532958984375,
      "logits/chosen": -0.919726550579071,
      "logits/rejected": -0.858593761920929,
      "logps/chosen": -0.5394531488418579,
      "logps/rejected": -1.024999976158142,
      "loss": 0.7727,
      "nll_loss": 0.728808581829071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0269775390625,
      "rewards/margins": 0.02430877648293972,
      "rewards/rejected": -0.05125732347369194,
      "step": 7710
    },
    {
      "epoch": 0.5624157651258515,
      "grad_norm": 2.153006753953898,
      "learning_rate": 5.690643170074818e-07,
      "log_odds_chosen": 1.1276366710662842,
      "log_odds_ratio": -0.405517578125,
      "logits/chosen": -0.953906238079071,
      "logits/rejected": -0.865429699420929,
      "logps/chosen": -0.5252929925918579,
      "logps/rejected": -1.1298828125,
      "loss": 0.7901,
      "nll_loss": 0.8169921636581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02626342698931694,
      "rewards/margins": 0.03019256517291069,
      "rewards/rejected": -0.05646972730755806,
      "step": 7720
    },
    {
      "epoch": 0.5631442829563253,
      "grad_norm": 2.18651432253299,
      "learning_rate": 5.686961097110808e-07,
      "log_odds_chosen": 0.889331042766571,
      "log_odds_ratio": -0.513378918170929,
      "logits/chosen": -0.985546886920929,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.5533202886581421,
      "logps/rejected": -1.0851562023162842,
      "loss": 0.7943,
      "nll_loss": 0.741015613079071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02769775316119194,
      "rewards/margins": 0.02658386155962944,
      "rewards/rejected": -0.05422363430261612,
      "step": 7730
    },
    {
      "epoch": 0.5638728007867992,
      "grad_norm": 2.2285151235409404,
      "learning_rate": 5.683286162253946e-07,
      "log_odds_chosen": 0.9374023675918579,
      "log_odds_ratio": -0.4955078065395355,
      "logits/chosen": -0.916796863079071,
      "logits/rejected": -0.852734386920929,
      "logps/chosen": -0.581347644329071,
      "logps/rejected": -1.1025390625,
      "loss": 0.7869,
      "nll_loss": 0.8167968988418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02907714806497097,
      "rewards/margins": 0.02606659010052681,
      "rewards/rejected": -0.05512695387005806,
      "step": 7740
    },
    {
      "epoch": 0.5646013186172731,
      "grad_norm": 1.7684247246261315,
      "learning_rate": 5.679618342470648e-07,
      "log_odds_chosen": 1.1956055164337158,
      "log_odds_ratio": -0.42158204317092896,
      "logits/chosen": -0.976367175579071,
      "logits/rejected": -0.8648437261581421,
      "logps/chosen": -0.5723632574081421,
      "logps/rejected": -1.310937523841858,
      "loss": 0.7965,
      "nll_loss": 0.799609363079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02860107459127903,
      "rewards/margins": 0.03692779690027237,
      "rewards/rejected": -0.06553955376148224,
      "step": 7750
    },
    {
      "epoch": 0.565329836447747,
      "grad_norm": 1.7385965092948865,
      "learning_rate": 5.675957614831248e-07,
      "log_odds_chosen": 1.1144530773162842,
      "log_odds_ratio": -0.4383789002895355,
      "logits/chosen": -0.975781261920929,
      "logits/rejected": -0.8603515625,
      "logps/chosen": -0.509960949420929,
      "logps/rejected": -1.1375000476837158,
      "loss": 0.7724,
      "nll_loss": 0.7796875238418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.025482177734375,
      "rewards/margins": 0.03137512132525444,
      "rewards/rejected": -0.05680542066693306,
      "step": 7760
    },
    {
      "epoch": 0.5660583542782209,
      "grad_norm": 1.9618732674761339,
      "learning_rate": 5.672303956509398e-07,
      "log_odds_chosen": 1.0178711414337158,
      "log_odds_ratio": -0.4859375059604645,
      "logits/chosen": -0.978320300579071,
      "logits/rejected": -0.8792968988418579,
      "logps/chosen": -0.5494140386581421,
      "logps/rejected": -1.1357421875,
      "loss": 0.7791,
      "nll_loss": 0.7845703363418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02749023400247097,
      "rewards/margins": 0.02926940843462944,
      "rewards/rejected": -0.05677490308880806,
      "step": 7770
    },
    {
      "epoch": 0.5667868721086948,
      "grad_norm": 2.065289657241756,
      "learning_rate": 5.668657344781478e-07,
      "log_odds_chosen": 0.8218749761581421,
      "log_odds_ratio": -0.5244140625,
      "logits/chosen": -0.9482421875,
      "logits/rejected": -0.8443359136581421,
      "logps/chosen": -0.5780273675918579,
      "logps/rejected": -1.083984375,
      "loss": 0.7736,
      "nll_loss": 0.7255859375,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02888183668255806,
      "rewards/margins": 0.02535095252096653,
      "rewards/rejected": -0.05423583835363388,
      "step": 7780
    },
    {
      "epoch": 0.5675153899391687,
      "grad_norm": 1.9619662105193927,
      "learning_rate": 5.66501775702599e-07,
      "log_odds_chosen": 1.157812476158142,
      "log_odds_ratio": -0.4217285215854645,
      "logits/chosen": -0.9203125238418579,
      "logits/rejected": -0.832812488079071,
      "logps/chosen": -0.5264648199081421,
      "logps/rejected": -1.205078125,
      "loss": 0.7864,
      "nll_loss": 0.778124988079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02629394456744194,
      "rewards/margins": 0.03394775465130806,
      "rewards/rejected": -0.060272216796875,
      "step": 7790
    },
    {
      "epoch": 0.5682439077696426,
      "grad_norm": 2.57605318312265,
      "learning_rate": 5.661385170722979e-07,
      "log_odds_chosen": 1.1648437976837158,
      "log_odds_ratio": -0.43115234375,
      "logits/chosen": -0.9634765386581421,
      "logits/rejected": -0.845898449420929,
      "logps/chosen": -0.4881835877895355,
      "logps/rejected": -1.149999976158142,
      "loss": 0.7841,
      "nll_loss": 0.73779296875,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02439575269818306,
      "rewards/margins": 0.03309021145105362,
      "rewards/rejected": -0.05747070163488388,
      "step": 7800
    },
    {
      "epoch": 0.5689724256001165,
      "grad_norm": 2.4345938935455136,
      "learning_rate": 5.657759563453445e-07,
      "log_odds_chosen": 0.8124023675918579,
      "log_odds_ratio": -0.547656238079071,
      "logits/chosen": -0.9427734613418579,
      "logits/rejected": -0.8792968988418579,
      "logps/chosen": -0.5909179449081421,
      "logps/rejected": -1.084570288658142,
      "loss": 0.7924,
      "nll_loss": 0.8291015625,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.029541015625,
      "rewards/margins": 0.0246124267578125,
      "rewards/rejected": -0.05415039137005806,
      "step": 7810
    },
    {
      "epoch": 0.5697009434305904,
      "grad_norm": 2.3862951363447564,
      "learning_rate": 5.654140912898758e-07,
      "log_odds_chosen": 1.1452147960662842,
      "log_odds_ratio": -0.42792969942092896,
      "logits/chosen": -0.954882800579071,
      "logits/rejected": -0.8853515386581421,
      "logps/chosen": -0.51171875,
      "logps/rejected": -1.176171898841858,
      "loss": 0.779,
      "nll_loss": 0.7281249761581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02557983435690403,
      "rewards/margins": 0.0332489013671875,
      "rewards/rejected": -0.05878906324505806,
      "step": 7820
    },
    {
      "epoch": 0.5704294612610644,
      "grad_norm": 3.1766324816450195,
      "learning_rate": 5.650529196840087e-07,
      "log_odds_chosen": 0.96142578125,
      "log_odds_ratio": -0.4866699278354645,
      "logits/chosen": -0.956835925579071,
      "logits/rejected": -0.8999999761581421,
      "logps/chosen": -0.559765636920929,
      "logps/rejected": -1.1189453601837158,
      "loss": 0.7679,
      "nll_loss": 0.7623046636581421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.027984619140625,
      "rewards/margins": 0.02794189378619194,
      "rewards/rejected": -0.05593261867761612,
      "step": 7830
    },
    {
      "epoch": 0.5711579790915383,
      "grad_norm": 1.9296691219718274,
      "learning_rate": 5.646924393157821e-07,
      "log_odds_chosen": 1.0281250476837158,
      "log_odds_ratio": -0.4598632752895355,
      "logits/chosen": -1.016992211341858,
      "logits/rejected": -0.8984375,
      "logps/chosen": -0.578125,
      "logps/rejected": -1.2166016101837158,
      "loss": 0.7802,
      "nll_loss": 0.785937488079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02891235426068306,
      "rewards/margins": 0.03191795200109482,
      "rewards/rejected": -0.06085815280675888,
      "step": 7840
    },
    {
      "epoch": 0.5718864969220122,
      "grad_norm": 1.961897717760353,
      "learning_rate": 5.643326479831004e-07,
      "log_odds_chosen": 0.9463866949081421,
      "log_odds_ratio": -0.4828124940395355,
      "logits/chosen": -0.9912109375,
      "logits/rejected": -0.8843749761581421,
      "logps/chosen": -0.536328136920929,
      "logps/rejected": -1.075585961341858,
      "loss": 0.7654,
      "nll_loss": 0.726367175579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02680053748190403,
      "rewards/margins": 0.02699127234518528,
      "rewards/rejected": -0.05378418043255806,
      "step": 7850
    },
    {
      "epoch": 0.5726150147524861,
      "grad_norm": 2.228128828656122,
      "learning_rate": 5.639735434936769e-07,
      "log_odds_chosen": 1.221289038658142,
      "log_odds_ratio": -0.399658203125,
      "logits/chosen": -0.9935547113418579,
      "logits/rejected": -0.8570312261581421,
      "logps/chosen": -0.5121093988418579,
      "logps/rejected": -1.24609375,
      "loss": 0.7623,
      "nll_loss": 0.759765625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.025604248046875,
      "rewards/margins": 0.036773681640625,
      "rewards/rejected": -0.06235351413488388,
      "step": 7860
    },
    {
      "epoch": 0.57334353258296,
      "grad_norm": 1.7662234838024482,
      "learning_rate": 5.636151236649777e-07,
      "log_odds_chosen": 1.2137329578399658,
      "log_odds_ratio": -0.43999022245407104,
      "logits/chosen": -0.9771484136581421,
      "logits/rejected": -0.8701171875,
      "logps/chosen": -0.5337890386581421,
      "logps/rejected": -1.2717773914337158,
      "loss": 0.7857,
      "nll_loss": 0.739453136920929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02669067308306694,
      "rewards/margins": 0.036914825439453125,
      "rewards/rejected": -0.063568115234375,
      "step": 7870
    },
    {
      "epoch": 0.5740720504134339,
      "grad_norm": 2.076171389255376,
      "learning_rate": 5.632573863241661e-07,
      "log_odds_chosen": 0.899609386920929,
      "log_odds_ratio": -0.50439453125,
      "logits/chosen": -1.0076172351837158,
      "logits/rejected": -0.889843761920929,
      "logps/chosen": -0.568164050579071,
      "logps/rejected": -1.0802733898162842,
      "loss": 0.8031,
      "nll_loss": 0.7978515625,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0284423828125,
      "rewards/margins": 0.02552947960793972,
      "rewards/rejected": -0.053985595703125,
      "step": 7880
    },
    {
      "epoch": 0.5748005682439078,
      "grad_norm": 2.101262188655448,
      "learning_rate": 5.629003293080469e-07,
      "log_odds_chosen": 0.9512695074081421,
      "log_odds_ratio": -0.4793457090854645,
      "logits/chosen": -0.9652343988418579,
      "logits/rejected": -0.880859375,
      "logps/chosen": -0.586718738079071,
      "logps/rejected": -1.153906226158142,
      "loss": 0.7732,
      "nll_loss": 0.8187500238418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02934570237994194,
      "rewards/margins": 0.02837982214987278,
      "rewards/rejected": -0.0577392578125,
      "step": 7890
    },
    {
      "epoch": 0.5755290860743817,
      "grad_norm": 1.7490307872914845,
      "learning_rate": 5.62543950463012e-07,
      "log_odds_chosen": 1.025537133216858,
      "log_odds_ratio": -0.4737304747104645,
      "logits/chosen": -0.9447265863418579,
      "logits/rejected": -0.8626953363418579,
      "logps/chosen": -0.5604492425918579,
      "logps/rejected": -1.1873047351837158,
      "loss": 0.7745,
      "nll_loss": 0.7818359136581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02803344652056694,
      "rewards/margins": 0.03135833889245987,
      "rewards/rejected": -0.05939941480755806,
      "step": 7900
    },
    {
      "epoch": 0.5762576039048556,
      "grad_norm": 2.9992111177484957,
      "learning_rate": 5.621882476449851e-07,
      "log_odds_chosen": 0.997119128704071,
      "log_odds_ratio": -0.4476562440395355,
      "logits/chosen": -1.021875023841858,
      "logits/rejected": -0.8968750238418579,
      "logps/chosen": -0.560742199420929,
      "logps/rejected": -1.1574218273162842,
      "loss": 0.7608,
      "nll_loss": 0.730664074420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02807006798684597,
      "rewards/margins": 0.02981262281537056,
      "rewards/rejected": -0.05787353590130806,
      "step": 7910
    },
    {
      "epoch": 0.5769861217353295,
      "grad_norm": 1.9705108507121523,
      "learning_rate": 5.618332187193684e-07,
      "log_odds_chosen": 1.177099585533142,
      "log_odds_ratio": -0.4666992127895355,
      "logits/chosen": -1.0105469226837158,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.553417980670929,
      "logps/rejected": -1.2703125476837158,
      "loss": 0.8016,
      "nll_loss": 0.7789062261581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.027679443359375,
      "rewards/margins": 0.03586120530962944,
      "rewards/rejected": -0.06356201320886612,
      "step": 7920
    },
    {
      "epoch": 0.5777146395658034,
      "grad_norm": 1.9277988149125065,
      "learning_rate": 5.614788615609884e-07,
      "log_odds_chosen": 0.933642566204071,
      "log_odds_ratio": -0.48291015625,
      "logits/chosen": -0.9794921875,
      "logits/rejected": -0.8462890386581421,
      "logps/chosen": -0.5625976324081421,
      "logps/rejected": -1.1396484375,
      "loss": 0.775,
      "nll_loss": 0.737988293170929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02812499925494194,
      "rewards/margins": 0.02884368970990181,
      "rewards/rejected": -0.05699462816119194,
      "step": 7930
    },
    {
      "epoch": 0.5784431573962773,
      "grad_norm": 1.801947714413946,
      "learning_rate": 5.611251740540419e-07,
      "log_odds_chosen": 1.1028320789337158,
      "log_odds_ratio": -0.4656738340854645,
      "logits/chosen": -0.95703125,
      "logits/rejected": -0.8853515386581421,
      "logps/chosen": -0.556445300579071,
      "logps/rejected": -1.2292969226837158,
      "loss": 0.7819,
      "nll_loss": 0.7914062738418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02783203125,
      "rewards/margins": 0.03366699069738388,
      "rewards/rejected": -0.06148071214556694,
      "step": 7940
    },
    {
      "epoch": 0.5791716752267512,
      "grad_norm": 1.8895524072079786,
      "learning_rate": 5.607721540920444e-07,
      "log_odds_chosen": 1.160546898841858,
      "log_odds_ratio": -0.44853514432907104,
      "logits/chosen": -0.983593761920929,
      "logits/rejected": -0.870312511920929,
      "logps/chosen": -0.5655273199081421,
      "logps/rejected": -1.259179711341858,
      "loss": 0.7656,
      "nll_loss": 0.823437511920929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02827148512005806,
      "rewards/margins": 0.03470153734087944,
      "rewards/rejected": -0.06300048530101776,
      "step": 7950
    },
    {
      "epoch": 0.5799001930572251,
      "grad_norm": 2.7622236961058,
      "learning_rate": 5.604197995777755e-07,
      "log_odds_chosen": 1.1730468273162842,
      "log_odds_ratio": -0.474853515625,
      "logits/chosen": -1.0212891101837158,
      "logits/rejected": -0.876953125,
      "logps/chosen": -0.576953113079071,
      "logps/rejected": -1.349218726158142,
      "loss": 0.7698,
      "nll_loss": 0.735156238079071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02882080152630806,
      "rewards/margins": 0.03871459886431694,
      "rewards/rejected": -0.06751708686351776,
      "step": 7960
    },
    {
      "epoch": 0.580628710887699,
      "grad_norm": 1.9902126109472433,
      "learning_rate": 5.600681084232282e-07,
      "log_odds_chosen": 1.105371117591858,
      "log_odds_ratio": -0.416748046875,
      "logits/chosen": -1.0076172351837158,
      "logits/rejected": -0.8828125,
      "logps/chosen": -0.568652331829071,
      "logps/rejected": -1.2195312976837158,
      "loss": 0.7636,
      "nll_loss": 0.7730468511581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02844848670065403,
      "rewards/margins": 0.03252563625574112,
      "rewards/rejected": -0.06096191331744194,
      "step": 7970
    },
    {
      "epoch": 0.5813572287181729,
      "grad_norm": 1.8509084412192118,
      "learning_rate": 5.597170785495563e-07,
      "log_odds_chosen": 1.002050757408142,
      "log_odds_ratio": -0.48212891817092896,
      "logits/chosen": -1.01171875,
      "logits/rejected": -0.88671875,
      "logps/chosen": -0.5409179925918579,
      "logps/rejected": -1.086328148841858,
      "loss": 0.7724,
      "nll_loss": 0.7447265386581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02704162523150444,
      "rewards/margins": 0.02726135216653347,
      "rewards/rejected": -0.054290771484375,
      "step": 7980
    },
    {
      "epoch": 0.5820857465486468,
      "grad_norm": 1.8286038861978076,
      "learning_rate": 5.593667078870224e-07,
      "log_odds_chosen": 1.0832397937774658,
      "log_odds_ratio": -0.4806152284145355,
      "logits/chosen": -0.950390636920929,
      "logits/rejected": -0.866015613079071,
      "logps/chosen": -0.558789074420929,
      "logps/rejected": -1.18359375,
      "loss": 0.7655,
      "nll_loss": 0.845410168170929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02794799767434597,
      "rewards/margins": 0.03128509595990181,
      "rewards/rejected": -0.05919189378619194,
      "step": 7990
    },
    {
      "epoch": 0.5828142643791207,
      "grad_norm": 2.56694767877769,
      "learning_rate": 5.590169943749475e-07,
      "log_odds_chosen": 0.989001452922821,
      "log_odds_ratio": -0.4578613340854645,
      "logits/chosen": -0.9521484375,
      "logits/rejected": -0.82177734375,
      "logps/chosen": -0.597460925579071,
      "logps/rejected": -1.1796875,
      "loss": 0.7733,
      "nll_loss": 0.762499988079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02990112267434597,
      "rewards/margins": 0.02907104417681694,
      "rewards/rejected": -0.05893554538488388,
      "step": 8000
    },
    {
      "epoch": 0.5835427822095945,
      "grad_norm": 1.9517159285591865,
      "learning_rate": 5.586679359616591e-07,
      "log_odds_chosen": 1.0618164539337158,
      "log_odds_ratio": -0.4754394590854645,
      "logits/chosen": -0.9427734613418579,
      "logits/rejected": -0.8832031488418579,
      "logps/chosen": -0.551464855670929,
      "logps/rejected": -1.194726586341858,
      "loss": 0.7838,
      "nll_loss": 0.7749999761581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02757568284869194,
      "rewards/margins": 0.03215331956744194,
      "rewards/rejected": -0.05973510816693306,
      "step": 8010
    },
    {
      "epoch": 0.5842713000400684,
      "grad_norm": 1.8511644786529629,
      "learning_rate": 5.583195306044417e-07,
      "log_odds_chosen": 1.00146484375,
      "log_odds_ratio": -0.4588867127895355,
      "logits/chosen": -0.96484375,
      "logits/rejected": -0.8619140386581421,
      "logps/chosen": -0.558789074420929,
      "logps/rejected": -1.1544921398162842,
      "loss": 0.7628,
      "nll_loss": 0.7447265386581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02794189378619194,
      "rewards/margins": 0.02978515625,
      "rewards/rejected": -0.05767822265625,
      "step": 8020
    },
    {
      "epoch": 0.5849998178705423,
      "grad_norm": 1.9204675178014814,
      "learning_rate": 5.579717762694858e-07,
      "log_odds_chosen": 1.0769531726837158,
      "log_odds_ratio": -0.4354492127895355,
      "logits/chosen": -0.948046863079071,
      "logits/rejected": -0.8583984375,
      "logps/chosen": -0.53955078125,
      "logps/rejected": -1.1759765148162842,
      "loss": 0.7842,
      "nll_loss": 0.744335949420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02696533128619194,
      "rewards/margins": 0.03185119479894638,
      "rewards/rejected": -0.05881347507238388,
      "step": 8030
    },
    {
      "epoch": 0.5857283357010162,
      "grad_norm": 1.921230706134836,
      "learning_rate": 5.576246709318382e-07,
      "log_odds_chosen": 1.048193335533142,
      "log_odds_ratio": -0.44731444120407104,
      "logits/chosen": -0.911328136920929,
      "logits/rejected": -0.7953125238418579,
      "logps/chosen": -0.556445300579071,
      "logps/rejected": -1.189843773841858,
      "loss": 0.7918,
      "nll_loss": 0.787304699420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02782592736184597,
      "rewards/margins": 0.03164062649011612,
      "rewards/rejected": -0.05943603441119194,
      "step": 8040
    },
    {
      "epoch": 0.5864568535314901,
      "grad_norm": 2.341589543107691,
      "learning_rate": 5.572782125753528e-07,
      "log_odds_chosen": 0.858935534954071,
      "log_odds_ratio": -0.5147460699081421,
      "logits/chosen": -0.9117187261581421,
      "logits/rejected": -0.81640625,
      "logps/chosen": -0.572070300579071,
      "logps/rejected": -1.102929711341858,
      "loss": 0.7686,
      "nll_loss": 0.7691406011581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02862548828125,
      "rewards/margins": 0.02653808519244194,
      "rewards/rejected": -0.05518798902630806,
      "step": 8050
    },
    {
      "epoch": 0.5871853713619641,
      "grad_norm": 1.917227873607951,
      "learning_rate": 5.569323991926414e-07,
      "log_odds_chosen": 0.723950207233429,
      "log_odds_ratio": -0.5693359375,
      "logits/chosen": -0.922656238079071,
      "logits/rejected": -0.825976550579071,
      "logps/chosen": -0.549121081829071,
      "logps/rejected": -0.9673827886581421,
      "loss": 0.7717,
      "nll_loss": 0.7625976800918579,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02745971642434597,
      "rewards/margins": 0.02088623121380806,
      "rewards/rejected": -0.04835815355181694,
      "step": 8060
    },
    {
      "epoch": 0.587913889192438,
      "grad_norm": 2.4353630715677355,
      "learning_rate": 5.565872287850237e-07,
      "log_odds_chosen": 0.86279296875,
      "log_odds_ratio": -0.5420898199081421,
      "logits/chosen": -0.970703125,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.5594726800918579,
      "logps/rejected": -1.0759766101837158,
      "loss": 0.7783,
      "nll_loss": 0.78125,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02797851525247097,
      "rewards/margins": 0.02581939660012722,
      "rewards/rejected": -0.05379638820886612,
      "step": 8070
    },
    {
      "epoch": 0.5886424070229119,
      "grad_norm": 1.7351435676872171,
      "learning_rate": 5.56242699362481e-07,
      "log_odds_chosen": 0.9808593988418579,
      "log_odds_ratio": -0.47319334745407104,
      "logits/chosen": -0.9830077886581421,
      "logits/rejected": -0.865429699420929,
      "logps/chosen": -0.540722668170929,
      "logps/rejected": -1.126562476158142,
      "loss": 0.7703,
      "nll_loss": 0.73291015625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02702636644244194,
      "rewards/margins": 0.02932128868997097,
      "rewards/rejected": -0.05635986477136612,
      "step": 8080
    },
    {
      "epoch": 0.5893709248533858,
      "grad_norm": 2.0761769469187197,
      "learning_rate": 5.558988089436059e-07,
      "log_odds_chosen": 1.0115234851837158,
      "log_odds_ratio": -0.4776855409145355,
      "logits/chosen": -0.9853515625,
      "logits/rejected": -0.8763672113418579,
      "logps/chosen": -0.5542968511581421,
      "logps/rejected": -1.163671851158142,
      "loss": 0.7746,
      "nll_loss": 0.7183593511581421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02771606482565403,
      "rewards/margins": 0.03049926832318306,
      "rewards/rejected": -0.05825195461511612,
      "step": 8090
    },
    {
      "epoch": 0.5900994426838597,
      "grad_norm": 1.7244029336335267,
      "learning_rate": 5.555555555555555e-07,
      "log_odds_chosen": 1.0603516101837158,
      "log_odds_ratio": -0.4595703184604645,
      "logits/chosen": -0.9458984136581421,
      "logits/rejected": -0.822070300579071,
      "logps/chosen": -0.592480480670929,
      "logps/rejected": -1.270117163658142,
      "loss": 0.7581,
      "nll_loss": 0.740234375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02963867224752903,
      "rewards/margins": 0.033935546875,
      "rewards/rejected": -0.06350097805261612,
      "step": 8100
    },
    {
      "epoch": 0.5908279605143336,
      "grad_norm": 2.0187029929475404,
      "learning_rate": 5.552129372340039e-07,
      "log_odds_chosen": 1.047753930091858,
      "log_odds_ratio": -0.45805662870407104,
      "logits/chosen": -0.9560546875,
      "logits/rejected": -0.8638671636581421,
      "logps/chosen": -0.535937488079071,
      "logps/rejected": -1.1759765148162842,
      "loss": 0.7728,
      "nll_loss": 0.7152343988418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02681884728372097,
      "rewards/margins": 0.03199157863855362,
      "rewards/rejected": -0.05882568284869194,
      "step": 8110
    },
    {
      "epoch": 0.5915564783448075,
      "grad_norm": 1.8442160110679295,
      "learning_rate": 5.548709520230942e-07,
      "log_odds_chosen": 0.9742187261581421,
      "log_odds_ratio": -0.4771484434604645,
      "logits/chosen": -0.968945324420929,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.5677734613418579,
      "logps/rejected": -1.1443359851837158,
      "loss": 0.75,
      "nll_loss": 0.759960949420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02839355543255806,
      "rewards/margins": 0.02882995642721653,
      "rewards/rejected": -0.05717773362994194,
      "step": 8120
    },
    {
      "epoch": 0.5922849961752814,
      "grad_norm": 2.1665861326827995,
      "learning_rate": 5.545295979753924e-07,
      "log_odds_chosen": 1.096777319908142,
      "log_odds_ratio": -0.43828123807907104,
      "logits/chosen": -0.9837890863418579,
      "logits/rejected": -0.869140625,
      "logps/chosen": -0.510449230670929,
      "logps/rejected": -1.167382836341858,
      "loss": 0.7656,
      "nll_loss": 0.6865234375,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02552490308880806,
      "rewards/margins": 0.032867431640625,
      "rewards/rejected": -0.05837402492761612,
      "step": 8130
    },
    {
      "epoch": 0.5930135140057553,
      "grad_norm": 1.9478879686444173,
      "learning_rate": 5.541888731518408e-07,
      "log_odds_chosen": 1.107031226158142,
      "log_odds_ratio": -0.4554687440395355,
      "logits/chosen": -0.9371093511581421,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.5521484613418579,
      "logps/rejected": -1.2107422351837158,
      "loss": 0.7768,
      "nll_loss": 0.7840820550918579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02761230431497097,
      "rewards/margins": 0.03296203538775444,
      "rewards/rejected": -0.060546875,
      "step": 8140
    },
    {
      "epoch": 0.5937420318362292,
      "grad_norm": 6.402184672897564,
      "learning_rate": 5.538487756217113e-07,
      "log_odds_chosen": 1.0725586414337158,
      "log_odds_ratio": -0.44658201932907104,
      "logits/chosen": -0.9521484375,
      "logits/rejected": -0.839062511920929,
      "logps/chosen": -0.513476550579071,
      "logps/rejected": -1.1462891101837158,
      "loss": 0.7628,
      "nll_loss": 0.7007812261581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02570190466940403,
      "rewards/margins": 0.03163909912109375,
      "rewards/rejected": -0.05734863132238388,
      "step": 8150
    },
    {
      "epoch": 0.5944705496667031,
      "grad_norm": 1.9977041511947455,
      "learning_rate": 5.535093034625597e-07,
      "log_odds_chosen": 1.0597655773162842,
      "log_odds_ratio": -0.4646972715854645,
      "logits/chosen": -0.9400390386581421,
      "logits/rejected": -0.8597656488418579,
      "logps/chosen": -0.5225585699081421,
      "logps/rejected": -1.172265648841858,
      "loss": 0.7775,
      "nll_loss": 0.734570324420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.026123046875,
      "rewards/margins": 0.03248138353228569,
      "rewards/rejected": -0.05864868313074112,
      "step": 8160
    },
    {
      "epoch": 0.595199067497177,
      "grad_norm": 2.2529914293964315,
      "learning_rate": 5.531704547601796e-07,
      "log_odds_chosen": 1.1433594226837158,
      "log_odds_ratio": -0.428466796875,
      "logits/chosen": -0.969531238079071,
      "logits/rejected": -0.8642578125,
      "logps/chosen": -0.563281238079071,
      "logps/rejected": -1.2412109375,
      "loss": 0.7778,
      "nll_loss": 0.734570324420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02816162072122097,
      "rewards/margins": 0.03394470363855362,
      "rewards/rejected": -0.06208496168255806,
      "step": 8170
    },
    {
      "epoch": 0.5959275853276509,
      "grad_norm": 2.425887168093698,
      "learning_rate": 5.528322276085582e-07,
      "log_odds_chosen": 0.9320312738418579,
      "log_odds_ratio": -0.49555665254592896,
      "logits/chosen": -1.006445288658142,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -0.5257812738418579,
      "logps/rejected": -1.0792968273162842,
      "loss": 0.7837,
      "nll_loss": 0.7484375238418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02631836012005806,
      "rewards/margins": 0.027704620733857155,
      "rewards/rejected": -0.05402221530675888,
      "step": 8180
    },
    {
      "epoch": 0.5966561031581248,
      "grad_norm": 2.478847502288701,
      "learning_rate": 5.5249462010983e-07,
      "log_odds_chosen": 1.0873534679412842,
      "log_odds_ratio": -0.4476562440395355,
      "logits/chosen": -0.9287109375,
      "logits/rejected": -0.820507824420929,
      "logps/chosen": -0.527539074420929,
      "logps/rejected": -1.2029297351837158,
      "loss": 0.7938,
      "nll_loss": 0.771289050579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02636108361184597,
      "rewards/margins": 0.03378906100988388,
      "rewards/rejected": -0.06010742112994194,
      "step": 8190
    },
    {
      "epoch": 0.5973846209885987,
      "grad_norm": 2.0997962561469987,
      "learning_rate": 5.521576303742327e-07,
      "log_odds_chosen": 0.8632446527481079,
      "log_odds_ratio": -0.539746105670929,
      "logits/chosen": -0.927734375,
      "logits/rejected": -0.8291015625,
      "logps/chosen": -0.562792956829071,
      "logps/rejected": -1.076757788658142,
      "loss": 0.7878,
      "nll_loss": 0.774218738079071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02813110314309597,
      "rewards/margins": 0.025717545300722122,
      "rewards/rejected": -0.05381469801068306,
      "step": 8200
    },
    {
      "epoch": 0.5981131388190726,
      "grad_norm": 2.2056817430739413,
      "learning_rate": 5.518212565200631e-07,
      "log_odds_chosen": 1.13037109375,
      "log_odds_ratio": -0.4339355528354645,
      "logits/chosen": -0.954296886920929,
      "logits/rejected": -0.858203113079071,
      "logps/chosen": -0.537402331829071,
      "logps/rejected": -1.196874976158142,
      "loss": 0.7691,
      "nll_loss": 0.746386706829071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02686767652630806,
      "rewards/margins": 0.03298645094037056,
      "rewards/rejected": -0.05986328050494194,
      "step": 8210
    },
    {
      "epoch": 0.5988416566495465,
      "grad_norm": 2.1023292287258872,
      "learning_rate": 5.514854966736319e-07,
      "log_odds_chosen": 1.1749999523162842,
      "log_odds_ratio": -0.4310058653354645,
      "logits/chosen": -0.9833984375,
      "logits/rejected": -0.847460925579071,
      "logps/chosen": -0.586230456829071,
      "logps/rejected": -1.3039062023162842,
      "loss": 0.7708,
      "nll_loss": 0.7578125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02932128868997097,
      "rewards/margins": 0.03586425632238388,
      "rewards/rejected": -0.06517334282398224,
      "step": 8220
    },
    {
      "epoch": 0.5995701744800204,
      "grad_norm": 2.0669120216887023,
      "learning_rate": 5.511503489692212e-07,
      "log_odds_chosen": 1.166894555091858,
      "log_odds_ratio": -0.4287109375,
      "logits/chosen": -0.9759765863418579,
      "logits/rejected": -0.8677734136581421,
      "logps/chosen": -0.589648425579071,
      "logps/rejected": -1.305078148841858,
      "loss": 0.7648,
      "nll_loss": 0.754687488079071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02949218824505806,
      "rewards/margins": 0.03572998195886612,
      "rewards/rejected": -0.06522216647863388,
      "step": 8230
    },
    {
      "epoch": 0.6002986923104943,
      "grad_norm": 1.8730448914341167,
      "learning_rate": 5.508158115490397e-07,
      "log_odds_chosen": 1.08935546875,
      "log_odds_ratio": -0.430908203125,
      "logits/chosen": -0.9644531011581421,
      "logits/rejected": -0.8500000238418579,
      "logps/chosen": -0.54150390625,
      "logps/rejected": -1.167382836341858,
      "loss": 0.7483,
      "nll_loss": 0.7607421875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02708740159869194,
      "rewards/margins": 0.0312652587890625,
      "rewards/rejected": -0.05839843675494194,
      "step": 8240
    },
    {
      "epoch": 0.6010272101409682,
      "grad_norm": 2.0592474869613446,
      "learning_rate": 5.504818825631803e-07,
      "log_odds_chosen": 1.0813477039337158,
      "log_odds_ratio": -0.4378418028354645,
      "logits/chosen": -0.947460949420929,
      "logits/rejected": -0.846875011920929,
      "logps/chosen": -0.4859375059604645,
      "logps/rejected": -1.1013672351837158,
      "loss": 0.7686,
      "nll_loss": 0.7406250238418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02430419996380806,
      "rewards/margins": 0.0308074951171875,
      "rewards/rejected": -0.05510253831744194,
      "step": 8250
    },
    {
      "epoch": 0.6017557279714421,
      "grad_norm": 2.152366300095569,
      "learning_rate": 5.50148560169577e-07,
      "log_odds_chosen": 1.394873023033142,
      "log_odds_ratio": -0.3858398497104645,
      "logits/chosen": -0.951171875,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.554492175579071,
      "logps/rejected": -1.429296851158142,
      "loss": 0.7606,
      "nll_loss": 0.7845703363418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.027740478515625,
      "rewards/margins": 0.04372863844037056,
      "rewards/rejected": -0.07144775241613388,
      "step": 8260
    },
    {
      "epoch": 0.602484245801916,
      "grad_norm": 2.2925016626943036,
      "learning_rate": 5.498158425339617e-07,
      "log_odds_chosen": 1.097070336341858,
      "log_odds_ratio": -0.44331055879592896,
      "logits/chosen": -0.9990234375,
      "logits/rejected": -0.8734375238418579,
      "logps/chosen": -0.566699206829071,
      "logps/rejected": -1.2166016101837158,
      "loss": 0.7765,
      "nll_loss": 0.7904297113418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02832641638815403,
      "rewards/margins": 0.03249511867761612,
      "rewards/rejected": -0.06085205078125,
      "step": 8270
    },
    {
      "epoch": 0.6032127636323898,
      "grad_norm": 1.8449001997289263,
      "learning_rate": 5.494837278298225e-07,
      "log_odds_chosen": 0.989916980266571,
      "log_odds_ratio": -0.4872070252895355,
      "logits/chosen": -0.927929699420929,
      "logits/rejected": -0.853710949420929,
      "logps/chosen": -0.529296875,
      "logps/rejected": -1.103515625,
      "loss": 0.7636,
      "nll_loss": 0.7250000238418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02647094801068306,
      "rewards/margins": 0.028730202466249466,
      "rewards/rejected": -0.05518798902630806,
      "step": 8280
    },
    {
      "epoch": 0.6039412814628639,
      "grad_norm": 2.1426612916581935,
      "learning_rate": 5.491522142383613e-07,
      "log_odds_chosen": 1.388671875,
      "log_odds_ratio": -0.36884766817092896,
      "logits/chosen": -0.95703125,
      "logits/rejected": -0.8521484136581421,
      "logps/chosen": -0.5244140625,
      "logps/rejected": -1.3894531726837158,
      "loss": 0.7766,
      "nll_loss": 0.7271484136581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02622070349752903,
      "rewards/margins": 0.04320678859949112,
      "rewards/rejected": -0.06943359225988388,
      "step": 8290
    },
    {
      "epoch": 0.6046697992933378,
      "grad_norm": 2.1005489438450096,
      "learning_rate": 5.488212999484518e-07,
      "log_odds_chosen": 0.820117175579071,
      "log_odds_ratio": -0.5262695550918579,
      "logits/chosen": -0.9482421875,
      "logits/rejected": -0.839062511920929,
      "logps/chosen": -0.5708984136581421,
      "logps/rejected": -1.088476538658142,
      "loss": 0.764,
      "nll_loss": 0.7748047113418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02855224534869194,
      "rewards/margins": 0.02588958665728569,
      "rewards/rejected": -0.05441894382238388,
      "step": 8300
    },
    {
      "epoch": 0.6053983171238116,
      "grad_norm": 1.8930183487273715,
      "learning_rate": 5.484909831565981e-07,
      "log_odds_chosen": 1.11328125,
      "log_odds_ratio": -0.4405273497104645,
      "logits/chosen": -0.999804675579071,
      "logits/rejected": -0.8642578125,
      "logps/chosen": -0.5638672113418579,
      "logps/rejected": -1.2664062976837158,
      "loss": 0.7391,
      "nll_loss": 0.744433581829071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02821044996380806,
      "rewards/margins": 0.03513183444738388,
      "rewards/rejected": -0.06333007663488388,
      "step": 8310
    },
    {
      "epoch": 0.6061268349542855,
      "grad_norm": 1.7755653442265216,
      "learning_rate": 5.481612620668933e-07,
      "log_odds_chosen": 0.9192870855331421,
      "log_odds_ratio": -0.49028319120407104,
      "logits/chosen": -0.986328125,
      "logits/rejected": -0.880859375,
      "logps/chosen": -0.5791991949081421,
      "logps/rejected": -1.1384766101837158,
      "loss": 0.788,
      "nll_loss": 0.7713867425918579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02897338941693306,
      "rewards/margins": 0.02798767015337944,
      "rewards/rejected": -0.05695800855755806,
      "step": 8320
    },
    {
      "epoch": 0.6068553527847594,
      "grad_norm": 2.364982607534129,
      "learning_rate": 5.478321348909789e-07,
      "log_odds_chosen": 1.246679663658142,
      "log_odds_ratio": -0.4325195252895355,
      "logits/chosen": -0.951367199420929,
      "logits/rejected": -0.8599609136581421,
      "logps/chosen": -0.547558605670929,
      "logps/rejected": -1.305273413658142,
      "loss": 0.7779,
      "nll_loss": 0.7818359136581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02736205980181694,
      "rewards/margins": 0.03791198879480362,
      "rewards/rejected": -0.06522216647863388,
      "step": 8330
    },
    {
      "epoch": 0.6075838706152333,
      "grad_norm": 2.0242434459488465,
      "learning_rate": 5.475035998480036e-07,
      "log_odds_chosen": 1.050683617591858,
      "log_odds_ratio": -0.47285157442092896,
      "logits/chosen": -0.969531238079071,
      "logits/rejected": -0.868945300579071,
      "logps/chosen": -0.5362304449081421,
      "logps/rejected": -1.1505858898162842,
      "loss": 0.7663,
      "nll_loss": 0.690234363079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02681274339556694,
      "rewards/margins": 0.03073730506002903,
      "rewards/rejected": -0.05756836012005806,
      "step": 8340
    },
    {
      "epoch": 0.6083123884457072,
      "grad_norm": 2.105019507742359,
      "learning_rate": 5.471756551645828e-07,
      "log_odds_chosen": 1.0353515148162842,
      "log_odds_ratio": -0.47758787870407104,
      "logits/chosen": -0.9593750238418579,
      "logits/rejected": -0.867968738079071,
      "logps/chosen": -0.5743163824081421,
      "logps/rejected": -1.181249976158142,
      "loss": 0.7702,
      "nll_loss": 0.788867175579071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02869873121380806,
      "rewards/margins": 0.03037109412252903,
      "rewards/rejected": -0.05906982347369194,
      "step": 8350
    },
    {
      "epoch": 0.6090409062761811,
      "grad_norm": 2.1732640984499962,
      "learning_rate": 5.46848299074759e-07,
      "log_odds_chosen": 1.1045043468475342,
      "log_odds_ratio": -0.4488769471645355,
      "logits/chosen": -0.9615234136581421,
      "logits/rejected": -0.8525390625,
      "logps/chosen": -0.55517578125,
      "logps/rejected": -1.254492163658142,
      "loss": 0.7966,
      "nll_loss": 0.7314453125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.027740478515625,
      "rewards/margins": 0.034976959228515625,
      "rewards/rejected": -0.06275634467601776,
      "step": 8360
    },
    {
      "epoch": 0.609769424106655,
      "grad_norm": 1.8698615493370774,
      "learning_rate": 5.465215298199609e-07,
      "log_odds_chosen": 1.213952660560608,
      "log_odds_ratio": -0.4251464903354645,
      "logits/chosen": -0.9248046875,
      "logits/rejected": -0.8646484613418579,
      "logps/chosen": -0.517285168170929,
      "logps/rejected": -1.218164086341858,
      "loss": 0.7654,
      "nll_loss": 0.692187488079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.025848388671875,
      "rewards/margins": 0.03498535230755806,
      "rewards/rejected": -0.06090087816119194,
      "step": 8370
    },
    {
      "epoch": 0.6104979419371289,
      "grad_norm": 1.8730702218797366,
      "learning_rate": 5.461953456489647e-07,
      "log_odds_chosen": 1.0594482421875,
      "log_odds_ratio": -0.4642578065395355,
      "logits/chosen": -0.964648425579071,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -0.549121081829071,
      "logps/rejected": -1.2130858898162842,
      "loss": 0.7532,
      "nll_loss": 0.7093750238418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02744751051068306,
      "rewards/margins": 0.03321380540728569,
      "rewards/rejected": -0.06065673753619194,
      "step": 8380
    },
    {
      "epoch": 0.6112264597676028,
      "grad_norm": 2.163651797251693,
      "learning_rate": 5.458697448178539e-07,
      "log_odds_chosen": 1.0573241710662842,
      "log_odds_ratio": -0.454833984375,
      "logits/chosen": -0.980664074420929,
      "logits/rejected": -0.888867199420929,
      "logps/chosen": -0.5386718511581421,
      "logps/rejected": -1.167382836341858,
      "loss": 0.7608,
      "nll_loss": 0.7162109613418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02695312537252903,
      "rewards/margins": 0.03145752102136612,
      "rewards/rejected": -0.05839843675494194,
      "step": 8390
    },
    {
      "epoch": 0.6119549775980767,
      "grad_norm": 2.142347272883295,
      "learning_rate": 5.45544725589981e-07,
      "log_odds_chosen": 1.0974609851837158,
      "log_odds_ratio": -0.4541015625,
      "logits/chosen": -0.970703125,
      "logits/rejected": -0.8714843988418579,
      "logps/chosen": -0.5458008050918579,
      "logps/rejected": -1.254296898841858,
      "loss": 0.7512,
      "nll_loss": 0.7314453125,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02727050706744194,
      "rewards/margins": 0.03542480617761612,
      "rewards/rejected": -0.06276855617761612,
      "step": 8400
    },
    {
      "epoch": 0.6126834954285506,
      "grad_norm": 1.9083984896491646,
      "learning_rate": 5.452202862359276e-07,
      "log_odds_chosen": 1.025720238685608,
      "log_odds_ratio": -0.465576171875,
      "logits/chosen": -1.0001952648162842,
      "logits/rejected": -0.8607422113418579,
      "logps/chosen": -0.54150390625,
      "logps/rejected": -1.1677734851837158,
      "loss": 0.7674,
      "nll_loss": 0.731640636920929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02708740159869194,
      "rewards/margins": 0.03130798414349556,
      "rewards/rejected": -0.05841064453125,
      "step": 8410
    },
    {
      "epoch": 0.6134120132590245,
      "grad_norm": 3.203300558223136,
      "learning_rate": 5.448964250334661e-07,
      "log_odds_chosen": 1.2454102039337158,
      "log_odds_ratio": -0.4305419921875,
      "logits/chosen": -1.029882788658142,
      "logits/rejected": -0.8998047113418579,
      "logps/chosen": -0.56884765625,
      "logps/rejected": -1.3640625476837158,
      "loss": 0.7581,
      "nll_loss": 0.7388671636581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0284423828125,
      "rewards/margins": 0.03970489650964737,
      "rewards/rejected": -0.06813964992761612,
      "step": 8420
    },
    {
      "epoch": 0.6141405310894984,
      "grad_norm": 2.2464303657997484,
      "learning_rate": 5.445731402675221e-07,
      "log_odds_chosen": 1.18408203125,
      "log_odds_ratio": -0.4466552734375,
      "logits/chosen": -0.9990234375,
      "logits/rejected": -0.894726574420929,
      "logps/chosen": -0.545703113079071,
      "logps/rejected": -1.2517578601837158,
      "loss": 0.7565,
      "nll_loss": 0.781542956829071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02728271484375,
      "rewards/margins": 0.0352630615234375,
      "rewards/rejected": -0.06251220405101776,
      "step": 8430
    },
    {
      "epoch": 0.6148690489199723,
      "grad_norm": 1.8498185536897653,
      "learning_rate": 5.442504302301351e-07,
      "log_odds_chosen": 0.8279663324356079,
      "log_odds_ratio": -0.49677735567092896,
      "logits/chosen": -1.002539038658142,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.5472656488418579,
      "logps/rejected": -1.0300781726837158,
      "loss": 0.7667,
      "nll_loss": 0.7142578363418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02736205980181694,
      "rewards/margins": 0.024129485711455345,
      "rewards/rejected": -0.05148925632238388,
      "step": 8440
    },
    {
      "epoch": 0.6155975667504462,
      "grad_norm": 2.0823851227329233,
      "learning_rate": 5.439282932204212e-07,
      "log_odds_chosen": 1.090917944908142,
      "log_odds_ratio": -0.463623046875,
      "logits/chosen": -0.9322265386581421,
      "logits/rejected": -0.860156238079071,
      "logps/chosen": -0.5643554925918579,
      "logps/rejected": -1.189453125,
      "loss": 0.7738,
      "nll_loss": 0.721386730670929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.028228759765625,
      "rewards/margins": 0.03122558631002903,
      "rewards/rejected": -0.05946044996380806,
      "step": 8450
    },
    {
      "epoch": 0.6163260845809201,
      "grad_norm": 2.097644358376746,
      "learning_rate": 5.436067275445352e-07,
      "log_odds_chosen": 0.9251953363418579,
      "log_odds_ratio": -0.505078136920929,
      "logits/chosen": -0.957812488079071,
      "logits/rejected": -0.8460937738418579,
      "logps/chosen": -0.597851574420929,
      "logps/rejected": -1.1867187023162842,
      "loss": 0.7703,
      "nll_loss": 0.748828113079071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0299072265625,
      "rewards/margins": 0.02942047081887722,
      "rewards/rejected": -0.05928344652056694,
      "step": 8460
    },
    {
      "epoch": 0.617054602411394,
      "grad_norm": 2.6574224116480503,
      "learning_rate": 5.432857315156336e-07,
      "log_odds_chosen": 1.134374976158142,
      "log_odds_ratio": -0.45673829317092896,
      "logits/chosen": -0.9384765625,
      "logits/rejected": -0.8330078125,
      "logps/chosen": -0.5582031011581421,
      "logps/rejected": -1.2322266101837158,
      "loss": 0.7887,
      "nll_loss": 0.796679675579071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02789917029440403,
      "rewards/margins": 0.03366393968462944,
      "rewards/rejected": -0.06156005710363388,
      "step": 8470
    },
    {
      "epoch": 0.6177831202418679,
      "grad_norm": 2.232543032954091,
      "learning_rate": 5.429653034538368e-07,
      "log_odds_chosen": 0.961474597454071,
      "log_odds_ratio": -0.5086914300918579,
      "logits/chosen": -0.973437488079071,
      "logits/rejected": -0.8773437738418579,
      "logps/chosen": -0.54541015625,
      "logps/rejected": -1.1535155773162842,
      "loss": 0.7708,
      "nll_loss": 0.774218738079071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02726440504193306,
      "rewards/margins": 0.03040466271340847,
      "rewards/rejected": -0.05762939527630806,
      "step": 8480
    },
    {
      "epoch": 0.6185116380723418,
      "grad_norm": 1.716357077535131,
      "learning_rate": 5.426454416861928e-07,
      "log_odds_chosen": 0.8999999761581421,
      "log_odds_ratio": -0.5096191167831421,
      "logits/chosen": -0.9740234613418579,
      "logits/rejected": -0.8558593988418579,
      "logps/chosen": -0.543652355670929,
      "logps/rejected": -1.108984351158142,
      "loss": 0.7569,
      "nll_loss": 0.7369140386581421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.027191162109375,
      "rewards/margins": 0.02823028527200222,
      "rewards/rejected": -0.05539550632238388,
      "step": 8490
    },
    {
      "epoch": 0.6192401559028157,
      "grad_norm": 2.0399742643591994,
      "learning_rate": 5.423261445466405e-07,
      "log_odds_chosen": 0.973437488079071,
      "log_odds_ratio": -0.4931640625,
      "logits/chosen": -0.9175781011581421,
      "logits/rejected": -0.802734375,
      "logps/chosen": -0.5645507574081421,
      "logps/rejected": -1.1417968273162842,
      "loss": 0.7694,
      "nll_loss": 0.69140625,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02824706956744194,
      "rewards/margins": 0.02884216234087944,
      "rewards/rejected": -0.05711669847369194,
      "step": 8500
    },
    {
      "epoch": 0.6199686737332897,
      "grad_norm": 1.7748999332148963,
      "learning_rate": 5.420074103759724e-07,
      "log_odds_chosen": 1.035058617591858,
      "log_odds_ratio": -0.454345703125,
      "logits/chosen": -0.936718761920929,
      "logits/rejected": -0.834179699420929,
      "logps/chosen": -0.5416015386581421,
      "logps/rejected": -1.170312523841858,
      "loss": 0.7594,
      "nll_loss": 0.766796886920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02708740159869194,
      "rewards/margins": 0.03141479566693306,
      "rewards/rejected": -0.05852050706744194,
      "step": 8510
    },
    {
      "epoch": 0.6206971915637636,
      "grad_norm": 1.9782774193890988,
      "learning_rate": 5.416892375217995e-07,
      "log_odds_chosen": 1.1040771007537842,
      "log_odds_ratio": -0.442138671875,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.879687488079071,
      "logps/chosen": -0.55810546875,
      "logps/rejected": -1.2224609851837158,
      "loss": 0.77,
      "nll_loss": 0.7772461175918579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02789917029440403,
      "rewards/margins": 0.03321533277630806,
      "rewards/rejected": -0.06112060695886612,
      "step": 8520
    },
    {
      "epoch": 0.6214257093942375,
      "grad_norm": 2.116663834465466,
      "learning_rate": 5.413716243385144e-07,
      "log_odds_chosen": 1.055419921875,
      "log_odds_ratio": -0.44755858182907104,
      "logits/chosen": -1.0119140148162842,
      "logits/rejected": -0.922656238079071,
      "logps/chosen": -0.527148425579071,
      "logps/rejected": -1.139257788658142,
      "loss": 0.7587,
      "nll_loss": 0.698535144329071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02634887769818306,
      "rewards/margins": 0.030609130859375,
      "rewards/rejected": -0.05701904371380806,
      "step": 8530
    },
    {
      "epoch": 0.6221542272247114,
      "grad_norm": 2.118546539530103,
      "learning_rate": 5.410545691872563e-07,
      "log_odds_chosen": 1.125878930091858,
      "log_odds_ratio": -0.43242186307907104,
      "logits/chosen": -0.984570324420929,
      "logits/rejected": -0.853320300579071,
      "logps/chosen": -0.5517578125,
      "logps/rejected": -1.224218726158142,
      "loss": 0.7608,
      "nll_loss": 0.778515636920929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02755126915872097,
      "rewards/margins": 0.03361053392291069,
      "rewards/rejected": -0.06114501878619194,
      "step": 8540
    },
    {
      "epoch": 0.6228827450551853,
      "grad_norm": 2.0726370703974575,
      "learning_rate": 5.407380704358752e-07,
      "log_odds_chosen": 0.856213390827179,
      "log_odds_ratio": -0.521435558795929,
      "logits/chosen": -0.953320324420929,
      "logits/rejected": -0.8177734613418579,
      "logps/chosen": -0.5732421875,
      "logps/rejected": -1.07421875,
      "loss": 0.7645,
      "nll_loss": 0.779296875,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02868041954934597,
      "rewards/margins": 0.02503814734518528,
      "rewards/rejected": -0.05368652194738388,
      "step": 8550
    },
    {
      "epoch": 0.6236112628856592,
      "grad_norm": 1.9734114877718014,
      "learning_rate": 5.404221264588962e-07,
      "log_odds_chosen": 1.1276366710662842,
      "log_odds_ratio": -0.43964844942092896,
      "logits/chosen": -0.9632812738418579,
      "logits/rejected": -0.855664074420929,
      "logps/chosen": -0.578808605670929,
      "logps/rejected": -1.265234351158142,
      "loss": 0.786,
      "nll_loss": 0.73193359375,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02896728552877903,
      "rewards/margins": 0.03430480882525444,
      "rewards/rejected": -0.06322631984949112,
      "step": 8560
    },
    {
      "epoch": 0.624339780716133,
      "grad_norm": 2.027444922459674,
      "learning_rate": 5.40106735637485e-07,
      "log_odds_chosen": 1.1744873523712158,
      "log_odds_ratio": -0.40869140625,
      "logits/chosen": -0.9912109375,
      "logits/rejected": -0.836718738079071,
      "logps/chosen": -0.538867175579071,
      "logps/rejected": -1.260156273841858,
      "loss": 0.7696,
      "nll_loss": 0.704882800579071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.026947021484375,
      "rewards/margins": 0.03606567531824112,
      "rewards/rejected": -0.06300048530101776,
      "step": 8570
    },
    {
      "epoch": 0.625068298546607,
      "grad_norm": 2.330558598644585,
      "learning_rate": 5.397918963594132e-07,
      "log_odds_chosen": 1.0890624523162842,
      "log_odds_ratio": -0.46308594942092896,
      "logits/chosen": -0.944531261920929,
      "logits/rejected": -0.857226550579071,
      "logps/chosen": -0.54736328125,
      "logps/rejected": -1.18359375,
      "loss": 0.7524,
      "nll_loss": 0.7334960699081421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02738647535443306,
      "rewards/margins": 0.03180847316980362,
      "rewards/rejected": -0.05913085862994194,
      "step": 8580
    },
    {
      "epoch": 0.6257968163770808,
      "grad_norm": 2.5976988963904466,
      "learning_rate": 5.394776070190225e-07,
      "log_odds_chosen": 0.907519519329071,
      "log_odds_ratio": -0.49970704317092896,
      "logits/chosen": -0.952929675579071,
      "logits/rejected": -0.847460925579071,
      "logps/chosen": -0.5601562261581421,
      "logps/rejected": -1.1154296398162842,
      "loss": 0.7518,
      "nll_loss": 0.7461913824081421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.02800903283059597,
      "rewards/margins": 0.02778015099465847,
      "rewards/rejected": -0.0557861328125,
      "step": 8590
    },
    {
      "epoch": 0.6265253342075547,
      "grad_norm": 2.010896679801412,
      "learning_rate": 5.391638660171921e-07,
      "log_odds_chosen": 1.016699194908142,
      "log_odds_ratio": -0.47441405057907104,
      "logits/chosen": -0.9662109613418579,
      "logits/rejected": -0.8642578125,
      "logps/chosen": -0.564746081829071,
      "logps/rejected": -1.190820336341858,
      "loss": 0.7643,
      "nll_loss": 0.7474609613418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02823486365377903,
      "rewards/margins": 0.03133239597082138,
      "rewards/rejected": -0.05954589694738388,
      "step": 8600
    },
    {
      "epoch": 0.6272538520380286,
      "grad_norm": 2.356773149119603,
      "learning_rate": 5.38850671761303e-07,
      "log_odds_chosen": 1.1481444835662842,
      "log_odds_ratio": -0.4488281309604645,
      "logits/chosen": -0.9468749761581421,
      "logits/rejected": -0.833984375,
      "logps/chosen": -0.546679675579071,
      "logps/rejected": -1.264062523841858,
      "loss": 0.7532,
      "nll_loss": 0.786914050579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02734375,
      "rewards/margins": 0.03588256984949112,
      "rewards/rejected": -0.06324462592601776,
      "step": 8610
    },
    {
      "epoch": 0.6279823698685025,
      "grad_norm": 1.8318074212197804,
      "learning_rate": 5.385380226652047e-07,
      "log_odds_chosen": 1.0317871570587158,
      "log_odds_ratio": -0.47050780057907104,
      "logits/chosen": -0.977343738079071,
      "logits/rejected": -0.851367175579071,
      "logps/chosen": -0.569531261920929,
      "logps/rejected": -1.206445336341858,
      "loss": 0.7922,
      "nll_loss": 0.778124988079071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02846069261431694,
      "rewards/margins": 0.03186798095703125,
      "rewards/rejected": -0.060333251953125,
      "step": 8620
    },
    {
      "epoch": 0.6287108876989764,
      "grad_norm": 1.9021940812485545,
      "learning_rate": 5.382259171491816e-07,
      "log_odds_chosen": 1.0017578601837158,
      "log_odds_ratio": -0.4608398377895355,
      "logits/chosen": -0.97265625,
      "logits/rejected": -0.8431640863418579,
      "logps/chosen": -0.564257800579071,
      "logps/rejected": -1.1962890625,
      "loss": 0.7763,
      "nll_loss": 0.7578125,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02821655198931694,
      "rewards/margins": 0.0316009521484375,
      "rewards/rejected": -0.05980224534869194,
      "step": 8630
    },
    {
      "epoch": 0.6294394055294503,
      "grad_norm": 2.1241083056576042,
      "learning_rate": 5.37914353639919e-07,
      "log_odds_chosen": 0.978515625,
      "log_odds_ratio": -0.46826171875,
      "logits/chosen": -0.984179675579071,
      "logits/rejected": -0.891406238079071,
      "logps/chosen": -0.5677734613418579,
      "logps/rejected": -1.130859375,
      "loss": 0.782,
      "nll_loss": 0.776171863079071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02839355543255806,
      "rewards/margins": 0.02810363844037056,
      "rewards/rejected": -0.05653076246380806,
      "step": 8640
    },
    {
      "epoch": 0.6301679233599242,
      "grad_norm": 3.438175299866991,
      "learning_rate": 5.376033305704704e-07,
      "log_odds_chosen": 1.0756347179412842,
      "log_odds_ratio": -0.4593261778354645,
      "logits/chosen": -0.9853515625,
      "logits/rejected": -0.859570324420929,
      "logps/chosen": -0.58935546875,
      "logps/rejected": -1.2365233898162842,
      "loss": 0.7787,
      "nll_loss": 0.7646484375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02947387658059597,
      "rewards/margins": 0.03236084058880806,
      "rewards/rejected": -0.06180419772863388,
      "step": 8650
    },
    {
      "epoch": 0.6308964411903981,
      "grad_norm": 2.372176371395733,
      "learning_rate": 5.372928463802237e-07,
      "log_odds_chosen": 1.2661621570587158,
      "log_odds_ratio": -0.42436522245407104,
      "logits/chosen": -0.9478515386581421,
      "logits/rejected": -0.845507800579071,
      "logps/chosen": -0.48408204317092896,
      "logps/rejected": -1.236328125,
      "loss": 0.7704,
      "nll_loss": 0.658886730670929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02421264722943306,
      "rewards/margins": 0.0375823974609375,
      "rewards/rejected": -0.061767578125,
      "step": 8660
    },
    {
      "epoch": 0.631624959020872,
      "grad_norm": 1.8382411946588264,
      "learning_rate": 5.369828995148688e-07,
      "log_odds_chosen": 1.064550757408142,
      "log_odds_ratio": -0.46162110567092896,
      "logits/chosen": -0.974804699420929,
      "logits/rejected": -0.8716796636581421,
      "logps/chosen": -0.5152343511581421,
      "logps/rejected": -1.138281226158142,
      "loss": 0.7623,
      "nll_loss": 0.699414074420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02573852613568306,
      "rewards/margins": 0.031158447265625,
      "rewards/rejected": -0.05693969875574112,
      "step": 8670
    },
    {
      "epoch": 0.6323534768513459,
      "grad_norm": 2.379165128694144,
      "learning_rate": 5.366734884263649e-07,
      "log_odds_chosen": 1.3742187023162842,
      "log_odds_ratio": -0.3993164002895355,
      "logits/chosen": -0.981249988079071,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -0.549023449420929,
      "logps/rejected": -1.4093749523162842,
      "loss": 0.7677,
      "nll_loss": 0.764843761920929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02744751051068306,
      "rewards/margins": 0.04306640475988388,
      "rewards/rejected": -0.07056884467601776,
      "step": 8680
    },
    {
      "epoch": 0.6330819946818198,
      "grad_norm": 1.8064331411380055,
      "learning_rate": 5.363646115729076e-07,
      "log_odds_chosen": 0.8921753168106079,
      "log_odds_ratio": -0.5348144769668579,
      "logits/chosen": -0.9140625,
      "logits/rejected": -0.8160156011581421,
      "logps/chosen": -0.5757812261581421,
      "logps/rejected": -1.1062500476837158,
      "loss": 0.783,
      "nll_loss": 0.7591797113418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02878418006002903,
      "rewards/margins": 0.026508713141083717,
      "rewards/rejected": -0.05532226711511612,
      "step": 8690
    },
    {
      "epoch": 0.6338105125122937,
      "grad_norm": 1.9306878401832686,
      "learning_rate": 5.360562674188974e-07,
      "log_odds_chosen": 1.08740234375,
      "log_odds_ratio": -0.4736328125,
      "logits/chosen": -0.9527343511581421,
      "logits/rejected": -0.8550781011581421,
      "logps/chosen": -0.515429675579071,
      "logps/rejected": -1.1550781726837158,
      "loss": 0.7581,
      "nll_loss": 0.7544921636581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02576904371380806,
      "rewards/margins": 0.03196411207318306,
      "rewards/rejected": -0.05775146558880806,
      "step": 8700
    },
    {
      "epoch": 0.6345390303427676,
      "grad_norm": 1.9974758626727527,
      "learning_rate": 5.35748454434907e-07,
      "log_odds_chosen": 1.1465332508087158,
      "log_odds_ratio": -0.427001953125,
      "logits/chosen": -0.9566406011581421,
      "logits/rejected": -0.833789050579071,
      "logps/chosen": -0.533886730670929,
      "logps/rejected": -1.236230492591858,
      "loss": 0.7665,
      "nll_loss": 0.723437488079071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02669067308306694,
      "rewards/margins": 0.035118866711854935,
      "rewards/rejected": -0.06177978590130806,
      "step": 8710
    },
    {
      "epoch": 0.6352675481732415,
      "grad_norm": 2.385343800217785,
      "learning_rate": 5.354411710976492e-07,
      "log_odds_chosen": 0.975048840045929,
      "log_odds_ratio": -0.4977050721645355,
      "logits/chosen": -0.945507824420929,
      "logits/rejected": -0.869335949420929,
      "logps/chosen": -0.5396484136581421,
      "logps/rejected": -1.1121094226837158,
      "loss": 0.7452,
      "nll_loss": 0.733203113079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02697143517434597,
      "rewards/margins": 0.02864685095846653,
      "rewards/rejected": -0.05567016452550888,
      "step": 8720
    },
    {
      "epoch": 0.6359960660037154,
      "grad_norm": 1.82842482923396,
      "learning_rate": 5.351344158899464e-07,
      "log_odds_chosen": 1.1828124523162842,
      "log_odds_ratio": -0.40961915254592896,
      "logits/chosen": -0.94140625,
      "logits/rejected": -0.8271484375,
      "logps/chosen": -0.5556640625,
      "logps/rejected": -1.25390625,
      "loss": 0.7474,
      "nll_loss": 0.7567383050918579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02780761756002903,
      "rewards/margins": 0.03484802320599556,
      "rewards/rejected": -0.06268310546875,
      "step": 8730
    },
    {
      "epoch": 0.6367245838341894,
      "grad_norm": 2.462902460388442,
      "learning_rate": 5.348281873006976e-07,
      "log_odds_chosen": 1.205957055091858,
      "log_odds_ratio": -0.4104980528354645,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.875781238079071,
      "logps/chosen": -0.553515613079071,
      "logps/rejected": -1.263085961341858,
      "loss": 0.7662,
      "nll_loss": 0.742382824420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02767333947122097,
      "rewards/margins": 0.03552703931927681,
      "rewards/rejected": -0.06314697116613388,
      "step": 8740
    },
    {
      "epoch": 0.6374531016646633,
      "grad_norm": 2.1352408543041967,
      "learning_rate": 5.345224838248488e-07,
      "log_odds_chosen": 1.090722680091858,
      "log_odds_ratio": -0.44453126192092896,
      "logits/chosen": -0.9853515625,
      "logits/rejected": -0.8994140625,
      "logps/chosen": -0.573925793170929,
      "logps/rejected": -1.210351586341858,
      "loss": 0.7812,
      "nll_loss": 0.7774413824081421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0286865234375,
      "rewards/margins": 0.03190002590417862,
      "rewards/rejected": -0.06053466722369194,
      "step": 8750
    },
    {
      "epoch": 0.6381816194951372,
      "grad_norm": 2.0243980770077257,
      "learning_rate": 5.342173039633604e-07,
      "log_odds_chosen": 1.112158179283142,
      "log_odds_ratio": -0.4695800840854645,
      "logits/chosen": -0.986328125,
      "logits/rejected": -0.8701171875,
      "logps/chosen": -0.5289062261581421,
      "logps/rejected": -1.1755859851837158,
      "loss": 0.7568,
      "nll_loss": 0.755859375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02645263634622097,
      "rewards/margins": 0.03232727199792862,
      "rewards/rejected": -0.05876464769244194,
      "step": 8760
    },
    {
      "epoch": 0.6389101373256111,
      "grad_norm": 4.394923362228619,
      "learning_rate": 5.33912646223177e-07,
      "log_odds_chosen": 1.1064941883087158,
      "log_odds_ratio": -0.4386230409145355,
      "logits/chosen": -0.949999988079071,
      "logits/rejected": -0.8462890386581421,
      "logps/chosen": -0.5645507574081421,
      "logps/rejected": -1.2058594226837158,
      "loss": 0.7516,
      "nll_loss": 0.703125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02822265587747097,
      "rewards/margins": 0.032167814671993256,
      "rewards/rejected": -0.06037597730755806,
      "step": 8770
    },
    {
      "epoch": 0.639638655156085,
      "grad_norm": 2.9397283542696235,
      "learning_rate": 5.336085091171972e-07,
      "log_odds_chosen": 1.108300805091858,
      "log_odds_ratio": -0.4910644590854645,
      "logits/chosen": -0.971875011920929,
      "logits/rejected": -0.846875011920929,
      "logps/chosen": -0.544140636920929,
      "logps/rejected": -1.2312500476837158,
      "loss": 0.7521,
      "nll_loss": 0.732421875,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02720336988568306,
      "rewards/margins": 0.0343780517578125,
      "rewards/rejected": -0.06154785305261612,
      "step": 8780
    },
    {
      "epoch": 0.6403671729865589,
      "grad_norm": 2.0421723873841704,
      "learning_rate": 5.333048911642421e-07,
      "log_odds_chosen": 1.3494141101837158,
      "log_odds_ratio": -0.39213865995407104,
      "logits/chosen": -0.9662109613418579,
      "logits/rejected": -0.8550781011581421,
      "logps/chosen": -0.521679699420929,
      "logps/rejected": -1.349218726158142,
      "loss": 0.7772,
      "nll_loss": 0.796875,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.02607421949505806,
      "rewards/margins": 0.04137573391199112,
      "rewards/rejected": -0.06744384765625,
      "step": 8790
    },
    {
      "epoch": 0.6410956908170328,
      "grad_norm": 2.814634013947336,
      "learning_rate": 5.33001790889026e-07,
      "log_odds_chosen": 1.291015625,
      "log_odds_ratio": -0.3915039002895355,
      "logits/chosen": -1.0212891101837158,
      "logits/rejected": -0.895703136920929,
      "logps/chosen": -0.5347656011581421,
      "logps/rejected": -1.3125,
      "loss": 0.7505,
      "nll_loss": 0.78125,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.02672729454934597,
      "rewards/margins": 0.0388336181640625,
      "rewards/rejected": -0.06563720852136612,
      "step": 8800
    },
    {
      "epoch": 0.6418242086475067,
      "grad_norm": 3.2251060231234407,
      "learning_rate": 5.326992068221256e-07,
      "log_odds_chosen": 1.1765625476837158,
      "log_odds_ratio": -0.4373535215854645,
      "logits/chosen": -0.935253918170929,
      "logits/rejected": -0.865234375,
      "logps/chosen": -0.5311523675918579,
      "logps/rejected": -1.265234351158142,
      "loss": 0.7554,
      "nll_loss": 0.7647460699081421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02655029296875,
      "rewards/margins": 0.03666992112994194,
      "rewards/rejected": -0.06319580227136612,
      "step": 8810
    },
    {
      "epoch": 0.6425527264779806,
      "grad_norm": 2.561942596769418,
      "learning_rate": 5.323971374999499e-07,
      "log_odds_chosen": 1.308349609375,
      "log_odds_ratio": -0.39204102754592896,
      "logits/chosen": -0.9937499761581421,
      "logits/rejected": -0.87109375,
      "logps/chosen": -0.536328136920929,
      "logps/rejected": -1.334375023841858,
      "loss": 0.7381,
      "nll_loss": 0.66845703125,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.02681274339556694,
      "rewards/margins": 0.03987579420208931,
      "rewards/rejected": -0.06668701022863388,
      "step": 8820
    },
    {
      "epoch": 0.6432812443084545,
      "grad_norm": 1.7862093257962777,
      "learning_rate": 5.320955814647117e-07,
      "log_odds_chosen": 1.201757788658142,
      "log_odds_ratio": -0.4720214903354645,
      "logits/chosen": -0.98046875,
      "logits/rejected": -0.8597656488418579,
      "logps/chosen": -0.5428711175918579,
      "logps/rejected": -1.3126952648162842,
      "loss": 0.7389,
      "nll_loss": 0.7369140386581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02715454064309597,
      "rewards/margins": 0.03850402683019638,
      "rewards/rejected": -0.06569824367761612,
      "step": 8830
    },
    {
      "epoch": 0.6440097621389284,
      "grad_norm": 3.1926262353171797,
      "learning_rate": 5.317945372643964e-07,
      "log_odds_chosen": 1.0334961414337158,
      "log_odds_ratio": -0.4715332090854645,
      "logits/chosen": -0.9791015386581421,
      "logits/rejected": -0.9033203125,
      "logps/chosen": -0.5868164300918579,
      "logps/rejected": -1.2257812023162842,
      "loss": 0.7614,
      "nll_loss": 0.7353515625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02933960035443306,
      "rewards/margins": 0.0319366455078125,
      "rewards/rejected": -0.06126708909869194,
      "step": 8840
    },
    {
      "epoch": 0.6447382799694023,
      "grad_norm": 2.6370698627189677,
      "learning_rate": 5.31494003452734e-07,
      "log_odds_chosen": 1.0158202648162842,
      "log_odds_ratio": -0.4651855528354645,
      "logits/chosen": -0.9429687261581421,
      "logits/rejected": -0.8443359136581421,
      "logps/chosen": -0.5406249761581421,
      "logps/rejected": -1.1652343273162842,
      "loss": 0.7651,
      "nll_loss": 0.7763671875,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02702636644244194,
      "rewards/margins": 0.03127746656537056,
      "rewards/rejected": -0.05830078199505806,
      "step": 8850
    },
    {
      "epoch": 0.6454667977998761,
      "grad_norm": 1.919231373364904,
      "learning_rate": 5.311939785891691e-07,
      "log_odds_chosen": 1.1593749523162842,
      "log_odds_ratio": -0.41801756620407104,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.835156261920929,
      "logps/chosen": -0.52978515625,
      "logps/rejected": -1.2228515148162842,
      "loss": 0.7619,
      "nll_loss": 0.748828113079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02647094801068306,
      "rewards/margins": 0.0346527099609375,
      "rewards/rejected": -0.06116943433880806,
      "step": 8860
    },
    {
      "epoch": 0.64619531563035,
      "grad_norm": 1.9613380807461163,
      "learning_rate": 5.308944612388322e-07,
      "log_odds_chosen": 1.14990234375,
      "log_odds_ratio": -0.43852537870407104,
      "logits/chosen": -0.9228515625,
      "logits/rejected": -0.827343761920929,
      "logps/chosen": -0.49687498807907104,
      "logps/rejected": -1.173828125,
      "loss": 0.7726,
      "nll_loss": 0.7017577886581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02486572228372097,
      "rewards/margins": 0.033879853785037994,
      "rewards/rejected": -0.05873413011431694,
      "step": 8870
    },
    {
      "epoch": 0.6469238334608239,
      "grad_norm": 2.095147825911441,
      "learning_rate": 5.305954499725112e-07,
      "log_odds_chosen": 1.068823218345642,
      "log_odds_ratio": -0.4500976502895355,
      "logits/chosen": -0.9808593988418579,
      "logits/rejected": -0.835742175579071,
      "logps/chosen": -0.528027355670929,
      "logps/rejected": -1.162695288658142,
      "loss": 0.7458,
      "nll_loss": 0.7060546875,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02640991285443306,
      "rewards/margins": 0.03170929104089737,
      "rewards/rejected": -0.05809326097369194,
      "step": 8880
    },
    {
      "epoch": 0.6476523512912978,
      "grad_norm": 1.8618935851259444,
      "learning_rate": 5.302969433666218e-07,
      "log_odds_chosen": 1.146484375,
      "log_odds_ratio": -0.44208985567092896,
      "logits/chosen": -0.974414050579071,
      "logits/rejected": -0.8509765863418579,
      "logps/chosen": -0.55322265625,
      "logps/rejected": -1.238671898841858,
      "loss": 0.7561,
      "nll_loss": 0.772265613079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.027679443359375,
      "rewards/margins": 0.03424682468175888,
      "rewards/rejected": -0.06192626804113388,
      "step": 8890
    },
    {
      "epoch": 0.6483808691217717,
      "grad_norm": 2.0661507899765383,
      "learning_rate": 5.299989400031801e-07,
      "log_odds_chosen": 1.1955077648162842,
      "log_odds_ratio": -0.4256347715854645,
      "logits/chosen": -0.9183593988418579,
      "logits/rejected": -0.806640625,
      "logps/chosen": -0.5391601324081421,
      "logps/rejected": -1.232812523841858,
      "loss": 0.7772,
      "nll_loss": 0.777636706829071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02696533128619194,
      "rewards/margins": 0.03465728834271431,
      "rewards/rejected": -0.06165771558880806,
      "step": 8900
    },
    {
      "epoch": 0.6491093869522456,
      "grad_norm": 1.8349412987859663,
      "learning_rate": 5.297014384697736e-07,
      "log_odds_chosen": 1.1486327648162842,
      "log_odds_ratio": -0.438720703125,
      "logits/chosen": -0.9205077886581421,
      "logits/rejected": -0.8267577886581421,
      "logps/chosen": -0.5575195550918579,
      "logps/rejected": -1.248632788658142,
      "loss": 0.7483,
      "nll_loss": 0.717968761920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02783813513815403,
      "rewards/margins": 0.03457031399011612,
      "rewards/rejected": -0.06241454929113388,
      "step": 8910
    },
    {
      "epoch": 0.6498379047827195,
      "grad_norm": 2.3014944632224643,
      "learning_rate": 5.294044373595335e-07,
      "log_odds_chosen": 1.1588866710662842,
      "log_odds_ratio": -0.4291015565395355,
      "logits/chosen": -0.9564453363418579,
      "logits/rejected": -0.873046875,
      "logps/chosen": -0.512011706829071,
      "logps/rejected": -1.2087891101837158,
      "loss": 0.76,
      "nll_loss": 0.702343761920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02561035193502903,
      "rewards/margins": 0.03493652492761612,
      "rewards/rejected": -0.06055908277630806,
      "step": 8920
    },
    {
      "epoch": 0.6505664226131934,
      "grad_norm": 1.7363807776923812,
      "learning_rate": 5.291079352711064e-07,
      "log_odds_chosen": 1.097387671470642,
      "log_odds_ratio": -0.46064454317092896,
      "logits/chosen": -0.992968738079071,
      "logits/rejected": -0.8984375,
      "logps/chosen": -0.551464855670929,
      "logps/rejected": -1.178320288658142,
      "loss": 0.7419,
      "nll_loss": 0.70068359375,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02756652794778347,
      "rewards/margins": 0.03130645677447319,
      "rewards/rejected": -0.05894775316119194,
      "step": 8930
    },
    {
      "epoch": 0.6512949404436673,
      "grad_norm": 1.9578357299073879,
      "learning_rate": 5.28811930808627e-07,
      "log_odds_chosen": 0.995898425579071,
      "log_odds_ratio": -0.47382813692092896,
      "logits/chosen": -0.943554699420929,
      "logits/rejected": -0.833203136920929,
      "logps/chosen": -0.587207019329071,
      "logps/rejected": -1.173437476158142,
      "loss": 0.757,
      "nll_loss": 0.7896484136581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02933349646627903,
      "rewards/margins": 0.02926330640912056,
      "rewards/rejected": -0.05861816555261612,
      "step": 8940
    },
    {
      "epoch": 0.6520234582741412,
      "grad_norm": 2.0251772167056155,
      "learning_rate": 5.2851642258169e-07,
      "log_odds_chosen": 1.072167992591858,
      "log_odds_ratio": -0.4715820252895355,
      "logits/chosen": -1.0001952648162842,
      "logits/rejected": -0.8662109375,
      "logps/chosen": -0.54248046875,
      "logps/rejected": -1.19775390625,
      "loss": 0.7564,
      "nll_loss": 0.73828125,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.027130126953125,
      "rewards/margins": 0.03279266506433487,
      "rewards/rejected": -0.05989379808306694,
      "step": 8950
    },
    {
      "epoch": 0.6527519761046151,
      "grad_norm": 3.032676177931267,
      "learning_rate": 5.282214092053229e-07,
      "log_odds_chosen": 1.1702880859375,
      "log_odds_ratio": -0.4287109375,
      "logits/chosen": -0.9306640625,
      "logits/rejected": -0.8775390386581421,
      "logps/chosen": -0.5316406488418579,
      "logps/rejected": -1.2322266101837158,
      "loss": 0.7335,
      "nll_loss": 0.7046874761581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02657470665872097,
      "rewards/margins": 0.03499298170208931,
      "rewards/rejected": -0.06154785305261612,
      "step": 8960
    },
    {
      "epoch": 0.6534804939350891,
      "grad_norm": 1.9982620249385803,
      "learning_rate": 5.279268892999588e-07,
      "log_odds_chosen": 1.0463378429412842,
      "log_odds_ratio": -0.46538084745407104,
      "logits/chosen": -0.988085925579071,
      "logits/rejected": -0.87109375,
      "logps/chosen": -0.533984363079071,
      "logps/rejected": -1.145898461341858,
      "loss": 0.7348,
      "nll_loss": 0.7054687738418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02667846716940403,
      "rewards/margins": 0.03059997595846653,
      "rewards/rejected": -0.05729980394244194,
      "step": 8970
    },
    {
      "epoch": 0.654209011765563,
      "grad_norm": 3.3147785744425753,
      "learning_rate": 5.276328614914092e-07,
      "log_odds_chosen": 1.13525390625,
      "log_odds_ratio": -0.4771484434604645,
      "logits/chosen": -0.9833984375,
      "logits/rejected": -0.85546875,
      "logps/chosen": -0.56103515625,
      "logps/rejected": -1.2976562976837158,
      "loss": 0.7674,
      "nll_loss": 0.7178710699081421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02803955040872097,
      "rewards/margins": 0.03688659518957138,
      "rewards/rejected": -0.06495361030101776,
      "step": 8980
    },
    {
      "epoch": 0.6549375295960369,
      "grad_norm": 1.7343920873027978,
      "learning_rate": 5.273393244108369e-07,
      "log_odds_chosen": 1.17236328125,
      "log_odds_ratio": -0.45463865995407104,
      "logits/chosen": -0.968554675579071,
      "logits/rejected": -0.8832031488418579,
      "logps/chosen": -0.5176757574081421,
      "logps/rejected": -1.161523461341858,
      "loss": 0.7639,
      "nll_loss": 0.733593761920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02589721605181694,
      "rewards/margins": 0.03220672532916069,
      "rewards/rejected": -0.05809326097369194,
      "step": 8990
    },
    {
      "epoch": 0.6556660474265108,
      "grad_norm": 2.00803049848092,
      "learning_rate": 5.270462766947299e-07,
      "log_odds_chosen": 0.9716796875,
      "log_odds_ratio": -0.513671875,
      "logits/chosen": -0.9642578363418579,
      "logits/rejected": -0.8724609613418579,
      "logps/chosen": -0.618359386920929,
      "logps/rejected": -1.234375,
      "loss": 0.7702,
      "nll_loss": 0.7974609136581421,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.03090820275247097,
      "rewards/margins": 0.03078613243997097,
      "rewards/rejected": -0.06175537034869194,
      "step": 9000
    },
    {
      "epoch": 0.6563945652569847,
      "grad_norm": 1.9024068433332129,
      "learning_rate": 5.26753716984874e-07,
      "log_odds_chosen": 1.149023413658142,
      "log_odds_ratio": -0.43095701932907104,
      "logits/chosen": -0.9857422113418579,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.590136706829071,
      "logps/rejected": -1.36328125,
      "loss": 0.7809,
      "nll_loss": 0.736621081829071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02949829027056694,
      "rewards/margins": 0.03864746168255806,
      "rewards/rejected": -0.06820068508386612,
      "step": 9010
    },
    {
      "epoch": 0.6571230830874586,
      "grad_norm": 2.378279505213707,
      "learning_rate": 5.264616439283267e-07,
      "log_odds_chosen": 0.964599609375,
      "log_odds_ratio": -0.5157226324081421,
      "logits/chosen": -0.9759765863418579,
      "logits/rejected": -0.8734375238418579,
      "logps/chosen": -0.581250011920929,
      "logps/rejected": -1.176171898841858,
      "loss": 0.7587,
      "nll_loss": 0.741015613079071,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02904663048684597,
      "rewards/margins": 0.02968749962747097,
      "rewards/rejected": -0.05875243991613388,
      "step": 9020
    },
    {
      "epoch": 0.6578516009179325,
      "grad_norm": 2.8439632203372196,
      "learning_rate": 5.26170056177391e-07,
      "log_odds_chosen": 1.1134765148162842,
      "log_odds_ratio": -0.4757324159145355,
      "logits/chosen": -0.9853515625,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -0.5630859136581421,
      "logps/rejected": -1.229882836341858,
      "loss": 0.764,
      "nll_loss": 0.7212890386581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.028167724609375,
      "rewards/margins": 0.0333251953125,
      "rewards/rejected": -0.06147461012005806,
      "step": 9030
    },
    {
      "epoch": 0.6585801187484064,
      "grad_norm": 2.068630215816338,
      "learning_rate": 5.258789523895891e-07,
      "log_odds_chosen": 1.0041015148162842,
      "log_odds_ratio": -0.46367186307907104,
      "logits/chosen": -0.9859374761581421,
      "logits/rejected": -0.849804699420929,
      "logps/chosen": -0.533398449420929,
      "logps/rejected": -1.130468726158142,
      "loss": 0.76,
      "nll_loss": 0.6922851800918579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02666625939309597,
      "rewards/margins": 0.02988586388528347,
      "rewards/rejected": -0.05654297024011612,
      "step": 9040
    },
    {
      "epoch": 0.6593086365788803,
      "grad_norm": 2.323400595243177,
      "learning_rate": 5.255883312276367e-07,
      "log_odds_chosen": 1.1155273914337158,
      "log_odds_ratio": -0.42646485567092896,
      "logits/chosen": -0.9974609613418579,
      "logits/rejected": -0.8773437738418579,
      "logps/chosen": -0.51953125,
      "logps/rejected": -1.209570288658142,
      "loss": 0.7602,
      "nll_loss": 0.667675793170929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02597656287252903,
      "rewards/margins": 0.03450927883386612,
      "rewards/rejected": -0.06051025539636612,
      "step": 9050
    },
    {
      "epoch": 0.6600371544093542,
      "grad_norm": 1.973422005252143,
      "learning_rate": 5.25298191359417e-07,
      "log_odds_chosen": 1.116601586341858,
      "log_odds_ratio": -0.4427734315395355,
      "logits/chosen": -1.008203148841858,
      "logits/rejected": -0.8990234136581421,
      "logps/chosen": -0.560742199420929,
      "logps/rejected": -1.2390625476837158,
      "loss": 0.7521,
      "nll_loss": 0.709179699420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02802124060690403,
      "rewards/margins": 0.03394164890050888,
      "rewards/rejected": -0.06201171875,
      "step": 9060
    },
    {
      "epoch": 0.6607656722398281,
      "grad_norm": 2.199384349862651,
      "learning_rate": 5.250085314579549e-07,
      "log_odds_chosen": 1.1423828601837158,
      "log_odds_ratio": -0.44414061307907104,
      "logits/chosen": -1.005859375,
      "logits/rejected": -0.8763672113418579,
      "logps/chosen": -0.503613293170929,
      "logps/rejected": -1.119726538658142,
      "loss": 0.7601,
      "nll_loss": 0.69091796875,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02516479417681694,
      "rewards/margins": 0.03078918531537056,
      "rewards/rejected": -0.05594482272863388,
      "step": 9070
    },
    {
      "epoch": 0.661494190070302,
      "grad_norm": 1.9601033022502643,
      "learning_rate": 5.24719350201392e-07,
      "log_odds_chosen": 1.047265648841858,
      "log_odds_ratio": -0.4947265684604645,
      "logits/chosen": -0.958984375,
      "logits/rejected": -0.880859375,
      "logps/chosen": -0.553906261920929,
      "logps/rejected": -1.170312523841858,
      "loss": 0.7657,
      "nll_loss": 0.7705078125,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02771606482565403,
      "rewards/margins": 0.03082733228802681,
      "rewards/rejected": -0.05849609524011612,
      "step": 9080
    },
    {
      "epoch": 0.6622227079007759,
      "grad_norm": 2.531721629751283,
      "learning_rate": 5.244306462729606e-07,
      "log_odds_chosen": 1.043798804283142,
      "log_odds_ratio": -0.46342772245407104,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.8695312738418579,
      "logps/chosen": -0.566210925579071,
      "logps/rejected": -1.2041015625,
      "loss": 0.791,
      "nll_loss": 0.80859375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0283203125,
      "rewards/margins": 0.03190460056066513,
      "rewards/rejected": -0.06019287183880806,
      "step": 9090
    },
    {
      "epoch": 0.6629512257312498,
      "grad_norm": 2.094869793791772,
      "learning_rate": 5.241424183609592e-07,
      "log_odds_chosen": 1.20947265625,
      "log_odds_ratio": -0.44677734375,
      "logits/chosen": -0.9476562738418579,
      "logits/rejected": -0.809374988079071,
      "logps/chosen": -0.5404297113418579,
      "logps/rejected": -1.292578101158142,
      "loss": 0.7583,
      "nll_loss": 0.741406261920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02701416052877903,
      "rewards/margins": 0.03768615797162056,
      "rewards/rejected": -0.06468506157398224,
      "step": 9100
    },
    {
      "epoch": 0.6636797435617237,
      "grad_norm": 2.0063411554110258,
      "learning_rate": 5.23854665158727e-07,
      "log_odds_chosen": 1.035742163658142,
      "log_odds_ratio": -0.45991212129592896,
      "logits/chosen": -0.9955078363418579,
      "logits/rejected": -0.905468761920929,
      "logps/chosen": -0.547070324420929,
      "logps/rejected": -1.1749999523162842,
      "loss": 0.7502,
      "nll_loss": 0.759570300579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02734375,
      "rewards/margins": 0.03142852708697319,
      "rewards/rejected": -0.05876464769244194,
      "step": 9110
    },
    {
      "epoch": 0.6644082613921976,
      "grad_norm": 1.7393259218450932,
      "learning_rate": 5.235673853646194e-07,
      "log_odds_chosen": 1.0868651866912842,
      "log_odds_ratio": -0.4623046815395355,
      "logits/chosen": -1.0048828125,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.537890613079071,
      "logps/rejected": -1.1494140625,
      "loss": 0.7568,
      "nll_loss": 0.7476562261581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.026885986328125,
      "rewards/margins": 0.03061523474752903,
      "rewards/rejected": -0.0574951171875,
      "step": 9120
    },
    {
      "epoch": 0.6651367792226714,
      "grad_norm": 2.218803696697085,
      "learning_rate": 5.232805776819827e-07,
      "log_odds_chosen": 1.007470726966858,
      "log_odds_ratio": -0.4723144471645355,
      "logits/chosen": -0.9652343988418579,
      "logits/rejected": -0.8609374761581421,
      "logps/chosen": -0.5386718511581421,
      "logps/rejected": -1.1513671875,
      "loss": 0.7425,
      "nll_loss": 0.7328125238418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02694091759622097,
      "rewards/margins": 0.030609130859375,
      "rewards/rejected": -0.05758056789636612,
      "step": 9130
    },
    {
      "epoch": 0.6658652970531453,
      "grad_norm": 2.119422873603559,
      "learning_rate": 5.2299424081913e-07,
      "log_odds_chosen": 1.198632836341858,
      "log_odds_ratio": -0.4119628965854645,
      "logits/chosen": -0.9876953363418579,
      "logits/rejected": -0.8853515386581421,
      "logps/chosen": -0.53662109375,
      "logps/rejected": -1.224218726158142,
      "loss": 0.7323,
      "nll_loss": 0.71044921875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02681884728372097,
      "rewards/margins": 0.034371186047792435,
      "rewards/rejected": -0.06124267727136612,
      "step": 9140
    },
    {
      "epoch": 0.6665938148836192,
      "grad_norm": 1.9700726644264412,
      "learning_rate": 5.227083734893167e-07,
      "log_odds_chosen": 1.0096435546875,
      "log_odds_ratio": -0.502148449420929,
      "logits/chosen": -1.0056641101837158,
      "logits/rejected": -0.8921874761581421,
      "logps/chosen": -0.54345703125,
      "logps/rejected": -1.1906249523162842,
      "loss": 0.7502,
      "nll_loss": 0.703417956829071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02716064453125,
      "rewards/margins": 0.03229980543255806,
      "rewards/rejected": -0.0594482421875,
      "step": 9150
    },
    {
      "epoch": 0.6673223327140931,
      "grad_norm": 2.5986941707965494,
      "learning_rate": 5.224229744107161e-07,
      "log_odds_chosen": 1.1628906726837158,
      "log_odds_ratio": -0.42329102754592896,
      "logits/chosen": -0.957226574420929,
      "logits/rejected": -0.8365234136581421,
      "logps/chosen": -0.5362304449081421,
      "logps/rejected": -1.197656273841858,
      "loss": 0.7479,
      "nll_loss": 0.763476550579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02678222581744194,
      "rewards/margins": 0.03307800367474556,
      "rewards/rejected": -0.05989990383386612,
      "step": 9160
    },
    {
      "epoch": 0.668050850544567,
      "grad_norm": 1.876492595702677,
      "learning_rate": 5.221380423063954e-07,
      "log_odds_chosen": 1.309179663658142,
      "log_odds_ratio": -0.3746581971645355,
      "logits/chosen": -1.0320312976837158,
      "logits/rejected": -0.8921874761581421,
      "logps/chosen": -0.53759765625,
      "logps/rejected": -1.29296875,
      "loss": 0.7465,
      "nll_loss": 0.716601550579071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.02685546875,
      "rewards/margins": 0.03775329515337944,
      "rewards/rejected": -0.06456299126148224,
      "step": 9170
    },
    {
      "epoch": 0.6687793683750409,
      "grad_norm": 2.0253597969637496,
      "learning_rate": 5.218535759042913e-07,
      "log_odds_chosen": 0.990039050579071,
      "log_odds_ratio": -0.47880858182907104,
      "logits/chosen": -0.999218761920929,
      "logits/rejected": -0.8955078125,
      "logps/chosen": -0.556445300579071,
      "logps/rejected": -1.12890625,
      "loss": 0.7677,
      "nll_loss": 0.755078136920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02782592736184597,
      "rewards/margins": 0.028620148077607155,
      "rewards/rejected": -0.05642089992761612,
      "step": 9180
    },
    {
      "epoch": 0.6695078862055149,
      "grad_norm": 2.699981154919819,
      "learning_rate": 5.215695739371868e-07,
      "log_odds_chosen": 1.311914086341858,
      "log_odds_ratio": -0.4130615293979645,
      "logits/chosen": -0.991015613079071,
      "logits/rejected": -0.869921863079071,
      "logps/chosen": -0.5322265625,
      "logps/rejected": -1.3308594226837158,
      "loss": 0.7461,
      "nll_loss": 0.698046863079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02661743201315403,
      "rewards/margins": 0.0398712158203125,
      "rewards/rejected": -0.06645508110523224,
      "step": 9190
    },
    {
      "epoch": 0.6702364040359888,
      "grad_norm": 1.918714949539686,
      "learning_rate": 5.21286035142687e-07,
      "log_odds_chosen": 1.111474633216858,
      "log_odds_ratio": -0.441162109375,
      "logits/chosen": -0.962695300579071,
      "logits/rejected": -0.869140625,
      "logps/chosen": -0.4869140684604645,
      "logps/rejected": -1.1062500476837158,
      "loss": 0.7496,
      "nll_loss": 0.699999988079071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02433471754193306,
      "rewards/margins": 0.03095703199505806,
      "rewards/rejected": -0.05531005933880806,
      "step": 9200
    },
    {
      "epoch": 0.6709649218664627,
      "grad_norm": 12.934487791928326,
      "learning_rate": 5.210029582631956e-07,
      "log_odds_chosen": 1.0731933116912842,
      "log_odds_ratio": -0.4793945252895355,
      "logits/chosen": -0.989453136920929,
      "logits/rejected": -0.880664050579071,
      "logps/chosen": -0.5550781488418579,
      "logps/rejected": -1.204492211341858,
      "loss": 0.7533,
      "nll_loss": 0.7206054925918579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.027740478515625,
      "rewards/margins": 0.03250007703900337,
      "rewards/rejected": -0.06024169921875,
      "step": 9210
    },
    {
      "epoch": 0.6716934396969366,
      "grad_norm": 5.519860602757679,
      "learning_rate": 5.207203420458918e-07,
      "log_odds_chosen": 1.1980469226837158,
      "log_odds_ratio": -0.4359374940395355,
      "logits/chosen": -1.024804711341858,
      "logits/rejected": -0.900195300579071,
      "logps/chosen": -0.5791015625,
      "logps/rejected": -1.319726586341858,
      "loss": 0.7351,
      "nll_loss": 0.7210937738418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.028961181640625,
      "rewards/margins": 0.036991119384765625,
      "rewards/rejected": -0.06597900390625,
      "step": 9220
    },
    {
      "epoch": 0.6724219575274105,
      "grad_norm": 1.9585383400907388,
      "learning_rate": 5.204381852427064e-07,
      "log_odds_chosen": 1.06005859375,
      "log_odds_ratio": -0.4610351622104645,
      "logits/chosen": -1.0080077648162842,
      "logits/rejected": -0.881640613079071,
      "logps/chosen": -0.5235351324081421,
      "logps/rejected": -1.156640648841858,
      "loss": 0.7637,
      "nll_loss": 0.707324206829071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02615966834127903,
      "rewards/margins": 0.031644441187381744,
      "rewards/rejected": -0.057769775390625,
      "step": 9230
    },
    {
      "epoch": 0.6731504753578844,
      "grad_norm": 2.2810985309088525,
      "learning_rate": 5.201564866102994e-07,
      "log_odds_chosen": 1.0164062976837158,
      "log_odds_ratio": -0.4419921934604645,
      "logits/chosen": -0.985546886920929,
      "logits/rejected": -0.907421886920929,
      "logps/chosen": -0.519238293170929,
      "logps/rejected": -1.109375,
      "loss": 0.7367,
      "nll_loss": 0.6788085699081421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02595825120806694,
      "rewards/margins": 0.029505157843232155,
      "rewards/rejected": -0.05549316480755806,
      "step": 9240
    },
    {
      "epoch": 0.6738789931883583,
      "grad_norm": 2.2977373908853274,
      "learning_rate": 5.198752449100365e-07,
      "log_odds_chosen": 1.198828101158142,
      "log_odds_ratio": -0.41254884004592896,
      "logits/chosen": -0.9619140625,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -0.5238281488418579,
      "logps/rejected": -1.157617211341858,
      "loss": 0.7591,
      "nll_loss": 0.762890636920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02619628980755806,
      "rewards/margins": 0.031658172607421875,
      "rewards/rejected": -0.05780029296875,
      "step": 9250
    },
    {
      "epoch": 0.6746075110188322,
      "grad_norm": 2.1862940673467333,
      "learning_rate": 5.195944589079658e-07,
      "log_odds_chosen": 1.220703125,
      "log_odds_ratio": -0.42460936307907104,
      "logits/chosen": -0.932812511920929,
      "logits/rejected": -0.8482421636581421,
      "logps/chosen": -0.519726574420929,
      "logps/rejected": -1.2179687023162842,
      "loss": 0.7316,
      "nll_loss": 0.7420898675918579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02597656287252903,
      "rewards/margins": 0.03490600734949112,
      "rewards/rejected": -0.06083984300494194,
      "step": 9260
    },
    {
      "epoch": 0.6753360288493061,
      "grad_norm": 1.9254333248687256,
      "learning_rate": 5.193141273747966e-07,
      "log_odds_chosen": 1.0504882335662842,
      "log_odds_ratio": -0.462646484375,
      "logits/chosen": -0.9794921875,
      "logits/rejected": -0.845507800579071,
      "logps/chosen": -0.5372070074081421,
      "logps/rejected": -1.1912109851837158,
      "loss": 0.7591,
      "nll_loss": 0.7193359136581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.026885986328125,
      "rewards/margins": 0.03269348293542862,
      "rewards/rejected": -0.05954589694738388,
      "step": 9270
    },
    {
      "epoch": 0.67606454667978,
      "grad_norm": 2.1073011120812946,
      "learning_rate": 5.190342490858748e-07,
      "log_odds_chosen": 1.0008056163787842,
      "log_odds_ratio": -0.4767089784145355,
      "logits/chosen": -0.9720703363418579,
      "logits/rejected": -0.8603515625,
      "logps/chosen": -0.547558605670929,
      "logps/rejected": -1.142968773841858,
      "loss": 0.7559,
      "nll_loss": 0.765820324420929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02737121656537056,
      "rewards/margins": 0.02982482872903347,
      "rewards/rejected": -0.05712280422449112,
      "step": 9280
    },
    {
      "epoch": 0.6767930645102539,
      "grad_norm": 2.2365940848712245,
      "learning_rate": 5.18754822821162e-07,
      "log_odds_chosen": 1.258398413658142,
      "log_odds_ratio": -0.43134766817092896,
      "logits/chosen": -0.971875011920929,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.49345701932907104,
      "logps/rejected": -1.2000000476837158,
      "loss": 0.761,
      "nll_loss": 0.684374988079071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02465209923684597,
      "rewards/margins": 0.03534545749425888,
      "rewards/rejected": -0.06000976637005806,
      "step": 9290
    },
    {
      "epoch": 0.6775215823407278,
      "grad_norm": 2.1973769581581397,
      "learning_rate": 5.184758473652127e-07,
      "log_odds_chosen": 0.9650024175643921,
      "log_odds_ratio": -0.4922851622104645,
      "logits/chosen": -0.9916015863418579,
      "logits/rejected": -0.9056640863418579,
      "logps/chosen": -0.5528320074081421,
      "logps/rejected": -1.130273461341858,
      "loss": 0.7718,
      "nll_loss": 0.763867199420929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02763671800494194,
      "rewards/margins": 0.028830338269472122,
      "rewards/rejected": -0.05648193508386612,
      "step": 9300
    },
    {
      "epoch": 0.6782501001712017,
      "grad_norm": 2.0883810206251354,
      "learning_rate": 5.181973215071512e-07,
      "log_odds_chosen": 0.847644031047821,
      "log_odds_ratio": -0.525830090045929,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.9212890863418579,
      "logps/chosen": -0.5428711175918579,
      "logps/rejected": -1.0476562976837158,
      "loss": 0.7501,
      "nll_loss": 0.7015625238418579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02715454064309597,
      "rewards/margins": 0.025188064202666283,
      "rewards/rejected": -0.05233154445886612,
      "step": 9310
    },
    {
      "epoch": 0.6789786180016756,
      "grad_norm": 2.3261645561803683,
      "learning_rate": 5.179192440406511e-07,
      "log_odds_chosen": 0.8928467035293579,
      "log_odds_ratio": -0.520214855670929,
      "logits/chosen": -0.9375,
      "logits/rejected": -0.8368164300918579,
      "logps/chosen": -0.5391601324081421,
      "logps/rejected": -1.053125023841858,
      "loss": 0.7634,
      "nll_loss": 0.773242175579071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02698364295065403,
      "rewards/margins": 0.025696564465761185,
      "rewards/rejected": -0.05268554762005806,
      "step": 9320
    },
    {
      "epoch": 0.6797071358321495,
      "grad_norm": 2.000886176450421,
      "learning_rate": 5.176416137639121e-07,
      "log_odds_chosen": 1.1750366687774658,
      "log_odds_ratio": -0.45844727754592896,
      "logits/chosen": -0.913867175579071,
      "logits/rejected": -0.869921863079071,
      "logps/chosen": -0.5438476800918579,
      "logps/rejected": -1.2078125476837158,
      "loss": 0.7677,
      "nll_loss": 0.775390625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02719726599752903,
      "rewards/margins": 0.033234406262636185,
      "rewards/rejected": -0.06043701246380806,
      "step": 9330
    },
    {
      "epoch": 0.6804356536626234,
      "grad_norm": 1.8334580965288099,
      "learning_rate": 5.173644294796389e-07,
      "log_odds_chosen": 1.095361351966858,
      "log_odds_ratio": -0.4383789002895355,
      "logits/chosen": -0.9736328125,
      "logits/rejected": -0.863476574420929,
      "logps/chosen": -0.538281261920929,
      "logps/rejected": -1.195703148841858,
      "loss": 0.7409,
      "nll_loss": 0.724414050579071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02691650390625,
      "rewards/margins": 0.03288574144244194,
      "rewards/rejected": -0.059814453125,
      "step": 9340
    },
    {
      "epoch": 0.6811641714930973,
      "grad_norm": 2.258009003881909,
      "learning_rate": 5.170876899950191e-07,
      "log_odds_chosen": 1.394189476966858,
      "log_odds_ratio": -0.36835938692092896,
      "logits/chosen": -0.9750000238418579,
      "logits/rejected": -0.8408203125,
      "logps/chosen": -0.536914050579071,
      "logps/rejected": -1.379492163658142,
      "loss": 0.7425,
      "nll_loss": 0.745410144329071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.02684326097369194,
      "rewards/margins": 0.0420989990234375,
      "rewards/rejected": -0.06896362453699112,
      "step": 9350
    },
    {
      "epoch": 0.6818926893235712,
      "grad_norm": 2.1664666393988767,
      "learning_rate": 5.16811394121702e-07,
      "log_odds_chosen": 1.1015625,
      "log_odds_ratio": -0.45893555879592896,
      "logits/chosen": -0.987500011920929,
      "logits/rejected": -0.883007824420929,
      "logps/chosen": -0.538281261920929,
      "logps/rejected": -1.1884765625,
      "loss": 0.7434,
      "nll_loss": 0.720507800579071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02691040001809597,
      "rewards/margins": 0.03252868726849556,
      "rewards/rejected": -0.05942382663488388,
      "step": 9360
    },
    {
      "epoch": 0.6826212071540451,
      "grad_norm": 2.2025160458524398,
      "learning_rate": 5.165355406757765e-07,
      "log_odds_chosen": 0.9463866949081421,
      "log_odds_ratio": -0.50537109375,
      "logits/chosen": -0.972851574420929,
      "logits/rejected": -0.889843761920929,
      "logps/chosen": -0.588085949420929,
      "logps/rejected": -1.1789062023162842,
      "loss": 0.7576,
      "nll_loss": 0.717968761920929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02938232384622097,
      "rewards/margins": 0.02949981763958931,
      "rewards/rejected": -0.05894775316119194,
      "step": 9370
    },
    {
      "epoch": 0.683349724984519,
      "grad_norm": 2.290008499431981,
      "learning_rate": 5.162601284777506e-07,
      "log_odds_chosen": 1.1496093273162842,
      "log_odds_ratio": -0.4381347596645355,
      "logits/chosen": -1.021484375,
      "logits/rejected": -0.8726562261581421,
      "logps/chosen": -0.5967773199081421,
      "logps/rejected": -1.324609398841858,
      "loss": 0.7509,
      "nll_loss": 0.7127929925918579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.029876708984375,
      "rewards/margins": 0.03643493726849556,
      "rewards/rejected": -0.06624756008386612,
      "step": 9380
    },
    {
      "epoch": 0.6840782428149929,
      "grad_norm": 2.406327541304091,
      "learning_rate": 5.159851563525301e-07,
      "log_odds_chosen": 1.1531250476837158,
      "log_odds_ratio": -0.42241209745407104,
      "logits/chosen": -0.9400390386581421,
      "logits/rejected": -0.801953136920929,
      "logps/chosen": -0.52001953125,
      "logps/rejected": -1.1759765148162842,
      "loss": 0.7746,
      "nll_loss": 0.7720702886581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0260009765625,
      "rewards/margins": 0.03278198093175888,
      "rewards/rejected": -0.05881347507238388,
      "step": 9390
    },
    {
      "epoch": 0.6848067606454667,
      "grad_norm": 2.3594042259965486,
      "learning_rate": 5.157106231293968e-07,
      "log_odds_chosen": 1.2204101085662842,
      "log_odds_ratio": -0.4327636659145355,
      "logits/chosen": -0.939257800579071,
      "logits/rejected": -0.8412109613418579,
      "logps/chosen": -0.515917956829071,
      "logps/rejected": -1.251562476158142,
      "loss": 0.7465,
      "nll_loss": 0.7353515625,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02583007887005806,
      "rewards/margins": 0.03674926608800888,
      "rewards/rejected": -0.06263427436351776,
      "step": 9400
    },
    {
      "epoch": 0.6855352784759406,
      "grad_norm": 2.4238597157392654,
      "learning_rate": 5.154365276419883e-07,
      "log_odds_chosen": 1.17822265625,
      "log_odds_ratio": -0.4363769590854645,
      "logits/chosen": -0.969531238079071,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.523242175579071,
      "logps/rejected": -1.2029297351837158,
      "loss": 0.7471,
      "nll_loss": 0.7217773199081421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02613525465130806,
      "rewards/margins": 0.03399658203125,
      "rewards/rejected": -0.06014404445886612,
      "step": 9410
    },
    {
      "epoch": 0.6862637963064147,
      "grad_norm": 2.301915384191374,
      "learning_rate": 5.151628687282773e-07,
      "log_odds_chosen": 1.0185058116912842,
      "log_odds_ratio": -0.448486328125,
      "logits/chosen": -0.9501953125,
      "logits/rejected": -0.8783203363418579,
      "logps/chosen": -0.5555664300918579,
      "logps/rejected": -1.137109398841858,
      "loss": 0.7568,
      "nll_loss": 0.7879883050918579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02777709998190403,
      "rewards/margins": 0.02907409705221653,
      "rewards/rejected": -0.05684814602136612,
      "step": 9420
    },
    {
      "epoch": 0.6869923141368885,
      "grad_norm": 2.7285936439070233,
      "learning_rate": 5.148896452305499e-07,
      "log_odds_chosen": 1.184179663658142,
      "log_odds_ratio": -0.42338865995407104,
      "logits/chosen": -0.980273425579071,
      "logits/rejected": -0.8812500238418579,
      "logps/chosen": -0.527050793170929,
      "logps/rejected": -1.2041015625,
      "loss": 0.7417,
      "nll_loss": 0.755566418170929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02635497972369194,
      "rewards/margins": 0.0338287353515625,
      "rewards/rejected": -0.06020507961511612,
      "step": 9430
    },
    {
      "epoch": 0.6877208319673624,
      "grad_norm": 2.055583673362566,
      "learning_rate": 5.146168559953865e-07,
      "log_odds_chosen": 0.734509289264679,
      "log_odds_ratio": -0.5545409917831421,
      "logits/chosen": -0.9798828363418579,
      "logits/rejected": -0.874218761920929,
      "logps/chosen": -0.578417956829071,
      "logps/rejected": -1.0078125,
      "loss": 0.7668,
      "nll_loss": 0.7535156011581421,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0289306640625,
      "rewards/margins": 0.021485138684511185,
      "rewards/rejected": -0.05043945461511612,
      "step": 9440
    },
    {
      "epoch": 0.6884493497978363,
      "grad_norm": 2.1184215989238155,
      "learning_rate": 5.143444998736397e-07,
      "log_odds_chosen": 0.885937511920929,
      "log_odds_ratio": -0.4996093809604645,
      "logits/chosen": -0.974414050579071,
      "logits/rejected": -0.8746093511581421,
      "logps/chosen": -0.5557616949081421,
      "logps/rejected": -1.066015601158142,
      "loss": 0.7531,
      "nll_loss": 0.7447265386581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02777099609375,
      "rewards/margins": 0.02551574632525444,
      "rewards/rejected": -0.053314208984375,
      "step": 9450
    },
    {
      "epoch": 0.6891778676283102,
      "grad_norm": 2.2193627108361667,
      "learning_rate": 5.140725757204157e-07,
      "log_odds_chosen": 1.1171875,
      "log_odds_ratio": -0.45405274629592896,
      "logits/chosen": -1.0203125476837158,
      "logits/rejected": -0.910937488079071,
      "logps/chosen": -0.5474609136581421,
      "logps/rejected": -1.2060546875,
      "loss": 0.7515,
      "nll_loss": 0.732226550579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02739868126809597,
      "rewards/margins": 0.03289489820599556,
      "rewards/rejected": -0.06029052659869194,
      "step": 9460
    },
    {
      "epoch": 0.6899063854587841,
      "grad_norm": 2.0273689676494757,
      "learning_rate": 5.138010823950523e-07,
      "log_odds_chosen": 0.974682629108429,
      "log_odds_ratio": -0.4537597596645355,
      "logits/chosen": -0.988476574420929,
      "logits/rejected": -0.8853515386581421,
      "logps/chosen": -0.5245116949081421,
      "logps/rejected": -1.067773461341858,
      "loss": 0.7387,
      "nll_loss": 0.714160144329071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02622680738568306,
      "rewards/margins": 0.02721557579934597,
      "rewards/rejected": -0.053466796875,
      "step": 9470
    },
    {
      "epoch": 0.690634903289258,
      "grad_norm": 2.2366488237483324,
      "learning_rate": 5.135300187611007e-07,
      "log_odds_chosen": 0.9944823980331421,
      "log_odds_ratio": -0.4447265565395355,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.9169921875,
      "logps/chosen": -0.5443359613418579,
      "logps/rejected": -1.12109375,
      "loss": 0.7535,
      "nll_loss": 0.7100585699081421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02721557579934597,
      "rewards/margins": 0.02879943884909153,
      "rewards/rejected": -0.05605468899011612,
      "step": 9480
    },
    {
      "epoch": 0.6913634211197319,
      "grad_norm": 2.4859288413871115,
      "learning_rate": 5.132593836863036e-07,
      "log_odds_chosen": 1.095312476158142,
      "log_odds_ratio": -0.4375976622104645,
      "logits/chosen": -1.0185546875,
      "logits/rejected": -0.920117199420929,
      "logps/chosen": -0.537890613079071,
      "logps/rejected": -1.228515625,
      "loss": 0.7443,
      "nll_loss": 0.715039074420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02690429612994194,
      "rewards/margins": 0.03453369066119194,
      "rewards/rejected": -0.06145019456744194,
      "step": 9490
    },
    {
      "epoch": 0.6920919389502058,
      "grad_norm": 2.2488202922682654,
      "learning_rate": 5.129891760425771e-07,
      "log_odds_chosen": 1.054931640625,
      "log_odds_ratio": -0.47661131620407104,
      "logits/chosen": -0.973437488079071,
      "logits/rejected": -0.8480468988418579,
      "logps/chosen": -0.539257824420929,
      "logps/rejected": -1.139062523841858,
      "loss": 0.7357,
      "nll_loss": 0.7530273199081421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02698974683880806,
      "rewards/margins": 0.02995605394244194,
      "rewards/rejected": -0.05694580078125,
      "step": 9500
    },
    {
      "epoch": 0.6928204567806797,
      "grad_norm": 2.741573938264991,
      "learning_rate": 5.127193947059894e-07,
      "log_odds_chosen": 1.151953101158142,
      "log_odds_ratio": -0.41801756620407104,
      "logits/chosen": -1.005859375,
      "logits/rejected": -0.887499988079071,
      "logps/chosen": -0.5174804925918579,
      "logps/rejected": -1.175390601158142,
      "loss": 0.745,
      "nll_loss": 0.693652331829071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02587890625,
      "rewards/margins": 0.03288879245519638,
      "rewards/rejected": -0.05874023586511612,
      "step": 9510
    },
    {
      "epoch": 0.6935489746111536,
      "grad_norm": 2.5769250054707853,
      "learning_rate": 5.124500385567424e-07,
      "log_odds_chosen": 0.979736328125,
      "log_odds_ratio": -0.474365234375,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.884765625,
      "logps/chosen": -0.515820324420929,
      "logps/rejected": -1.051171898841858,
      "loss": 0.75,
      "nll_loss": 0.6595703363418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02581482008099556,
      "rewards/margins": 0.02676544152200222,
      "rewards/rejected": -0.05254516750574112,
      "step": 9520
    },
    {
      "epoch": 0.6942774924416275,
      "grad_norm": 2.18298492695888,
      "learning_rate": 5.121811064791514e-07,
      "log_odds_chosen": 1.1564452648162842,
      "log_odds_ratio": -0.4209960997104645,
      "logits/chosen": -1.024023413658142,
      "logits/rejected": -0.9164062738418579,
      "logps/chosen": -0.528124988079071,
      "logps/rejected": -1.2224609851837158,
      "loss": 0.7597,
      "nll_loss": 0.7002929449081421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02643432654440403,
      "rewards/margins": 0.03473663330078125,
      "rewards/rejected": -0.06122436374425888,
      "step": 9530
    },
    {
      "epoch": 0.6950060102721014,
      "grad_norm": 2.0557219480534368,
      "learning_rate": 5.11912597361626e-07,
      "log_odds_chosen": 1.140234351158142,
      "log_odds_ratio": -0.4342285096645355,
      "logits/chosen": -1.0060546398162842,
      "logits/rejected": -0.880078136920929,
      "logps/chosen": -0.534375011920929,
      "logps/rejected": -1.201171875,
      "loss": 0.7406,
      "nll_loss": 0.7227538824081421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02672119066119194,
      "rewards/margins": 0.03330077975988388,
      "rewards/rejected": -0.06004638597369194,
      "step": 9540
    },
    {
      "epoch": 0.6957345281025753,
      "grad_norm": 2.9307660431548164,
      "learning_rate": 5.116445100966509e-07,
      "log_odds_chosen": 1.1210448741912842,
      "log_odds_ratio": -0.42460936307907104,
      "logits/chosen": -1.014257788658142,
      "logits/rejected": -0.919921875,
      "logps/chosen": -0.53076171875,
      "logps/rejected": -1.167578101158142,
      "loss": 0.7434,
      "nll_loss": 0.691113293170929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02654418908059597,
      "rewards/margins": 0.03175506740808487,
      "rewards/rejected": -0.05833740159869194,
      "step": 9550
    },
    {
      "epoch": 0.6964630459330492,
      "grad_norm": 2.3554695610367857,
      "learning_rate": 5.113768435807665e-07,
      "log_odds_chosen": 1.2646484375,
      "log_odds_ratio": -0.41801756620407104,
      "logits/chosen": -0.998046875,
      "logits/rejected": -0.888867199420929,
      "logps/chosen": -0.48710936307907104,
      "logps/rejected": -1.1882812976837158,
      "loss": 0.7365,
      "nll_loss": 0.668164074420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02435913123190403,
      "rewards/margins": 0.03508300706744194,
      "rewards/rejected": -0.059356689453125,
      "step": 9560
    },
    {
      "epoch": 0.6971915637635231,
      "grad_norm": 2.0404953111954214,
      "learning_rate": 5.111095967145495e-07,
      "log_odds_chosen": 1.291406273841858,
      "log_odds_ratio": -0.4346679747104645,
      "logits/chosen": -1.00390625,
      "logits/rejected": -0.897265613079071,
      "logps/chosen": -0.551464855670929,
      "logps/rejected": -1.311132788658142,
      "loss": 0.745,
      "nll_loss": 0.698046863079071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02756347693502903,
      "rewards/margins": 0.03797607496380806,
      "rewards/rejected": -0.06560058891773224,
      "step": 9570
    },
    {
      "epoch": 0.697920081593997,
      "grad_norm": 2.1229132078269712,
      "learning_rate": 5.108427684025953e-07,
      "log_odds_chosen": 1.255468726158142,
      "log_odds_ratio": -0.3909668028354645,
      "logits/chosen": -0.9888671636581421,
      "logits/rejected": -0.836718738079071,
      "logps/chosen": -0.518750011920929,
      "logps/rejected": -1.221093773841858,
      "loss": 0.7396,
      "nll_loss": 0.7236328125,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.02592773362994194,
      "rewards/margins": 0.03513946384191513,
      "rewards/rejected": -0.06110839918255806,
      "step": 9580
    },
    {
      "epoch": 0.6986485994244709,
      "grad_norm": 2.1724313024339086,
      "learning_rate": 5.105763575534971e-07,
      "log_odds_chosen": 1.211328148841858,
      "log_odds_ratio": -0.42622071504592896,
      "logits/chosen": -1.0226562023162842,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.5606445074081421,
      "logps/rejected": -1.304101586341858,
      "loss": 0.7741,
      "nll_loss": 0.733105480670929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02803955040872097,
      "rewards/margins": 0.03717651218175888,
      "rewards/rejected": -0.06520996242761612,
      "step": 9590
    },
    {
      "epoch": 0.6993771172549448,
      "grad_norm": 2.574321287470622,
      "learning_rate": 5.103103630798288e-07,
      "log_odds_chosen": 0.9340575933456421,
      "log_odds_ratio": -0.47050780057907104,
      "logits/chosen": -0.9720703363418579,
      "logits/rejected": -0.8656250238418579,
      "logps/chosen": -0.5077148675918579,
      "logps/rejected": -1.0529296398162842,
      "loss": 0.7465,
      "nll_loss": 0.701171875,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02537231519818306,
      "rewards/margins": 0.027249908074736595,
      "rewards/rejected": -0.05262451246380806,
      "step": 9600
    },
    {
      "epoch": 0.7001056350854187,
      "grad_norm": 2.918940862532038,
      "learning_rate": 5.100447838981258e-07,
      "log_odds_chosen": 1.132421851158142,
      "log_odds_ratio": -0.4491210877895355,
      "logits/chosen": -0.9912109375,
      "logits/rejected": -0.865234375,
      "logps/chosen": -0.541210949420929,
      "logps/rejected": -1.202734351158142,
      "loss": 0.7445,
      "nll_loss": 0.7333984375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02704467810690403,
      "rewards/margins": 0.03307495266199112,
      "rewards/rejected": -0.06014404445886612,
      "step": 9610
    },
    {
      "epoch": 0.7008341529158926,
      "grad_norm": 1.9915684177713002,
      "learning_rate": 5.09779618928866e-07,
      "log_odds_chosen": 1.233789086341858,
      "log_odds_ratio": -0.42138671875,
      "logits/chosen": -1.0009765625,
      "logits/rejected": -0.896484375,
      "logps/chosen": -0.510058581829071,
      "logps/rejected": -1.2374999523162842,
      "loss": 0.7284,
      "nll_loss": 0.6834961175918579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.025482177734375,
      "rewards/margins": 0.03636779636144638,
      "rewards/rejected": -0.0618896484375,
      "step": 9620
    },
    {
      "epoch": 0.7015626707463665,
      "grad_norm": 2.029117225002583,
      "learning_rate": 5.095148670964526e-07,
      "log_odds_chosen": 1.053466796875,
      "log_odds_ratio": -0.4944824278354645,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.8853515386581421,
      "logps/chosen": -0.574023425579071,
      "logps/rejected": -1.1875,
      "loss": 0.746,
      "nll_loss": 0.768750011920929,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.02868957445025444,
      "rewards/margins": 0.03070983849465847,
      "rewards/rejected": -0.05939941480755806,
      "step": 9630
    },
    {
      "epoch": 0.7022911885768404,
      "grad_norm": 2.99025367615893,
      "learning_rate": 5.092505273291941e-07,
      "log_odds_chosen": 1.1453125476837158,
      "log_odds_ratio": -0.4366699159145355,
      "logits/chosen": -0.98046875,
      "logits/rejected": -0.8580077886581421,
      "logps/chosen": -0.5555664300918579,
      "logps/rejected": -1.235742211341858,
      "loss": 0.7513,
      "nll_loss": 0.762890636920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02777099609375,
      "rewards/margins": 0.03396911546587944,
      "rewards/rejected": -0.06177978590130806,
      "step": 9640
    },
    {
      "epoch": 0.7030197064073144,
      "grad_norm": 2.2261167682092005,
      "learning_rate": 5.089865985592877e-07,
      "log_odds_chosen": 0.8075195550918579,
      "log_odds_ratio": -0.5425781011581421,
      "logits/chosen": -0.990039050579071,
      "logits/rejected": -0.888476550579071,
      "logps/chosen": -0.584667980670929,
      "logps/rejected": -1.083593726158142,
      "loss": 0.7564,
      "nll_loss": 0.739453136920929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02925414964556694,
      "rewards/margins": 0.02491149865090847,
      "rewards/rejected": -0.05419921875,
      "step": 9650
    },
    {
      "epoch": 0.7037482242377883,
      "grad_norm": 2.6829233638148526,
      "learning_rate": 5.087230797227999e-07,
      "log_odds_chosen": 0.9297119379043579,
      "log_odds_ratio": -0.49785155057907104,
      "logits/chosen": -0.991992175579071,
      "logits/rejected": -0.891796886920929,
      "logps/chosen": -0.541308581829071,
      "logps/rejected": -1.1015625,
      "loss": 0.7284,
      "nll_loss": 0.6954101324081421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.027099609375,
      "rewards/margins": 0.028011322021484375,
      "rewards/rejected": -0.05502929538488388,
      "step": 9660
    },
    {
      "epoch": 0.7044767420682622,
      "grad_norm": 2.3120073053612136,
      "learning_rate": 5.084599697596493e-07,
      "log_odds_chosen": 1.189453125,
      "log_odds_ratio": -0.437255859375,
      "logits/chosen": -0.966992199420929,
      "logits/rejected": -0.8187500238418579,
      "logps/chosen": -0.587109386920929,
      "logps/rejected": -1.3251953125,
      "loss": 0.7444,
      "nll_loss": 0.769335925579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.029388427734375,
      "rewards/margins": 0.0369110107421875,
      "rewards/rejected": -0.06625976413488388,
      "step": 9670
    },
    {
      "epoch": 0.7052052598987361,
      "grad_norm": 2.3479202064804223,
      "learning_rate": 5.081972676135887e-07,
      "log_odds_chosen": 1.048486351966858,
      "log_odds_ratio": -0.451416015625,
      "logits/chosen": -1.0255858898162842,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.5321289300918579,
      "logps/rejected": -1.167578101158142,
      "loss": 0.7552,
      "nll_loss": 0.737500011920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02658691443502903,
      "rewards/margins": 0.03179015964269638,
      "rewards/rejected": -0.05841674655675888,
      "step": 9680
    },
    {
      "epoch": 0.70593377772921,
      "grad_norm": 2.369422183944735,
      "learning_rate": 5.079349722321863e-07,
      "log_odds_chosen": 1.09521484375,
      "log_odds_ratio": -0.45112305879592896,
      "logits/chosen": -0.9876953363418579,
      "logits/rejected": -0.8863281011581421,
      "logps/chosen": -0.549121081829071,
      "logps/rejected": -1.23046875,
      "loss": 0.7521,
      "nll_loss": 0.735546886920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02744751051068306,
      "rewards/margins": 0.03408203274011612,
      "rewards/rejected": -0.06154785305261612,
      "step": 9690
    },
    {
      "epoch": 0.7066622955596839,
      "grad_norm": 2.035245806589505,
      "learning_rate": 5.076730825668097e-07,
      "log_odds_chosen": 1.1555664539337158,
      "log_odds_ratio": -0.3966308534145355,
      "logits/chosen": -0.9917968511581421,
      "logits/rejected": -0.839062511920929,
      "logps/chosen": -0.49003905057907104,
      "logps/rejected": -1.119726538658142,
      "loss": 0.7445,
      "nll_loss": 0.6982421875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02452392503619194,
      "rewards/margins": 0.03144073486328125,
      "rewards/rejected": -0.05598144605755806,
      "step": 9700
    },
    {
      "epoch": 0.7073908133901577,
      "grad_norm": 2.013424732147912,
      "learning_rate": 5.074115975726062e-07,
      "log_odds_chosen": 1.056542992591858,
      "log_odds_ratio": -0.45478516817092896,
      "logits/chosen": -0.9833984375,
      "logits/rejected": -0.8837890625,
      "logps/chosen": -0.5478515625,
      "logps/rejected": -1.1525390148162842,
      "loss": 0.7406,
      "nll_loss": 0.72998046875,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.027374267578125,
      "rewards/margins": 0.03026122972369194,
      "rewards/rejected": -0.05760497972369194,
      "step": 9710
    },
    {
      "epoch": 0.7081193312206316,
      "grad_norm": 2.1437370736279218,
      "learning_rate": 5.071505162084872e-07,
      "log_odds_chosen": 1.194677710533142,
      "log_odds_ratio": -0.4051269590854645,
      "logits/chosen": -1.0167968273162842,
      "logits/rejected": -0.8701171875,
      "logps/chosen": -0.53173828125,
      "logps/rejected": -1.2216796875,
      "loss": 0.7469,
      "nll_loss": 0.754687488079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.026580810546875,
      "rewards/margins": 0.03452453762292862,
      "rewards/rejected": -0.06110839918255806,
      "step": 9720
    },
    {
      "epoch": 0.7088478490511055,
      "grad_norm": 2.173166360258777,
      "learning_rate": 5.068898374371097e-07,
      "log_odds_chosen": 1.204003930091858,
      "log_odds_ratio": -0.42036134004592896,
      "logits/chosen": -0.9720703363418579,
      "logits/rejected": -0.8666015863418579,
      "logps/chosen": -0.562792956829071,
      "logps/rejected": -1.324609398841858,
      "loss": 0.7537,
      "nll_loss": 0.709277331829071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.02813415601849556,
      "rewards/margins": 0.03804931789636612,
      "rewards/rejected": -0.06624756008386612,
      "step": 9730
    },
    {
      "epoch": 0.7095763668815794,
      "grad_norm": 2.184027051858119,
      "learning_rate": 5.066295602248595e-07,
      "log_odds_chosen": 1.107421875,
      "log_odds_ratio": -0.42875975370407104,
      "logits/chosen": -1.0183594226837158,
      "logits/rejected": -0.9087890386581421,
      "logps/chosen": -0.524707019329071,
      "logps/rejected": -1.170507788658142,
      "loss": 0.7561,
      "nll_loss": 0.717480480670929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02622680738568306,
      "rewards/margins": 0.03234253078699112,
      "rewards/rejected": -0.05859375,
      "step": 9740
    },
    {
      "epoch": 0.7103048847120533,
      "grad_norm": 1.972068148161154,
      "learning_rate": 5.063696835418334e-07,
      "log_odds_chosen": 1.128271460533142,
      "log_odds_ratio": -0.46406251192092896,
      "logits/chosen": -1.0027344226837158,
      "logits/rejected": -0.9154297113418579,
      "logps/chosen": -0.52734375,
      "logps/rejected": -1.177343726158142,
      "loss": 0.7558,
      "nll_loss": 0.7157226800918579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02638549730181694,
      "rewards/margins": 0.03253021091222763,
      "rewards/rejected": -0.05889892578125,
      "step": 9750
    },
    {
      "epoch": 0.7110334025425272,
      "grad_norm": 2.5023584067756883,
      "learning_rate": 5.061102063618226e-07,
      "log_odds_chosen": 1.180419921875,
      "log_odds_ratio": -0.4254394471645355,
      "logits/chosen": -0.976757824420929,
      "logits/rejected": -0.857226550579071,
      "logps/chosen": -0.5234375,
      "logps/rejected": -1.207617163658142,
      "loss": 0.7446,
      "nll_loss": 0.708984375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02619018591940403,
      "rewards/margins": 0.03417358547449112,
      "rewards/rejected": -0.060394287109375,
      "step": 9760
    },
    {
      "epoch": 0.7117619203730011,
      "grad_norm": 2.2733322055151386,
      "learning_rate": 5.058511276622955e-07,
      "log_odds_chosen": 0.9599609375,
      "log_odds_ratio": -0.5234375,
      "logits/chosen": -1.0251953601837158,
      "logits/rejected": -0.925976574420929,
      "logps/chosen": -0.58544921875,
      "logps/rejected": -1.171484351158142,
      "loss": 0.7403,
      "nll_loss": 0.7208007574081421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02926025353372097,
      "rewards/margins": 0.02929077111184597,
      "rewards/rejected": -0.05858154222369194,
      "step": 9770
    },
    {
      "epoch": 0.712490438203475,
      "grad_norm": 2.2877089145808935,
      "learning_rate": 5.055924464243811e-07,
      "log_odds_chosen": 1.0182373523712158,
      "log_odds_ratio": -0.49223631620407104,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.9027343988418579,
      "logps/chosen": -0.548535168170929,
      "logps/rejected": -1.1794922351837158,
      "loss": 0.7341,
      "nll_loss": 0.714062511920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02739868126809597,
      "rewards/margins": 0.03154601901769638,
      "rewards/rejected": -0.0589599609375,
      "step": 9780
    },
    {
      "epoch": 0.7132189560339489,
      "grad_norm": 2.3816591374277474,
      "learning_rate": 5.053341616328513e-07,
      "log_odds_chosen": 1.0804932117462158,
      "log_odds_ratio": -0.46513670682907104,
      "logits/chosen": -0.9849609136581421,
      "logits/rejected": -0.8763672113418579,
      "logps/chosen": -0.5419921875,
      "logps/rejected": -1.1984374523162842,
      "loss": 0.7389,
      "nll_loss": 0.756640613079071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.027099609375,
      "rewards/margins": 0.032738495618104935,
      "rewards/rejected": -0.05988769605755806,
      "step": 9790
    },
    {
      "epoch": 0.7139474738644228,
      "grad_norm": 2.4021707251718514,
      "learning_rate": 5.050762722761054e-07,
      "log_odds_chosen": 1.0861694812774658,
      "log_odds_ratio": -0.4599365293979645,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.5624023675918579,
      "logps/rejected": -1.182226538658142,
      "loss": 0.7477,
      "nll_loss": 0.70849609375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02813720703125,
      "rewards/margins": 0.03105621412396431,
      "rewards/rejected": -0.05919189378619194,
      "step": 9800
    },
    {
      "epoch": 0.7146759916948967,
      "grad_norm": 2.9840735013847652,
      "learning_rate": 5.048187773461522e-07,
      "log_odds_chosen": 1.1044921875,
      "log_odds_ratio": -0.44707030057907104,
      "logits/chosen": -0.9693359136581421,
      "logits/rejected": -0.871289074420929,
      "logps/chosen": -0.524121105670929,
      "logps/rejected": -1.1628906726837158,
      "loss": 0.7405,
      "nll_loss": 0.738574206829071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.026214599609375,
      "rewards/margins": 0.03192138671875,
      "rewards/rejected": -0.05817871168255806,
      "step": 9810
    },
    {
      "epoch": 0.7154045095253706,
      "grad_norm": 2.3158986066648324,
      "learning_rate": 5.045616758385946e-07,
      "log_odds_chosen": 0.99462890625,
      "log_odds_ratio": -0.46503907442092896,
      "logits/chosen": -1.052148461341858,
      "logits/rejected": -0.922070324420929,
      "logps/chosen": -0.550585925579071,
      "logps/rejected": -1.119531273841858,
      "loss": 0.7229,
      "nll_loss": 0.6693359613418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02749023400247097,
      "rewards/margins": 0.02844848670065403,
      "rewards/rejected": -0.05596923828125,
      "step": 9820
    },
    {
      "epoch": 0.7161330273558445,
      "grad_norm": 2.078723743309297,
      "learning_rate": 5.04304966752612e-07,
      "log_odds_chosen": 0.958300769329071,
      "log_odds_ratio": -0.46757811307907104,
      "logits/chosen": -0.985546886920929,
      "logits/rejected": -0.906054675579071,
      "logps/chosen": -0.523242175579071,
      "logps/rejected": -1.0535156726837158,
      "loss": 0.7303,
      "nll_loss": 0.7314453125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02617187425494194,
      "rewards/margins": 0.0265350341796875,
      "rewards/rejected": -0.05269775539636612,
      "step": 9830
    },
    {
      "epoch": 0.7168615451863184,
      "grad_norm": 2.0668949372311305,
      "learning_rate": 5.040486490909449e-07,
      "log_odds_chosen": 0.8531249761581421,
      "log_odds_ratio": -0.4928222596645355,
      "logits/chosen": -1.009374976158142,
      "logits/rejected": -0.905468761920929,
      "logps/chosen": -0.576367199420929,
      "logps/rejected": -1.0890624523162842,
      "loss": 0.765,
      "nll_loss": 0.749218761920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02879638597369194,
      "rewards/margins": 0.02562255784869194,
      "rewards/rejected": -0.05447997897863388,
      "step": 9840
    },
    {
      "epoch": 0.7175900630167923,
      "grad_norm": 2.447949852419168,
      "learning_rate": 5.037927218598784e-07,
      "log_odds_chosen": 0.914843738079071,
      "log_odds_ratio": -0.4925293028354645,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.57470703125,
      "logps/rejected": -1.1337890625,
      "loss": 0.7629,
      "nll_loss": 0.76123046875,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02872924879193306,
      "rewards/margins": 0.02797546423971653,
      "rewards/rejected": -0.05670166015625,
      "step": 9850
    },
    {
      "epoch": 0.7183185808472662,
      "grad_norm": 2.1002695007484977,
      "learning_rate": 5.035371840692256e-07,
      "log_odds_chosen": 1.178063988685608,
      "log_odds_ratio": -0.46464842557907104,
      "logits/chosen": -1.0167968273162842,
      "logits/rejected": -0.9208984375,
      "logps/chosen": -0.554003894329071,
      "logps/rejected": -1.2990233898162842,
      "loss": 0.7343,
      "nll_loss": 0.7376953363418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02769775316119194,
      "rewards/margins": 0.03723945468664169,
      "rewards/rejected": -0.06496582180261612,
      "step": 9860
    },
    {
      "epoch": 0.7190470986777402,
      "grad_norm": 4.112522435127364,
      "learning_rate": 5.032820347323117e-07,
      "log_odds_chosen": 1.218652367591858,
      "log_odds_ratio": -0.41987305879592896,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.905468761920929,
      "logps/chosen": -0.54541015625,
      "logps/rejected": -1.255273461341858,
      "loss": 0.7313,
      "nll_loss": 0.7455078363418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02724609337747097,
      "rewards/margins": 0.03558349609375,
      "rewards/rejected": -0.06284179538488388,
      "step": 9870
    },
    {
      "epoch": 0.7197756165082141,
      "grad_norm": 2.3866122962919154,
      "learning_rate": 5.030272728659587e-07,
      "log_odds_chosen": 1.083642601966858,
      "log_odds_ratio": -0.4525390565395355,
      "logits/chosen": -1.0056641101837158,
      "logits/rejected": -0.8941406011581421,
      "logps/chosen": -0.51904296875,
      "logps/rejected": -1.1455078125,
      "loss": 0.7248,
      "nll_loss": 0.68994140625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02595825120806694,
      "rewards/margins": 0.03135223314166069,
      "rewards/rejected": -0.05729980394244194,
      "step": 9880
    },
    {
      "epoch": 0.720504134338688,
      "grad_norm": 2.2602676455200945,
      "learning_rate": 5.02772897490468e-07,
      "log_odds_chosen": 1.131445288658142,
      "log_odds_ratio": -0.48408204317092896,
      "logits/chosen": -0.996289074420929,
      "logits/rejected": -0.914257824420929,
      "logps/chosen": -0.561816394329071,
      "logps/rejected": -1.2492187023162842,
      "loss": 0.7544,
      "nll_loss": 0.6650390625,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02807006798684597,
      "rewards/margins": 0.03436279296875,
      "rewards/rejected": -0.06246338039636612,
      "step": 9890
    },
    {
      "epoch": 0.7212326521691619,
      "grad_norm": 2.3481688553110027,
      "learning_rate": 5.025189076296061e-07,
      "log_odds_chosen": 1.2664062976837158,
      "log_odds_ratio": -0.4034179747104645,
      "logits/chosen": -1.046289086341858,
      "logits/rejected": -0.903515636920929,
      "logps/chosen": -0.5259765386581421,
      "logps/rejected": -1.2800781726837158,
      "loss": 0.7001,
      "nll_loss": 0.6783202886581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.02629394456744194,
      "rewards/margins": 0.03771362453699112,
      "rewards/rejected": -0.06402587890625,
      "step": 9900
    },
    {
      "epoch": 0.7219611699996358,
      "grad_norm": 1.9306884941534226,
      "learning_rate": 5.02265302310588e-07,
      "log_odds_chosen": 1.0615234375,
      "log_odds_ratio": -0.4659667909145355,
      "logits/chosen": -1.0060546398162842,
      "logits/rejected": -0.9072265625,
      "logps/chosen": -0.5248047113418579,
      "logps/rejected": -1.130761742591858,
      "loss": 0.7354,
      "nll_loss": 0.6986328363418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02622680738568306,
      "rewards/margins": 0.03023834154009819,
      "rewards/rejected": -0.05650634691119194,
      "step": 9910
    },
    {
      "epoch": 0.7226896878301097,
      "grad_norm": 2.0687416623868207,
      "learning_rate": 5.020120805640619e-07,
      "log_odds_chosen": 1.085839867591858,
      "log_odds_ratio": -0.4637695252895355,
      "logits/chosen": -1.011132836341858,
      "logits/rejected": -0.9134765863418579,
      "logps/chosen": -0.5448242425918579,
      "logps/rejected": -1.175195336341858,
      "loss": 0.7218,
      "nll_loss": 0.6826171875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02721557579934597,
      "rewards/margins": 0.03156127780675888,
      "rewards/rejected": -0.05872802808880806,
      "step": 9920
    },
    {
      "epoch": 0.7234182056605836,
      "grad_norm": 2.1426049373894918,
      "learning_rate": 5.017592414240933e-07,
      "log_odds_chosen": 1.1018555164337158,
      "log_odds_ratio": -0.4515136778354645,
      "logits/chosen": -0.9722656011581421,
      "logits/rejected": -0.859179675579071,
      "logps/chosen": -0.567089855670929,
      "logps/rejected": -1.2101562023162842,
      "loss": 0.7396,
      "nll_loss": 0.747265636920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02833862230181694,
      "rewards/margins": 0.03215942531824112,
      "rewards/rejected": -0.06052245944738388,
      "step": 9930
    },
    {
      "epoch": 0.7241467234910575,
      "grad_norm": 2.0545723143558763,
      "learning_rate": 5.015067839281497e-07,
      "log_odds_chosen": 1.186425805091858,
      "log_odds_ratio": -0.40605467557907104,
      "logits/chosen": -0.990429699420929,
      "logits/rejected": -0.849414050579071,
      "logps/chosen": -0.5234375,
      "logps/rejected": -1.221093773841858,
      "loss": 0.7287,
      "nll_loss": 0.682421863079071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.02618408203125,
      "rewards/margins": 0.03484497219324112,
      "rewards/rejected": -0.06107177585363388,
      "step": 9940
    },
    {
      "epoch": 0.7248752413215314,
      "grad_norm": 2.6081528322136376,
      "learning_rate": 5.012547071170856e-07,
      "log_odds_chosen": 1.1580078601837158,
      "log_odds_ratio": -0.431640625,
      "logits/chosen": -1.0283203125,
      "logits/rejected": -0.8916015625,
      "logps/chosen": -0.52099609375,
      "logps/rejected": -1.226171851158142,
      "loss": 0.7362,
      "nll_loss": 0.7119140625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02606201171875,
      "rewards/margins": 0.03524170070886612,
      "rewards/rejected": -0.06130371242761612,
      "step": 9950
    },
    {
      "epoch": 0.7256037591520053,
      "grad_norm": 2.220616283511376,
      "learning_rate": 5.010030100351265e-07,
      "log_odds_chosen": 1.071874976158142,
      "log_odds_ratio": -0.4341796934604645,
      "logits/chosen": -1.0105469226837158,
      "logits/rejected": -0.8843749761581421,
      "logps/chosen": -0.5423828363418579,
      "logps/rejected": -1.16796875,
      "loss": 0.7326,
      "nll_loss": 0.704296886920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02711791917681694,
      "rewards/margins": 0.03130187839269638,
      "rewards/rejected": -0.05839843675494194,
      "step": 9960
    },
    {
      "epoch": 0.7263322769824792,
      "grad_norm": 2.116385379187578,
      "learning_rate": 5.007516917298542e-07,
      "log_odds_chosen": 0.977490246295929,
      "log_odds_ratio": -0.49604493379592896,
      "logits/chosen": -0.986523449420929,
      "logits/rejected": -0.89453125,
      "logps/chosen": -0.5433593988418579,
      "logps/rejected": -1.111425757408142,
      "loss": 0.7362,
      "nll_loss": 0.6913086175918579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02714843675494194,
      "rewards/margins": 0.02843913994729519,
      "rewards/rejected": -0.05558471754193306,
      "step": 9970
    },
    {
      "epoch": 0.727060794812953,
      "grad_norm": 2.0370903268787517,
      "learning_rate": 5.005007512521914e-07,
      "log_odds_chosen": 1.0275390148162842,
      "log_odds_ratio": -0.45795899629592896,
      "logits/chosen": -0.9876953363418579,
      "logits/rejected": -0.8607422113418579,
      "logps/chosen": -0.54345703125,
      "logps/rejected": -1.1160156726837158,
      "loss": 0.7378,
      "nll_loss": 0.7286132574081421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0272064208984375,
      "rewards/margins": 0.028618622571229935,
      "rewards/rejected": -0.05581054836511612,
      "step": 9980
    },
    {
      "epoch": 0.7277893126434269,
      "grad_norm": 1.9436414787122638,
      "learning_rate": 5.002501876563868e-07,
      "log_odds_chosen": 1.0779297351837158,
      "log_odds_ratio": -0.46162110567092896,
      "logits/chosen": -1.0226562023162842,
      "logits/rejected": -0.898242175579071,
      "logps/chosen": -0.5406249761581421,
      "logps/rejected": -1.191015601158142,
      "loss": 0.7418,
      "nll_loss": 0.6949218511581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02704467810690403,
      "rewards/margins": 0.0324554443359375,
      "rewards/rejected": -0.05952148512005806,
      "step": 9990
    },
    {
      "epoch": 0.7285178304739008,
      "grad_norm": 2.4914063212421813,
      "learning_rate": 5.000000000000001e-07,
      "log_odds_chosen": 1.0949218273162842,
      "log_odds_ratio": -0.4705566465854645,
      "logits/chosen": -1.0212891101837158,
      "logits/rejected": -0.909375011920929,
      "logps/chosen": -0.601757824420929,
      "logps/rejected": -1.3220703601837158,
      "loss": 0.7343,
      "nll_loss": 0.758007824420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.03005371056497097,
      "rewards/margins": 0.03604278713464737,
      "rewards/rejected": -0.06611327826976776,
      "step": 10000
    },
    {
      "epoch": 0.7285178304739008,
      "eval_log_odds_chosen": 0.9230248332023621,
      "eval_log_odds_ratio": -0.5139190554618835,
      "eval_logits/chosen": -0.882161021232605,
      "eval_logits/rejected": -0.7823830246925354,
      "eval_logps/chosen": -0.6252197027206421,
      "eval_logps/rejected": -1.228542447090149,
      "eval_loss": 1.046362042427063,
      "eval_nll_loss": 1.0217794179916382,
      "eval_rewards/accuracies": 0.7051994800567627,
      "eval_rewards/chosen": -0.031262174248695374,
      "eval_rewards/margins": 0.03016679547727108,
      "eval_rewards/rejected": -0.06142931059002876,
      "eval_runtime": 1411.3952,
      "eval_samples_per_second": 69.307,
      "eval_steps_per_second": 1.083,
      "step": 10000
    },
    {
      "epoch": 0.7292463483043747,
      "grad_norm": 2.5536359200480248,
      "learning_rate": 4.997501873438866e-07,
      "log_odds_chosen": 0.981005847454071,
      "log_odds_ratio": -0.49565428495407104,
      "logits/chosen": -0.9951171875,
      "logits/rejected": -0.9189453125,
      "logps/chosen": -0.5741211175918579,
      "logps/rejected": -1.1335937976837158,
      "loss": 0.7354,
      "nll_loss": 0.7671874761581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.0286865234375,
      "rewards/margins": 0.02800903283059597,
      "rewards/rejected": -0.05665283277630806,
      "step": 10010
    },
    {
      "epoch": 0.7299748661348486,
      "grad_norm": 1.9753093109952546,
      "learning_rate": 4.995007487521837e-07,
      "log_odds_chosen": 0.948925793170929,
      "log_odds_ratio": -0.5091797113418579,
      "logits/chosen": -0.971484363079071,
      "logits/rejected": -0.919140636920929,
      "logps/chosen": -0.5645507574081421,
      "logps/rejected": -1.1404297351837158,
      "loss": 0.7316,
      "nll_loss": 0.721386730670929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02823486365377903,
      "rewards/margins": 0.02881774865090847,
      "rewards/rejected": -0.05701904371380806,
      "step": 10020
    },
    {
      "epoch": 0.7307033839653225,
      "grad_norm": 2.310085060177436,
      "learning_rate": 4.992516832922944e-07,
      "log_odds_chosen": 0.9840332269668579,
      "log_odds_ratio": -0.46630859375,
      "logits/chosen": -1.0060546398162842,
      "logits/rejected": -0.8916015625,
      "logps/chosen": -0.555957019329071,
      "logps/rejected": -1.151757836341858,
      "loss": 0.7395,
      "nll_loss": 0.7294921875,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.027801513671875,
      "rewards/margins": 0.02981262281537056,
      "rewards/rejected": -0.0576171875,
      "step": 10030
    },
    {
      "epoch": 0.7314319017957964,
      "grad_norm": 2.046037175306457,
      "learning_rate": 4.990029900348745e-07,
      "log_odds_chosen": 0.9476562738418579,
      "log_odds_ratio": -0.49360352754592896,
      "logits/chosen": -1.0048828125,
      "logits/rejected": -0.875195324420929,
      "logps/chosen": -0.5472656488418579,
      "logps/rejected": -1.111718773841858,
      "loss": 0.7375,
      "nll_loss": 0.7425781488418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02736816368997097,
      "rewards/margins": 0.02818908728659153,
      "rewards/rejected": -0.05554809421300888,
      "step": 10040
    },
    {
      "epoch": 0.7321604196262703,
      "grad_norm": 2.3485495424474374,
      "learning_rate": 4.987546680538164e-07,
      "log_odds_chosen": 1.162011742591858,
      "log_odds_ratio": -0.44233399629592896,
      "logits/chosen": -1.0029296875,
      "logits/rejected": -0.884960949420929,
      "logps/chosen": -0.520800769329071,
      "logps/rejected": -1.2087891101837158,
      "loss": 0.7276,
      "nll_loss": 0.685546875,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02604370191693306,
      "rewards/margins": 0.03440551832318306,
      "rewards/rejected": -0.06044922024011612,
      "step": 10050
    },
    {
      "epoch": 0.7328889374567442,
      "grad_norm": 2.1097550414232638,
      "learning_rate": 4.98506716426236e-07,
      "log_odds_chosen": 1.0810546875,
      "log_odds_ratio": -0.47285157442092896,
      "logits/chosen": -1.031640648841858,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.56640625,
      "logps/rejected": -1.252343773841858,
      "loss": 0.7309,
      "nll_loss": 0.698925793170929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02829589881002903,
      "rewards/margins": 0.03429565578699112,
      "rewards/rejected": -0.06259765475988388,
      "step": 10060
    },
    {
      "epoch": 0.7336174552872181,
      "grad_norm": 2.4575164778398335,
      "learning_rate": 4.982591342324569e-07,
      "log_odds_chosen": 1.042382836341858,
      "log_odds_ratio": -0.4501953125,
      "logits/chosen": -0.916210949420929,
      "logits/rejected": -0.8345702886581421,
      "logps/chosen": -0.539746105670929,
      "logps/rejected": -1.16015625,
      "loss": 0.7439,
      "nll_loss": 0.6924804449081421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02699584886431694,
      "rewards/margins": 0.03103027306497097,
      "rewards/rejected": -0.05800781399011612,
      "step": 10070
    },
    {
      "epoch": 0.734345973117692,
      "grad_norm": 2.407453820169158,
      "learning_rate": 4.980119205559974e-07,
      "log_odds_chosen": 0.9537109136581421,
      "log_odds_ratio": -0.501708984375,
      "logits/chosen": -0.98046875,
      "logits/rejected": -0.9027343988418579,
      "logps/chosen": -0.544726550579071,
      "logps/rejected": -1.1101562976837158,
      "loss": 0.7556,
      "nll_loss": 0.7347656488418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02724609337747097,
      "rewards/margins": 0.02823181077837944,
      "rewards/rejected": -0.05549316480755806,
      "step": 10080
    },
    {
      "epoch": 0.7350744909481659,
      "grad_norm": 2.0846351255662845,
      "learning_rate": 4.977650744835554e-07,
      "log_odds_chosen": 1.107031226158142,
      "log_odds_ratio": -0.44987791776657104,
      "logits/chosen": -0.9574218988418579,
      "logits/rejected": -0.8402343988418579,
      "logps/chosen": -0.534960925579071,
      "logps/rejected": -1.170312523841858,
      "loss": 0.7491,
      "nll_loss": 0.7236328125,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02675781212747097,
      "rewards/margins": 0.03173217922449112,
      "rewards/rejected": -0.05853271484375,
      "step": 10090
    },
    {
      "epoch": 0.7358030087786399,
      "grad_norm": 2.0177327491517563,
      "learning_rate": 4.975185951049946e-07,
      "log_odds_chosen": 1.2335937023162842,
      "log_odds_ratio": -0.41010743379592896,
      "logits/chosen": -0.9898437261581421,
      "logits/rejected": -0.867382824420929,
      "logps/chosen": -0.5191406011581421,
      "logps/rejected": -1.244531273841858,
      "loss": 0.7407,
      "nll_loss": 0.708691418170929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02595825120806694,
      "rewards/margins": 0.03625793382525444,
      "rewards/rejected": -0.06220703199505806,
      "step": 10100
    },
    {
      "epoch": 0.7365315266091138,
      "grad_norm": 2.0324607300237085,
      "learning_rate": 4.972724815133302e-07,
      "log_odds_chosen": 1.1650390625,
      "log_odds_ratio": -0.42670899629592896,
      "logits/chosen": -0.9833984375,
      "logits/rejected": -0.8656250238418579,
      "logps/chosen": -0.551464855670929,
      "logps/rejected": -1.247656226158142,
      "loss": 0.756,
      "nll_loss": 0.713085949420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02756347693502903,
      "rewards/margins": 0.03481597825884819,
      "rewards/rejected": -0.06240234524011612,
      "step": 10110
    },
    {
      "epoch": 0.7372600444395877,
      "grad_norm": 2.068055070346846,
      "learning_rate": 4.97026732804715e-07,
      "log_odds_chosen": 1.0863769054412842,
      "log_odds_ratio": -0.442138671875,
      "logits/chosen": -1.007421851158142,
      "logits/rejected": -0.887890636920929,
      "logps/chosen": -0.521484375,
      "logps/rejected": -1.1828124523162842,
      "loss": 0.7592,
      "nll_loss": 0.7339843511581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02610473707318306,
      "rewards/margins": 0.03300781175494194,
      "rewards/rejected": -0.05911865085363388,
      "step": 10120
    },
    {
      "epoch": 0.7379885622700616,
      "grad_norm": 2.68853022887846,
      "learning_rate": 4.967813480784258e-07,
      "log_odds_chosen": 0.974853515625,
      "log_odds_ratio": -0.4686035215854645,
      "logits/chosen": -1.0076172351837158,
      "logits/rejected": -0.9027343988418579,
      "logps/chosen": -0.539257824420929,
      "logps/rejected": -1.1376953125,
      "loss": 0.7333,
      "nll_loss": 0.690625011920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02696533128619194,
      "rewards/margins": 0.02991638146340847,
      "rewards/rejected": -0.05686035007238388,
      "step": 10130
    },
    {
      "epoch": 0.7387170801005355,
      "grad_norm": 1.9875065998481452,
      "learning_rate": 4.965363264368484e-07,
      "log_odds_chosen": 1.204687476158142,
      "log_odds_ratio": -0.42802733182907104,
      "logits/chosen": -1.010156273841858,
      "logits/rejected": -0.883593738079071,
      "logps/chosen": -0.5303710699081421,
      "logps/rejected": -1.2716796398162842,
      "loss": 0.7338,
      "nll_loss": 0.6937500238418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.026519775390625,
      "rewards/margins": 0.03708801418542862,
      "rewards/rejected": -0.06361083686351776,
      "step": 10140
    },
    {
      "epoch": 0.7394455979310094,
      "grad_norm": 2.0913968385244144,
      "learning_rate": 4.962916669854651e-07,
      "log_odds_chosen": 1.051428198814392,
      "log_odds_ratio": -0.45625001192092896,
      "logits/chosen": -1.01953125,
      "logits/rejected": -0.932421863079071,
      "logps/chosen": -0.5140625238418579,
      "logps/rejected": -1.138085961341858,
      "loss": 0.7167,
      "nll_loss": 0.6539062261581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.025726318359375,
      "rewards/margins": 0.03118438646197319,
      "rewards/rejected": -0.05686035007238388,
      "step": 10150
    },
    {
      "epoch": 0.7401741157614833,
      "grad_norm": 2.46423169865306,
      "learning_rate": 4.960473688328407e-07,
      "log_odds_chosen": 1.244873046875,
      "log_odds_ratio": -0.3944335877895355,
      "logits/chosen": -1.0107421875,
      "logits/rejected": -0.883593738079071,
      "logps/chosen": -0.5067383050918579,
      "logps/rejected": -1.2615234851837158,
      "loss": 0.7468,
      "nll_loss": 0.697265625,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02533569373190403,
      "rewards/margins": 0.037737272679805756,
      "rewards/rejected": -0.06305541843175888,
      "step": 10160
    },
    {
      "epoch": 0.7409026335919572,
      "grad_norm": 2.1685779509515375,
      "learning_rate": 4.95803431090608e-07,
      "log_odds_chosen": 1.129638671875,
      "log_odds_ratio": -0.46240234375,
      "logits/chosen": -0.998242199420929,
      "logits/rejected": -0.900390625,
      "logps/chosen": -0.5166991949081421,
      "logps/rejected": -1.175195336341858,
      "loss": 0.7436,
      "nll_loss": 0.7201172113418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.025848388671875,
      "rewards/margins": 0.03293304517865181,
      "rewards/rejected": -0.05877075344324112,
      "step": 10170
    },
    {
      "epoch": 0.7416311514224311,
      "grad_norm": 2.213817841118827,
      "learning_rate": 4.955598528734554e-07,
      "log_odds_chosen": 1.0837891101837158,
      "log_odds_ratio": -0.4588378965854645,
      "logits/chosen": -0.990039050579071,
      "logits/rejected": -0.8740234375,
      "logps/chosen": -0.5806640386581421,
      "logps/rejected": -1.2548828125,
      "loss": 0.7403,
      "nll_loss": 0.7069336175918579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02901611290872097,
      "rewards/margins": 0.03372345119714737,
      "rewards/rejected": -0.062744140625,
      "step": 10180
    },
    {
      "epoch": 0.742359669252905,
      "grad_norm": 2.3054089872101327,
      "learning_rate": 4.953166332991126e-07,
      "log_odds_chosen": 1.0634765625,
      "log_odds_ratio": -0.4556640684604645,
      "logits/chosen": -1.034570336341858,
      "logits/rejected": -0.94140625,
      "logps/chosen": -0.571484386920929,
      "logps/rejected": -1.228124976158142,
      "loss": 0.7535,
      "nll_loss": 0.7315429449081421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02855224534869194,
      "rewards/margins": 0.0328216552734375,
      "rewards/rejected": -0.06136474758386612,
      "step": 10190
    },
    {
      "epoch": 0.7430881870833789,
      "grad_norm": 2.381280465174799,
      "learning_rate": 4.950737714883372e-07,
      "log_odds_chosen": 1.163671851158142,
      "log_odds_ratio": -0.42680662870407104,
      "logits/chosen": -1.007226586341858,
      "logits/rejected": -0.890625,
      "logps/chosen": -0.5547851324081421,
      "logps/rejected": -1.2332031726837158,
      "loss": 0.7259,
      "nll_loss": 0.6943359375,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02773437462747097,
      "rewards/margins": 0.03391418606042862,
      "rewards/rejected": -0.06165771558880806,
      "step": 10200
    },
    {
      "epoch": 0.7438167049138528,
      "grad_norm": 1.846435626891483,
      "learning_rate": 4.948312665649023e-07,
      "log_odds_chosen": 1.0245361328125,
      "log_odds_ratio": -0.48979490995407104,
      "logits/chosen": -0.9886718988418579,
      "logits/rejected": -0.916796863079071,
      "logps/chosen": -0.551464855670929,
      "logps/rejected": -1.1513671875,
      "loss": 0.7519,
      "nll_loss": 0.7607421875,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02756347693502903,
      "rewards/margins": 0.03000335767865181,
      "rewards/rejected": -0.05759277194738388,
      "step": 10210
    },
    {
      "epoch": 0.7445452227443267,
      "grad_norm": 2.2392943249775,
      "learning_rate": 4.945891176555817e-07,
      "log_odds_chosen": 1.1167113780975342,
      "log_odds_ratio": -0.42939454317092896,
      "logits/chosen": -1.0017578601837158,
      "logits/rejected": -0.882617175579071,
      "logps/chosen": -0.525097668170929,
      "logps/rejected": -1.1667969226837158,
      "loss": 0.7474,
      "nll_loss": 0.7279297113418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02625427208840847,
      "rewards/margins": 0.032106779515743256,
      "rewards/rejected": -0.05833740159869194,
      "step": 10220
    },
    {
      "epoch": 0.7452737405748006,
      "grad_norm": 2.6108166956564283,
      "learning_rate": 4.943473238901382e-07,
      "log_odds_chosen": 1.1789062023162842,
      "log_odds_ratio": -0.42363280057907104,
      "logits/chosen": -0.988085925579071,
      "logits/rejected": -0.883007824420929,
      "logps/chosen": -0.538378894329071,
      "logps/rejected": -1.2771484851837158,
      "loss": 0.7278,
      "nll_loss": 0.7206054925918579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02689819410443306,
      "rewards/margins": 0.03696288913488388,
      "rewards/rejected": -0.06381835788488388,
      "step": 10230
    },
    {
      "epoch": 0.7460022584052745,
      "grad_norm": 2.3299713812793748,
      "learning_rate": 4.941058844013093e-07,
      "log_odds_chosen": 1.2838866710662842,
      "log_odds_ratio": -0.38093262910842896,
      "logits/chosen": -0.9849609136581421,
      "logits/rejected": -0.8642578125,
      "logps/chosen": -0.49238282442092896,
      "logps/rejected": -1.2169921398162842,
      "loss": 0.7109,
      "nll_loss": 0.66455078125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02463378943502903,
      "rewards/margins": 0.03621215745806694,
      "rewards/rejected": -0.06081543117761612,
      "step": 10240
    },
    {
      "epoch": 0.7467307762357484,
      "grad_norm": 2.293766100353748,
      "learning_rate": 4.938647983247948e-07,
      "log_odds_chosen": 1.095117211341858,
      "log_odds_ratio": -0.45429688692092896,
      "logits/chosen": -1.0027344226837158,
      "logits/rejected": -0.8833984136581421,
      "logps/chosen": -0.5462890863418579,
      "logps/rejected": -1.191015601158142,
      "loss": 0.7427,
      "nll_loss": 0.71337890625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02734985388815403,
      "rewards/margins": 0.032196044921875,
      "rewards/rejected": -0.05954589694738388,
      "step": 10250
    },
    {
      "epoch": 0.7474592940662222,
      "grad_norm": 2.239397281374027,
      "learning_rate": 4.936240647992436e-07,
      "log_odds_chosen": 0.976367175579071,
      "log_odds_ratio": -0.47197264432907104,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.907421886920929,
      "logps/chosen": -0.5411132574081421,
      "logps/rejected": -1.107421875,
      "loss": 0.7245,
      "nll_loss": 0.6470702886581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02707519568502903,
      "rewards/margins": 0.02827758714556694,
      "rewards/rejected": -0.05535888671875,
      "step": 10260
    },
    {
      "epoch": 0.7481878118966961,
      "grad_norm": 2.2761743109865984,
      "learning_rate": 4.933836829662409e-07,
      "log_odds_chosen": 1.22607421875,
      "log_odds_ratio": -0.424072265625,
      "logits/chosen": -1.037109375,
      "logits/rejected": -0.9105468988418579,
      "logps/chosen": -0.5584961175918579,
      "logps/rejected": -1.3037109375,
      "loss": 0.7374,
      "nll_loss": 0.7251952886581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02794799767434597,
      "rewards/margins": 0.03724212571978569,
      "rewards/rejected": -0.06516113132238388,
      "step": 10270
    },
    {
      "epoch": 0.74891632972717,
      "grad_norm": 1.9811358662152423,
      "learning_rate": 4.931436519702948e-07,
      "log_odds_chosen": 1.154882788658142,
      "log_odds_ratio": -0.448486328125,
      "logits/chosen": -0.979296863079071,
      "logits/rejected": -0.8892577886581421,
      "logps/chosen": -0.5150390863418579,
      "logps/rejected": -1.1886718273162842,
      "loss": 0.7418,
      "nll_loss": 0.72021484375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02575073204934597,
      "rewards/margins": 0.03369445726275444,
      "rewards/rejected": -0.05946044996380806,
      "step": 10280
    },
    {
      "epoch": 0.7496448475576439,
      "grad_norm": 2.158982408551252,
      "learning_rate": 4.929039709588244e-07,
      "log_odds_chosen": 1.4767577648162842,
      "log_odds_ratio": -0.37080079317092896,
      "logits/chosen": -0.9896484613418579,
      "logits/rejected": -0.8550781011581421,
      "logps/chosen": -0.539355456829071,
      "logps/rejected": -1.442773461341858,
      "loss": 0.7376,
      "nll_loss": 0.7164062261581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02695312537252903,
      "rewards/margins": 0.04523620754480362,
      "rewards/rejected": -0.07216797024011612,
      "step": 10290
    },
    {
      "epoch": 0.7503733653881178,
      "grad_norm": 2.025103516851444,
      "learning_rate": 4.926646390821466e-07,
      "log_odds_chosen": 1.0435059070587158,
      "log_odds_ratio": -0.504931628704071,
      "logits/chosen": -0.9652343988418579,
      "logits/rejected": -0.884765625,
      "logps/chosen": -0.5455077886581421,
      "logps/rejected": -1.160742163658142,
      "loss": 0.7536,
      "nll_loss": 0.737109363079071,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02727661095559597,
      "rewards/margins": 0.030719757080078125,
      "rewards/rejected": -0.058013916015625,
      "step": 10300
    },
    {
      "epoch": 0.7511018832185917,
      "grad_norm": 3.2774736603285173,
      "learning_rate": 4.924256554934632e-07,
      "log_odds_chosen": 1.0786621570587158,
      "log_odds_ratio": -0.46186524629592896,
      "logits/chosen": -0.9789062738418579,
      "logits/rejected": -0.890625,
      "logps/chosen": -0.5418945550918579,
      "logps/rejected": -1.1652343273162842,
      "loss": 0.7208,
      "nll_loss": 0.650195300579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02711791917681694,
      "rewards/margins": 0.031115341931581497,
      "rewards/rejected": -0.05823974683880806,
      "step": 10310
    },
    {
      "epoch": 0.7518304010490656,
      "grad_norm": 2.1198094077894396,
      "learning_rate": 4.921870193488487e-07,
      "log_odds_chosen": 1.1112549304962158,
      "log_odds_ratio": -0.4629882872104645,
      "logits/chosen": -0.9955078363418579,
      "logits/rejected": -0.9017578363418579,
      "logps/chosen": -0.5526367425918579,
      "logps/rejected": -1.229882836341858,
      "loss": 0.7131,
      "nll_loss": 0.708691418170929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.027618408203125,
      "rewards/margins": 0.033872224390506744,
      "rewards/rejected": -0.06148071214556694,
      "step": 10320
    },
    {
      "epoch": 0.7525589188795396,
      "grad_norm": 2.056516543221994,
      "learning_rate": 4.919487298072376e-07,
      "log_odds_chosen": 1.0638427734375,
      "log_odds_ratio": -0.46113282442092896,
      "logits/chosen": -0.967578113079071,
      "logits/rejected": -0.8623046875,
      "logps/chosen": -0.5672851800918579,
      "logps/rejected": -1.1980469226837158,
      "loss": 0.7189,
      "nll_loss": 0.763671875,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02838134765625,
      "rewards/margins": 0.03155517578125,
      "rewards/rejected": -0.05991210788488388,
      "step": 10330
    },
    {
      "epoch": 0.7532874367100135,
      "grad_norm": 2.3790786480611743,
      "learning_rate": 4.917107860304123e-07,
      "log_odds_chosen": 0.9585937261581421,
      "log_odds_ratio": -0.5062500238418579,
      "logits/chosen": -1.031835913658142,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.5624023675918579,
      "logps/rejected": -1.1337890625,
      "loss": 0.7277,
      "nll_loss": 0.720019519329071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02809448167681694,
      "rewards/margins": 0.02856750413775444,
      "rewards/rejected": -0.05666504055261612,
      "step": 10340
    },
    {
      "epoch": 0.7540159545404874,
      "grad_norm": 2.371624688667264,
      "learning_rate": 4.914731871829905e-07,
      "log_odds_chosen": 1.028955101966858,
      "log_odds_ratio": -0.5042480230331421,
      "logits/chosen": -0.957226574420929,
      "logits/rejected": -0.8687499761581421,
      "logps/chosen": -0.5401366949081421,
      "logps/rejected": -1.120507836341858,
      "loss": 0.726,
      "nll_loss": 0.672070324420929,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02700195275247097,
      "rewards/margins": 0.02901916578412056,
      "rewards/rejected": -0.05601806566119194,
      "step": 10350
    },
    {
      "epoch": 0.7547444723709613,
      "grad_norm": 2.316664410764534,
      "learning_rate": 4.912359324324122e-07,
      "log_odds_chosen": 1.091210961341858,
      "log_odds_ratio": -0.45634764432907104,
      "logits/chosen": -1.007226586341858,
      "logits/rejected": -0.9056640863418579,
      "logps/chosen": -0.518261730670929,
      "logps/rejected": -1.1318359375,
      "loss": 0.737,
      "nll_loss": 0.714648425579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02594604529440403,
      "rewards/margins": 0.0306396484375,
      "rewards/rejected": -0.0565185546875,
      "step": 10360
    },
    {
      "epoch": 0.7554729902014352,
      "grad_norm": 2.230796789367601,
      "learning_rate": 4.909990209489283e-07,
      "log_odds_chosen": 1.1951172351837158,
      "log_odds_ratio": -0.3949218690395355,
      "logits/chosen": -1.013281226158142,
      "logits/rejected": -0.921875,
      "logps/chosen": -0.5157226324081421,
      "logps/rejected": -1.225000023841858,
      "loss": 0.7505,
      "nll_loss": 0.7181640863418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.025787353515625,
      "rewards/margins": 0.035427093505859375,
      "rewards/rejected": -0.06126708909869194,
      "step": 10370
    },
    {
      "epoch": 0.7562015080319091,
      "grad_norm": 2.4446421829503984,
      "learning_rate": 4.90762451905589e-07,
      "log_odds_chosen": 0.977709949016571,
      "log_odds_ratio": -0.47998046875,
      "logits/chosen": -0.970898449420929,
      "logits/rejected": -0.871289074420929,
      "logps/chosen": -0.5625976324081421,
      "logps/rejected": -1.120703101158142,
      "loss": 0.7292,
      "nll_loss": 0.7491210699081421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02812499925494194,
      "rewards/margins": 0.027858734130859375,
      "rewards/rejected": -0.05604248121380806,
      "step": 10380
    },
    {
      "epoch": 0.756930025862383,
      "grad_norm": 2.302861646557013,
      "learning_rate": 4.905262244782293e-07,
      "log_odds_chosen": 0.986523449420929,
      "log_odds_ratio": -0.46708983182907104,
      "logits/chosen": -0.9722656011581421,
      "logits/rejected": -0.8646484613418579,
      "logps/chosen": -0.53662109375,
      "logps/rejected": -1.0973632335662842,
      "loss": 0.7444,
      "nll_loss": 0.7318359613418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02680053748190403,
      "rewards/margins": 0.02806549146771431,
      "rewards/rejected": -0.05488281324505806,
      "step": 10390
    },
    {
      "epoch": 0.7576585436928569,
      "grad_norm": 2.1137884050567743,
      "learning_rate": 4.902903378454602e-07,
      "log_odds_chosen": 0.9530273675918579,
      "log_odds_ratio": -0.47563475370407104,
      "logits/chosen": -0.9755859375,
      "logits/rejected": -0.8726562261581421,
      "logps/chosen": -0.542187511920929,
      "logps/rejected": -1.122656226158142,
      "loss": 0.7446,
      "nll_loss": 0.695117175579071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.027099609375,
      "rewards/margins": 0.02901000902056694,
      "rewards/rejected": -0.05611572414636612,
      "step": 10400
    },
    {
      "epoch": 0.7583870615233308,
      "grad_norm": 2.1091090814264004,
      "learning_rate": 4.900547911886536e-07,
      "log_odds_chosen": 1.124755859375,
      "log_odds_ratio": -0.46406251192092896,
      "logits/chosen": -1.000390648841858,
      "logits/rejected": -0.8916015625,
      "logps/chosen": -0.5751953125,
      "logps/rejected": -1.296289086341858,
      "loss": 0.7376,
      "nll_loss": 0.7098633050918579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02877197228372097,
      "rewards/margins": 0.03608398512005806,
      "rewards/rejected": -0.06485595554113388,
      "step": 10410
    },
    {
      "epoch": 0.7591155793538047,
      "grad_norm": 2.5676980223676775,
      "learning_rate": 4.898195836919327e-07,
      "log_odds_chosen": 0.8645995855331421,
      "log_odds_ratio": -0.521679699420929,
      "logits/chosen": -1.006250023841858,
      "logits/rejected": -0.9091796875,
      "logps/chosen": -0.57861328125,
      "logps/rejected": -1.1064453125,
      "loss": 0.7407,
      "nll_loss": 0.722460925579071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02892456017434597,
      "rewards/margins": 0.026380157098174095,
      "rewards/rejected": -0.05534057691693306,
      "step": 10420
    },
    {
      "epoch": 0.7598440971842786,
      "grad_norm": 2.103819996423241,
      "learning_rate": 4.895847145421587e-07,
      "log_odds_chosen": 1.154687523841858,
      "log_odds_ratio": -0.4403320252895355,
      "logits/chosen": -0.937304675579071,
      "logits/rejected": -0.8589843511581421,
      "logps/chosen": -0.5430663824081421,
      "logps/rejected": -1.192773461341858,
      "loss": 0.727,
      "nll_loss": 0.7119140625,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02711181715130806,
      "rewards/margins": 0.03253173828125,
      "rewards/rejected": -0.05966796725988388,
      "step": 10430
    },
    {
      "epoch": 0.7605726150147525,
      "grad_norm": 1.8958157429943308,
      "learning_rate": 4.893501829289196e-07,
      "log_odds_chosen": 0.9852539300918579,
      "log_odds_ratio": -0.47382813692092896,
      "logits/chosen": -1.016210913658142,
      "logits/rejected": -0.912109375,
      "logps/chosen": -0.56640625,
      "logps/rejected": -1.176367163658142,
      "loss": 0.7201,
      "nll_loss": 0.712695300579071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02833252027630806,
      "rewards/margins": 0.030443191528320312,
      "rewards/rejected": -0.05878906324505806,
      "step": 10440
    },
    {
      "epoch": 0.7613011328452264,
      "grad_norm": 2.228115721410135,
      "learning_rate": 4.891159880445185e-07,
      "log_odds_chosen": 1.2482421398162842,
      "log_odds_ratio": -0.4151855409145355,
      "logits/chosen": -1.018164038658142,
      "logits/rejected": -0.8890625238418579,
      "logps/chosen": -0.537109375,
      "logps/rejected": -1.3337891101837158,
      "loss": 0.7351,
      "nll_loss": 0.7323242425918579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02685546875,
      "rewards/margins": 0.03984374925494194,
      "rewards/rejected": -0.06676025688648224,
      "step": 10450
    },
    {
      "epoch": 0.7620296506757003,
      "grad_norm": 2.306562102908152,
      "learning_rate": 4.888821290839617e-07,
      "log_odds_chosen": 1.178857445716858,
      "log_odds_ratio": -0.4308105409145355,
      "logits/chosen": -0.954296886920929,
      "logits/rejected": -0.853710949420929,
      "logps/chosen": -0.5103515386581421,
      "logps/rejected": -1.2091796398162842,
      "loss": 0.7295,
      "nll_loss": 0.723828136920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02550048753619194,
      "rewards/margins": 0.03493347018957138,
      "rewards/rejected": -0.0604248046875,
      "step": 10460
    },
    {
      "epoch": 0.7627581685061742,
      "grad_norm": 2.349632525027852,
      "learning_rate": 4.886486052449469e-07,
      "log_odds_chosen": 1.105932593345642,
      "log_odds_ratio": -0.44580078125,
      "logits/chosen": -1.0109374523162842,
      "logits/rejected": -0.912304699420929,
      "logps/chosen": -0.5423828363418579,
      "logps/rejected": -1.1628906726837158,
      "loss": 0.7433,
      "nll_loss": 0.724804699420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02711791917681694,
      "rewards/margins": 0.03102569654583931,
      "rewards/rejected": -0.05816040188074112,
      "step": 10470
    },
    {
      "epoch": 0.7634866863366481,
      "grad_norm": 2.405739974161283,
      "learning_rate": 4.884154157278522e-07,
      "log_odds_chosen": 1.187744140625,
      "log_odds_ratio": -0.4341796934604645,
      "logits/chosen": -0.9847656488418579,
      "logits/rejected": -0.898242175579071,
      "logps/chosen": -0.5396484136581421,
      "logps/rejected": -1.2082030773162842,
      "loss": 0.7237,
      "nll_loss": 0.676074206829071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02696533128619194,
      "rewards/margins": 0.03342590481042862,
      "rewards/rejected": -0.0604248046875,
      "step": 10480
    },
    {
      "epoch": 0.764215204167122,
      "grad_norm": 1.999179465606562,
      "learning_rate": 4.881825597357242e-07,
      "log_odds_chosen": 0.9983154535293579,
      "log_odds_ratio": -0.4503417909145355,
      "logits/chosen": -1.0242187976837158,
      "logits/rejected": -0.875195324420929,
      "logps/chosen": -0.525097668170929,
      "logps/rejected": -1.102929711341858,
      "loss": 0.7412,
      "nll_loss": 0.695605456829071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02623901329934597,
      "rewards/margins": 0.02884674072265625,
      "rewards/rejected": -0.05512695387005806,
      "step": 10490
    },
    {
      "epoch": 0.7649437219975959,
      "grad_norm": 2.3040795678662755,
      "learning_rate": 4.879500364742667e-07,
      "log_odds_chosen": 0.8088134527206421,
      "log_odds_ratio": -0.536376953125,
      "logits/chosen": -0.9632812738418579,
      "logits/rejected": -0.8833984136581421,
      "logps/chosen": -0.539746105670929,
      "logps/rejected": -0.985156238079071,
      "loss": 0.7326,
      "nll_loss": 0.7694336175918579,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.02698974683880806,
      "rewards/margins": 0.02222442626953125,
      "rewards/rejected": -0.04920043796300888,
      "step": 10500
    },
    {
      "epoch": 0.7656722398280698,
      "grad_norm": 2.195709629162305,
      "learning_rate": 4.877178451518289e-07,
      "log_odds_chosen": 1.079858422279358,
      "log_odds_ratio": -0.4581542909145355,
      "logits/chosen": -1.022070288658142,
      "logits/rejected": -0.887890636920929,
      "logps/chosen": -0.556445300579071,
      "logps/rejected": -1.2033202648162842,
      "loss": 0.7404,
      "nll_loss": 0.7484375238418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02781982347369194,
      "rewards/margins": 0.0323486328125,
      "rewards/rejected": -0.0601806640625,
      "step": 10510
    },
    {
      "epoch": 0.7664007576585437,
      "grad_norm": 2.3053278888166324,
      "learning_rate": 4.874859849793949e-07,
      "log_odds_chosen": 1.1309082508087158,
      "log_odds_ratio": -0.47900390625,
      "logits/chosen": -1.0068359375,
      "logits/rejected": -0.861132800579071,
      "logps/chosen": -0.56005859375,
      "logps/rejected": -1.246484398841858,
      "loss": 0.7235,
      "nll_loss": 0.726757824420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02802734449505806,
      "rewards/margins": 0.034293364733457565,
      "rewards/rejected": -0.06231689453125,
      "step": 10520
    },
    {
      "epoch": 0.7671292754890175,
      "grad_norm": 2.0382519700322757,
      "learning_rate": 4.872544551705718e-07,
      "log_odds_chosen": 1.1531250476837158,
      "log_odds_ratio": -0.4442382752895355,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.9443359375,
      "logps/chosen": -0.5184570550918579,
      "logps/rejected": -1.1974608898162842,
      "loss": 0.7227,
      "nll_loss": 0.7010742425918579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02591552771627903,
      "rewards/margins": 0.033945273607969284,
      "rewards/rejected": -0.05983886867761612,
      "step": 10530
    },
    {
      "epoch": 0.7678577933194914,
      "grad_norm": 2.624052561421337,
      "learning_rate": 4.870232549415787e-07,
      "log_odds_chosen": 0.991943359375,
      "log_odds_ratio": -0.4808593690395355,
      "logits/chosen": -1.0427734851837158,
      "logits/rejected": -0.931445300579071,
      "logps/chosen": -0.521484375,
      "logps/rejected": -1.099023461341858,
      "loss": 0.7419,
      "nll_loss": 0.73193359375,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02606201171875,
      "rewards/margins": 0.02887267991900444,
      "rewards/rejected": -0.05495605617761612,
      "step": 10540
    },
    {
      "epoch": 0.7685863111499653,
      "grad_norm": 2.2209739095535084,
      "learning_rate": 4.867923835112355e-07,
      "log_odds_chosen": 0.8505859375,
      "log_odds_ratio": -0.521777331829071,
      "logits/chosen": -1.0499999523162842,
      "logits/rejected": -0.911328136920929,
      "logps/chosen": -0.5640624761581421,
      "logps/rejected": -1.044335961341858,
      "loss": 0.7335,
      "nll_loss": 0.700976550579071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02821655198931694,
      "rewards/margins": 0.02401428297162056,
      "rewards/rejected": -0.05217285081744194,
      "step": 10550
    },
    {
      "epoch": 0.7693148289804393,
      "grad_norm": 2.281226638323642,
      "learning_rate": 4.865618401009519e-07,
      "log_odds_chosen": 1.116943359375,
      "log_odds_ratio": -0.46293944120407104,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.5389648675918579,
      "logps/rejected": -1.214453101158142,
      "loss": 0.738,
      "nll_loss": 0.7728515863418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02693481370806694,
      "rewards/margins": 0.03381957858800888,
      "rewards/rejected": -0.06076660007238388,
      "step": 10560
    },
    {
      "epoch": 0.7700433468109132,
      "grad_norm": 2.419156595132921,
      "learning_rate": 4.863316239347163e-07,
      "log_odds_chosen": 1.2465331554412842,
      "log_odds_ratio": -0.4208007752895355,
      "logits/chosen": -0.996874988079071,
      "logits/rejected": -0.8623046875,
      "logps/chosen": -0.576855480670929,
      "logps/rejected": -1.3214843273162842,
      "loss": 0.7291,
      "nll_loss": 0.7503906488418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.028839111328125,
      "rewards/margins": 0.037206269800662994,
      "rewards/rejected": -0.06608887016773224,
      "step": 10570
    },
    {
      "epoch": 0.7707718646413871,
      "grad_norm": 2.1258593589496515,
      "learning_rate": 4.861017342390847e-07,
      "log_odds_chosen": 1.320898413658142,
      "log_odds_ratio": -0.38066405057907104,
      "logits/chosen": -0.992382824420929,
      "logits/rejected": -0.882617175579071,
      "logps/chosen": -0.5381835699081421,
      "logps/rejected": -1.3273437023162842,
      "loss": 0.7438,
      "nll_loss": 0.7471679449081421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.02689819410443306,
      "rewards/margins": 0.03948974609375,
      "rewards/rejected": -0.06640625,
      "step": 10580
    },
    {
      "epoch": 0.771500382471861,
      "grad_norm": 2.5091790130108254,
      "learning_rate": 4.858721702431704e-07,
      "log_odds_chosen": 1.0728271007537842,
      "log_odds_ratio": -0.43876951932907104,
      "logits/chosen": -1.0197265148162842,
      "logits/rejected": -0.9173828363418579,
      "logps/chosen": -0.522753894329071,
      "logps/rejected": -1.1259765625,
      "loss": 0.7309,
      "nll_loss": 0.677929699420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02613525465130806,
      "rewards/margins": 0.03018341027200222,
      "rewards/rejected": -0.05632324144244194,
      "step": 10590
    },
    {
      "epoch": 0.7722289003023349,
      "grad_norm": 2.0766838259806732,
      "learning_rate": 4.856429311786321e-07,
      "log_odds_chosen": 1.190820336341858,
      "log_odds_ratio": -0.44340819120407104,
      "logits/chosen": -0.9769531488418579,
      "logits/rejected": -0.873242199420929,
      "logps/chosen": -0.5126953125,
      "logps/rejected": -1.228124976158142,
      "loss": 0.7285,
      "nll_loss": 0.7012695074081421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02561645582318306,
      "rewards/margins": 0.0357666015625,
      "rewards/rejected": -0.06136474758386612,
      "step": 10600
    },
    {
      "epoch": 0.7729574181328088,
      "grad_norm": 1.9385211371971913,
      "learning_rate": 4.854140162796642e-07,
      "log_odds_chosen": 1.2838013172149658,
      "log_odds_ratio": -0.41889649629592896,
      "logits/chosen": -1.0119140148162842,
      "logits/rejected": -0.876953125,
      "logps/chosen": -0.5550781488418579,
      "logps/rejected": -1.323632836341858,
      "loss": 0.7094,
      "nll_loss": 0.701953113079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02777099609375,
      "rewards/margins": 0.03842620924115181,
      "rewards/rejected": -0.06619872897863388,
      "step": 10610
    },
    {
      "epoch": 0.7736859359632827,
      "grad_norm": 2.189273704253368,
      "learning_rate": 4.85185424782985e-07,
      "log_odds_chosen": 1.0564453601837158,
      "log_odds_ratio": -0.4658203125,
      "logits/chosen": -1.0144531726837158,
      "logits/rejected": -0.8763672113418579,
      "logps/chosen": -0.5506836175918579,
      "logps/rejected": -1.1953125,
      "loss": 0.7303,
      "nll_loss": 0.7115234136581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.027557373046875,
      "rewards/margins": 0.03222503513097763,
      "rewards/rejected": -0.059844970703125,
      "step": 10620
    },
    {
      "epoch": 0.7744144537937566,
      "grad_norm": 2.546336524709832,
      "learning_rate": 4.849571559278272e-07,
      "log_odds_chosen": 0.9822632074356079,
      "log_odds_ratio": -0.4730468690395355,
      "logits/chosen": -0.9794921875,
      "logits/rejected": -0.8662109375,
      "logps/chosen": -0.528124988079071,
      "logps/rejected": -1.09375,
      "loss": 0.7168,
      "nll_loss": 0.701953113079071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02640991285443306,
      "rewards/margins": 0.028281021863222122,
      "rewards/rejected": -0.05468139797449112,
      "step": 10630
    },
    {
      "epoch": 0.7751429716242305,
      "grad_norm": 2.0076759517459517,
      "learning_rate": 4.847292089559258e-07,
      "log_odds_chosen": 0.881604015827179,
      "log_odds_ratio": -0.5240234136581421,
      "logits/chosen": -0.984570324420929,
      "logits/rejected": -0.8648437261581421,
      "logps/chosen": -0.578125,
      "logps/rejected": -1.1218750476837158,
      "loss": 0.737,
      "nll_loss": 0.7808593511581421,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.02890625037252903,
      "rewards/margins": 0.027213668450713158,
      "rewards/rejected": -0.05610961839556694,
      "step": 10640
    },
    {
      "epoch": 0.7758714894547044,
      "grad_norm": 2.7964076756722753,
      "learning_rate": 4.845015831115092e-07,
      "log_odds_chosen": 1.279394507408142,
      "log_odds_ratio": -0.43867188692092896,
      "logits/chosen": -0.999218761920929,
      "logits/rejected": -0.900195300579071,
      "logps/chosen": -0.548095703125,
      "logps/rejected": -1.328125,
      "loss": 0.7602,
      "nll_loss": 0.772656261920929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02739868126809597,
      "rewards/margins": 0.03898925706744194,
      "rewards/rejected": -0.06644286960363388,
      "step": 10650
    },
    {
      "epoch": 0.7766000072851783,
      "grad_norm": 2.382130842044504,
      "learning_rate": 4.842742776412873e-07,
      "log_odds_chosen": 1.365625023841858,
      "log_odds_ratio": -0.38920897245407104,
      "logits/chosen": -0.9673827886581421,
      "logits/rejected": -0.8685547113418579,
      "logps/chosen": -0.559277355670929,
      "logps/rejected": -1.399023413658142,
      "loss": 0.7295,
      "nll_loss": 0.7655273675918579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.02797241136431694,
      "rewards/margins": 0.04198303073644638,
      "rewards/rejected": -0.06995239108800888,
      "step": 10660
    },
    {
      "epoch": 0.7773285251156522,
      "grad_norm": 2.364300038674168,
      "learning_rate": 4.840472917944416e-07,
      "log_odds_chosen": 1.295312523841858,
      "log_odds_ratio": -0.39208984375,
      "logits/chosen": -0.974804699420929,
      "logits/rejected": -0.885937511920929,
      "logps/chosen": -0.5113281011581421,
      "logps/rejected": -1.2746093273162842,
      "loss": 0.7092,
      "nll_loss": 0.691601574420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02556762658059597,
      "rewards/margins": 0.03812255710363388,
      "rewards/rejected": -0.063690185546875,
      "step": 10670
    },
    {
      "epoch": 0.7780570429461261,
      "grad_norm": 2.253474721267822,
      "learning_rate": 4.838206248226148e-07,
      "log_odds_chosen": 1.1687500476837158,
      "log_odds_ratio": -0.45087891817092896,
      "logits/chosen": -0.923046886920929,
      "logits/rejected": -0.825390636920929,
      "logps/chosen": -0.5311523675918579,
      "logps/rejected": -1.2644531726837158,
      "loss": 0.7089,
      "nll_loss": 0.672656238079071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02656860277056694,
      "rewards/margins": 0.03667907789349556,
      "rewards/rejected": -0.063232421875,
      "step": 10680
    },
    {
      "epoch": 0.7787855607766,
      "grad_norm": 2.383597304111258,
      "learning_rate": 4.835942759799002e-07,
      "log_odds_chosen": 1.058984398841858,
      "log_odds_ratio": -0.46044921875,
      "logits/chosen": -0.9673827886581421,
      "logits/rejected": -0.845703125,
      "logps/chosen": -0.5677734613418579,
      "logps/rejected": -1.2177734375,
      "loss": 0.7216,
      "nll_loss": 0.744140625,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02839355543255806,
      "rewards/margins": 0.03249816969037056,
      "rewards/rejected": -0.06090087816119194,
      "step": 10690
    },
    {
      "epoch": 0.7795140786070739,
      "grad_norm": 2.655087009121431,
      "learning_rate": 4.833682445228318e-07,
      "log_odds_chosen": 1.147851586341858,
      "log_odds_ratio": -0.45683592557907104,
      "logits/chosen": -0.997265636920929,
      "logits/rejected": -0.897265613079071,
      "logps/chosen": -0.5228515863418579,
      "logps/rejected": -1.2521483898162842,
      "loss": 0.7386,
      "nll_loss": 0.688671886920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.026123046875,
      "rewards/margins": 0.03644103929400444,
      "rewards/rejected": -0.06260986626148224,
      "step": 10700
    },
    {
      "epoch": 0.7802425964375478,
      "grad_norm": 2.2572290248592184,
      "learning_rate": 4.831425297103737e-07,
      "log_odds_chosen": 0.9642578363418579,
      "log_odds_ratio": -0.5235351324081421,
      "logits/chosen": -0.9585937261581421,
      "logits/rejected": -0.8521484136581421,
      "logps/chosen": -0.5667968988418579,
      "logps/rejected": -1.107421875,
      "loss": 0.7282,
      "nll_loss": 0.7430664300918579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.028350830078125,
      "rewards/margins": 0.027043914422392845,
      "rewards/rejected": -0.05535278469324112,
      "step": 10710
    },
    {
      "epoch": 0.7809711142680217,
      "grad_norm": 2.336352837031554,
      "learning_rate": 4.829171308039099e-07,
      "log_odds_chosen": 1.219335913658142,
      "log_odds_ratio": -0.40083009004592896,
      "logits/chosen": -0.9779297113418579,
      "logits/rejected": -0.869140625,
      "logps/chosen": -0.4913085997104645,
      "logps/rejected": -1.190820336341858,
      "loss": 0.7298,
      "nll_loss": 0.6904296875,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.02453918382525444,
      "rewards/margins": 0.0349884033203125,
      "rewards/rejected": -0.05950927734375,
      "step": 10720
    },
    {
      "epoch": 0.7816996320984956,
      "grad_norm": 2.2616404229055456,
      "learning_rate": 4.826920470672344e-07,
      "log_odds_chosen": 1.04443359375,
      "log_odds_ratio": -0.45087891817092896,
      "logits/chosen": -1.0037109851837158,
      "logits/rejected": -0.865429699420929,
      "logps/chosen": -0.5484374761581421,
      "logps/rejected": -1.188085913658142,
      "loss": 0.7326,
      "nll_loss": 0.7158203125,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02742309495806694,
      "rewards/margins": 0.03194732591509819,
      "rewards/rejected": -0.059326171875,
      "step": 10730
    },
    {
      "epoch": 0.7824281499289695,
      "grad_norm": 2.32096523825661,
      "learning_rate": 4.824672777665406e-07,
      "log_odds_chosen": 1.06201171875,
      "log_odds_ratio": -0.4698730409145355,
      "logits/chosen": -0.965624988079071,
      "logits/rejected": -0.8544921875,
      "logps/chosen": -0.545703113079071,
      "logps/rejected": -1.137109398841858,
      "loss": 0.7365,
      "nll_loss": 0.6553710699081421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02727661095559597,
      "rewards/margins": 0.02959594689309597,
      "rewards/rejected": -0.056884765625,
      "step": 10740
    },
    {
      "epoch": 0.7831566677594434,
      "grad_norm": 2.5214740820301333,
      "learning_rate": 4.822428221704122e-07,
      "log_odds_chosen": 0.9988769292831421,
      "log_odds_ratio": -0.5459960699081421,
      "logits/chosen": -0.9609375,
      "logits/rejected": -0.8583984375,
      "logps/chosen": -0.5908203125,
      "logps/rejected": -1.242773413658142,
      "loss": 0.7528,
      "nll_loss": 0.758984386920929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02953491173684597,
      "rewards/margins": 0.03266143798828125,
      "rewards/rejected": -0.06214599683880806,
      "step": 10750
    },
    {
      "epoch": 0.7838851855899173,
      "grad_norm": 2.970152438803536,
      "learning_rate": 4.820186795498119e-07,
      "log_odds_chosen": 1.0695312023162842,
      "log_odds_ratio": -0.45561522245407104,
      "logits/chosen": -0.96875,
      "logits/rejected": -0.862109363079071,
      "logps/chosen": -0.538378894329071,
      "logps/rejected": -1.1505858898162842,
      "loss": 0.7041,
      "nll_loss": 0.703808605670929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02691040001809597,
      "rewards/margins": 0.03066101111471653,
      "rewards/rejected": -0.05754394456744194,
      "step": 10760
    },
    {
      "epoch": 0.7846137034203912,
      "grad_norm": 2.529401278352736,
      "learning_rate": 4.817948491780728e-07,
      "log_odds_chosen": 1.338476538658142,
      "log_odds_ratio": -0.39375001192092896,
      "logits/chosen": -1.03515625,
      "logits/rejected": -0.9027343988418579,
      "logps/chosen": -0.5425781011581421,
      "logps/rejected": -1.3681640625,
      "loss": 0.7394,
      "nll_loss": 0.7374023199081421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.02714233472943306,
      "rewards/margins": 0.04128418117761612,
      "rewards/rejected": -0.06842041015625,
      "step": 10770
    },
    {
      "epoch": 0.7853422212508652,
      "grad_norm": 2.2895188558075366,
      "learning_rate": 4.815713303308873e-07,
      "log_odds_chosen": 1.174560546875,
      "log_odds_ratio": -0.44780272245407104,
      "logits/chosen": -0.967578113079071,
      "logits/rejected": -0.8662109375,
      "logps/chosen": -0.545117199420929,
      "logps/rejected": -1.2761719226837158,
      "loss": 0.7023,
      "nll_loss": 0.733691394329071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02725830115377903,
      "rewards/margins": 0.036580659449100494,
      "rewards/rejected": -0.06380005180835724,
      "step": 10780
    },
    {
      "epoch": 0.7860707390813391,
      "grad_norm": 2.4068816282923633,
      "learning_rate": 4.81348122286298e-07,
      "log_odds_chosen": 1.220117211341858,
      "log_odds_ratio": -0.3944335877895355,
      "logits/chosen": -1.015234351158142,
      "logits/rejected": -0.907421886920929,
      "logps/chosen": -0.513378918170929,
      "logps/rejected": -1.213476538658142,
      "loss": 0.7208,
      "nll_loss": 0.6748046875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0256805419921875,
      "rewards/margins": 0.035003662109375,
      "rewards/rejected": -0.06072998046875,
      "step": 10790
    },
    {
      "epoch": 0.786799256911813,
      "grad_norm": 2.317523845384308,
      "learning_rate": 4.811252243246882e-07,
      "log_odds_chosen": 1.339941382408142,
      "log_odds_ratio": -0.4051269590854645,
      "logits/chosen": -1.004492163658142,
      "logits/rejected": -0.892382800579071,
      "logps/chosen": -0.5538085699081421,
      "logps/rejected": -1.3916015625,
      "loss": 0.7191,
      "nll_loss": 0.7696288824081421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02768554724752903,
      "rewards/margins": 0.04193420335650444,
      "rewards/rejected": -0.06965331733226776,
      "step": 10800
    },
    {
      "epoch": 0.7875277747422869,
      "grad_norm": 2.3203975295467054,
      "learning_rate": 4.809026357287708e-07,
      "log_odds_chosen": 1.0090820789337158,
      "log_odds_ratio": -0.49150389432907104,
      "logits/chosen": -0.9710937738418579,
      "logits/rejected": -0.8882812261581421,
      "logps/chosen": -0.5333007574081421,
      "logps/rejected": -1.130273461341858,
      "loss": 0.7296,
      "nll_loss": 0.692187488079071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02667236328125,
      "rewards/margins": 0.02977600134909153,
      "rewards/rejected": -0.05650634691119194,
      "step": 10810
    },
    {
      "epoch": 0.7882562925727608,
      "grad_norm": 2.5288835602651427,
      "learning_rate": 4.806803557835802e-07,
      "log_odds_chosen": 1.034033179283142,
      "log_odds_ratio": -0.45917969942092896,
      "logits/chosen": -1.0224609375,
      "logits/rejected": -0.8902343511581421,
      "logps/chosen": -0.5477539300918579,
      "logps/rejected": -1.1775391101837158,
      "loss": 0.7201,
      "nll_loss": 0.693164050579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02739868126809597,
      "rewards/margins": 0.031423188745975494,
      "rewards/rejected": -0.058807373046875,
      "step": 10820
    },
    {
      "epoch": 0.7889848104032346,
      "grad_norm": 2.8643879677490554,
      "learning_rate": 4.804583837764616e-07,
      "log_odds_chosen": 1.377832055091858,
      "log_odds_ratio": -0.36962890625,
      "logits/chosen": -1.000585913658142,
      "logits/rejected": -0.845507800579071,
      "logps/chosen": -0.521679699420929,
      "logps/rejected": -1.37109375,
      "loss": 0.7193,
      "nll_loss": 0.70849609375,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.02606811560690403,
      "rewards/margins": 0.04253845289349556,
      "rewards/rejected": -0.06861571967601776,
      "step": 10830
    },
    {
      "epoch": 0.7897133282337085,
      "grad_norm": 2.474541756098544,
      "learning_rate": 4.802367189970616e-07,
      "log_odds_chosen": 1.3088867664337158,
      "log_odds_ratio": -0.39155274629592896,
      "logits/chosen": -1.0037109851837158,
      "logits/rejected": -0.886523425579071,
      "logps/chosen": -0.5067383050918579,
      "logps/rejected": -1.2976562976837158,
      "loss": 0.7094,
      "nll_loss": 0.6861327886581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02532958984375,
      "rewards/margins": 0.03955078125,
      "rewards/rejected": -0.06488037109375,
      "step": 10840
    },
    {
      "epoch": 0.7904418460641824,
      "grad_norm": 2.586586249750705,
      "learning_rate": 4.800153607373193e-07,
      "log_odds_chosen": 1.507714867591858,
      "log_odds_ratio": -0.3415283262729645,
      "logits/chosen": -1.0001952648162842,
      "logits/rejected": -0.896679699420929,
      "logps/chosen": -0.48466795682907104,
      "logps/rejected": -1.3644530773162842,
      "loss": 0.7328,
      "nll_loss": 0.64404296875,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.02423095703125,
      "rewards/margins": 0.043914794921875,
      "rewards/rejected": -0.06821288913488388,
      "step": 10850
    },
    {
      "epoch": 0.7911703638946563,
      "grad_norm": 2.4323406836130093,
      "learning_rate": 4.797943082914559e-07,
      "log_odds_chosen": 1.448828101158142,
      "log_odds_ratio": -0.36909180879592896,
      "logits/chosen": -0.9927734136581421,
      "logits/rejected": -0.8970702886581421,
      "logps/chosen": -0.49853515625,
      "logps/rejected": -1.32421875,
      "loss": 0.7033,
      "nll_loss": 0.6200195550918579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.02490844763815403,
      "rewards/margins": 0.04134521633386612,
      "rewards/rejected": -0.06627197563648224,
      "step": 10860
    },
    {
      "epoch": 0.7918988817251302,
      "grad_norm": 2.1917811335753967,
      "learning_rate": 4.795735609559657e-07,
      "log_odds_chosen": 0.941113293170929,
      "log_odds_ratio": -0.46855467557907104,
      "logits/chosen": -0.9869140386581421,
      "logits/rejected": -0.8882812261581421,
      "logps/chosen": -0.53173828125,
      "logps/rejected": -1.1124999523162842,
      "loss": 0.7349,
      "nll_loss": 0.6982421875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02657775953412056,
      "rewards/margins": 0.02908630296587944,
      "rewards/rejected": -0.05563964694738388,
      "step": 10870
    },
    {
      "epoch": 0.7926273995556041,
      "grad_norm": 2.1855228211344166,
      "learning_rate": 4.793531180296065e-07,
      "log_odds_chosen": 1.27294921875,
      "log_odds_ratio": -0.4052734375,
      "logits/chosen": -1.0,
      "logits/rejected": -0.840039074420929,
      "logps/chosen": -0.5186523199081421,
      "logps/rejected": -1.31640625,
      "loss": 0.7179,
      "nll_loss": 0.7406250238418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02592163160443306,
      "rewards/margins": 0.0399169921875,
      "rewards/rejected": -0.06586913764476776,
      "step": 10880
    },
    {
      "epoch": 0.793355917386078,
      "grad_norm": 6.43868440664203,
      "learning_rate": 4.791329788133909e-07,
      "log_odds_chosen": 1.0060546398162842,
      "log_odds_ratio": -0.5040038824081421,
      "logits/chosen": -0.972460925579071,
      "logits/rejected": -0.8746093511581421,
      "logps/chosen": -0.549511730670929,
      "logps/rejected": -1.1808593273162842,
      "loss": 0.7505,
      "nll_loss": 0.706250011920929,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02747192420065403,
      "rewards/margins": 0.03162231296300888,
      "rewards/rejected": -0.05910034105181694,
      "step": 10890
    },
    {
      "epoch": 0.7940844352165519,
      "grad_norm": 1.9526265216109124,
      "learning_rate": 4.789131426105758e-07,
      "log_odds_chosen": 1.2490234375,
      "log_odds_ratio": -0.41259765625,
      "logits/chosen": -0.990429699420929,
      "logits/rejected": -0.873242199420929,
      "logps/chosen": -0.5550781488418579,
      "logps/rejected": -1.3429687023162842,
      "loss": 0.7449,
      "nll_loss": 0.7733398675918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02775268629193306,
      "rewards/margins": 0.03939209133386612,
      "rewards/rejected": -0.06715087592601776,
      "step": 10900
    },
    {
      "epoch": 0.7948129530470258,
      "grad_norm": 2.5506363211601,
      "learning_rate": 4.786936087266542e-07,
      "log_odds_chosen": 1.1552734375,
      "log_odds_ratio": -0.4281249940395355,
      "logits/chosen": -0.9652343988418579,
      "logits/rejected": -0.8441406488418579,
      "logps/chosen": -0.5263671875,
      "logps/rejected": -1.2392578125,
      "loss": 0.7204,
      "nll_loss": 0.6533203125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02631836012005806,
      "rewards/margins": 0.03565368801355362,
      "rewards/rejected": -0.06190185621380806,
      "step": 10910
    },
    {
      "epoch": 0.7955414708774997,
      "grad_norm": 4.486050011694473,
      "learning_rate": 4.784743764693455e-07,
      "log_odds_chosen": 1.184326171875,
      "log_odds_ratio": -0.4351562559604645,
      "logits/chosen": -0.9546874761581421,
      "logits/rejected": -0.862109363079071,
      "logps/chosen": -0.5146484375,
      "logps/rejected": -1.1589844226837158,
      "loss": 0.7207,
      "nll_loss": 0.7079101800918579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02572021447122097,
      "rewards/margins": 0.03218536451458931,
      "rewards/rejected": -0.05788574367761612,
      "step": 10920
    },
    {
      "epoch": 0.7962699887079736,
      "grad_norm": 2.54666116264057,
      "learning_rate": 4.782554451485866e-07,
      "log_odds_chosen": 1.1623046398162842,
      "log_odds_ratio": -0.4209960997104645,
      "logits/chosen": -1.0099608898162842,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.5166015625,
      "logps/rejected": -1.208398461341858,
      "loss": 0.7231,
      "nll_loss": 0.6947265863418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02579956129193306,
      "rewards/margins": 0.03467407077550888,
      "rewards/rejected": -0.06048583984375,
      "step": 10930
    },
    {
      "epoch": 0.7969985065384475,
      "grad_norm": 2.3569502565610283,
      "learning_rate": 4.780368140765222e-07,
      "log_odds_chosen": 0.9837890863418579,
      "log_odds_ratio": -0.48212891817092896,
      "logits/chosen": -1.0216796398162842,
      "logits/rejected": -0.895703136920929,
      "logps/chosen": -0.5474609136581421,
      "logps/rejected": -1.15625,
      "loss": 0.7258,
      "nll_loss": 0.694140613079071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02735595777630806,
      "rewards/margins": 0.03048400953412056,
      "rewards/rejected": -0.05781250074505806,
      "step": 10940
    },
    {
      "epoch": 0.7977270243689214,
      "grad_norm": 2.5719296822708633,
      "learning_rate": 4.778184825674966e-07,
      "log_odds_chosen": 1.276159644126892,
      "log_odds_ratio": -0.4642089903354645,
      "logits/chosen": -0.9791015386581421,
      "logits/rejected": -0.8765624761581421,
      "logps/chosen": -0.5545898675918579,
      "logps/rejected": -1.3777344226837158,
      "loss": 0.7041,
      "nll_loss": 0.72265625,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02773437462747097,
      "rewards/margins": 0.04117393493652344,
      "rewards/rejected": -0.06885375827550888,
      "step": 10950
    },
    {
      "epoch": 0.7984555421993953,
      "grad_norm": 2.2671893405051704,
      "learning_rate": 4.776004499380438e-07,
      "log_odds_chosen": 1.2291991710662842,
      "log_odds_ratio": -0.41596680879592896,
      "logits/chosen": -0.9828125238418579,
      "logits/rejected": -0.876953125,
      "logps/chosen": -0.5220702886581421,
      "logps/rejected": -1.2314453125,
      "loss": 0.6988,
      "nll_loss": 0.626660168170929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02611083909869194,
      "rewards/margins": 0.03543395921587944,
      "rewards/rejected": -0.0615234375,
      "step": 10960
    },
    {
      "epoch": 0.7991840600298692,
      "grad_norm": 4.660227811351288,
      "learning_rate": 4.773827155068792e-07,
      "log_odds_chosen": 1.24658203125,
      "log_odds_ratio": -0.4032226502895355,
      "logits/chosen": -0.9521484375,
      "logits/rejected": -0.8675781488418579,
      "logps/chosen": -0.46904295682907104,
      "logps/rejected": -1.135156273841858,
      "loss": 0.7141,
      "nll_loss": 0.6595703363418579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02344665490090847,
      "rewards/margins": 0.0333251953125,
      "rewards/rejected": -0.0567626953125,
      "step": 10970
    },
    {
      "epoch": 0.7999125778603431,
      "grad_norm": 2.5146574550033796,
      "learning_rate": 4.771652785948902e-07,
      "log_odds_chosen": 1.0775878429412842,
      "log_odds_ratio": -0.46318358182907104,
      "logits/chosen": -1.0246093273162842,
      "logits/rejected": -0.9134765863418579,
      "logps/chosen": -0.5425781011581421,
      "logps/rejected": -1.190820336341858,
      "loss": 0.7192,
      "nll_loss": 0.6888672113418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.027130126953125,
      "rewards/margins": 0.03238830715417862,
      "rewards/rejected": -0.05949706956744194,
      "step": 10980
    },
    {
      "epoch": 0.800641095690817,
      "grad_norm": 2.5304808833246075,
      "learning_rate": 4.769481385251275e-07,
      "log_odds_chosen": 1.0151855945587158,
      "log_odds_ratio": -0.4715332090854645,
      "logits/chosen": -1.005468726158142,
      "logits/rejected": -0.8857421875,
      "logps/chosen": -0.551562488079071,
      "logps/rejected": -1.1808593273162842,
      "loss": 0.7204,
      "nll_loss": 0.708203136920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02757568284869194,
      "rewards/margins": 0.03143310546875,
      "rewards/rejected": -0.05902709811925888,
      "step": 10990
    },
    {
      "epoch": 0.8013696135212909,
      "grad_norm": 1.9645023803059274,
      "learning_rate": 4.767312946227962e-07,
      "log_odds_chosen": 1.149023413658142,
      "log_odds_ratio": -0.4405273497104645,
      "logits/chosen": -0.982617199420929,
      "logits/rejected": -0.889453113079071,
      "logps/chosen": -0.5267578363418579,
      "logps/rejected": -1.1964843273162842,
      "loss": 0.7269,
      "nll_loss": 0.7138671875,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02634277381002903,
      "rewards/margins": 0.03348083421587944,
      "rewards/rejected": -0.05985107272863388,
      "step": 11000
    },
    {
      "epoch": 0.8020981313517649,
      "grad_norm": 2.2006306074004973,
      "learning_rate": 4.765147462152471e-07,
      "log_odds_chosen": 1.1736328601837158,
      "log_odds_ratio": -0.4404296875,
      "logits/chosen": -0.963671863079071,
      "logits/rejected": -0.8597656488418579,
      "logps/chosen": -0.552441418170929,
      "logps/rejected": -1.287695288658142,
      "loss": 0.7203,
      "nll_loss": 0.6826171875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02763671800494194,
      "rewards/margins": 0.0367431640625,
      "rewards/rejected": -0.06436767429113388,
      "step": 11010
    },
    {
      "epoch": 0.8028266491822388,
      "grad_norm": 2.546386549990972,
      "learning_rate": 4.7629849263196766e-07,
      "log_odds_chosen": 1.097753882408142,
      "log_odds_ratio": -0.44355469942092896,
      "logits/chosen": -1.0146484375,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -0.554980456829071,
      "logps/rejected": -1.2412109375,
      "loss": 0.7167,
      "nll_loss": 0.7334960699081421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.027740478515625,
      "rewards/margins": 0.03428344801068306,
      "rewards/rejected": -0.06207885593175888,
      "step": 11020
    },
    {
      "epoch": 0.8035551670127127,
      "grad_norm": 2.1816418775417787,
      "learning_rate": 4.760825332045739e-07,
      "log_odds_chosen": 1.141943335533142,
      "log_odds_ratio": -0.4576660096645355,
      "logits/chosen": -0.973828136920929,
      "logits/rejected": -0.8880859613418579,
      "logps/chosen": -0.5712890625,
      "logps/rejected": -1.2843749523162842,
      "loss": 0.7213,
      "nll_loss": 0.67822265625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02859191969037056,
      "rewards/margins": 0.03567542880773544,
      "rewards/rejected": -0.064239501953125,
      "step": 11030
    },
    {
      "epoch": 0.8042836848431866,
      "grad_norm": 3.6538586578591836,
      "learning_rate": 4.758668672668006e-07,
      "log_odds_chosen": 1.321386694908142,
      "log_odds_ratio": -0.4232421815395355,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.9302734136581421,
      "logps/chosen": -0.533007800579071,
      "logps/rejected": -1.342382788658142,
      "loss": 0.7195,
      "nll_loss": 0.6806640625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02663574181497097,
      "rewards/margins": 0.04048919677734375,
      "rewards/rejected": -0.06715087592601776,
      "step": 11040
    },
    {
      "epoch": 0.8050122026736605,
      "grad_norm": 2.1701381422193675,
      "learning_rate": 4.7565149415449407e-07,
      "log_odds_chosen": 1.1884765625,
      "log_odds_ratio": -0.447509765625,
      "logits/chosen": -1.011328101158142,
      "logits/rejected": -0.916796863079071,
      "logps/chosen": -0.52734375,
      "logps/rejected": -1.23046875,
      "loss": 0.723,
      "nll_loss": 0.717578113079071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02637939527630806,
      "rewards/margins": 0.03515625,
      "rewards/rejected": -0.06151122972369194,
      "step": 11050
    },
    {
      "epoch": 0.8057407205041344,
      "grad_norm": 3.3227647923194303,
      "learning_rate": 4.7543641320560246e-07,
      "log_odds_chosen": 1.0221679210662842,
      "log_odds_ratio": -0.482177734375,
      "logits/chosen": -1.003320336341858,
      "logits/rejected": -0.8671875,
      "logps/chosen": -0.554394543170929,
      "logps/rejected": -1.1716797351837158,
      "loss": 0.7302,
      "nll_loss": 0.7315429449081421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02769165113568306,
      "rewards/margins": 0.03087158128619194,
      "rewards/rejected": -0.05855713039636612,
      "step": 11060
    },
    {
      "epoch": 0.8064692383346083,
      "grad_norm": 2.270874884301976,
      "learning_rate": 4.7522162376016755e-07,
      "log_odds_chosen": 1.0347168445587158,
      "log_odds_ratio": -0.4749999940395355,
      "logits/chosen": -1.0037109851837158,
      "logits/rejected": -0.9330078363418579,
      "logps/chosen": -0.5313476324081421,
      "logps/rejected": -1.1134765148162842,
      "loss": 0.7205,
      "nll_loss": 0.6832031011581421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02656860277056694,
      "rewards/margins": 0.02908783033490181,
      "rewards/rejected": -0.05568237230181694,
      "step": 11070
    },
    {
      "epoch": 0.8071977561650822,
      "grad_norm": 2.4272209645630167,
      "learning_rate": 4.750071251603165e-07,
      "log_odds_chosen": 0.963671863079071,
      "log_odds_ratio": -0.4947265684604645,
      "logits/chosen": -1.027929663658142,
      "logits/rejected": -0.916015625,
      "logps/chosen": -0.5523437261581421,
      "logps/rejected": -1.1318359375,
      "loss": 0.7078,
      "nll_loss": 0.666210949420929,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.02760620042681694,
      "rewards/margins": 0.029006194323301315,
      "rewards/rejected": -0.05659179762005806,
      "step": 11080
    },
    {
      "epoch": 0.807926273995556,
      "grad_norm": 3.8357674596660543,
      "learning_rate": 4.747929167502531e-07,
      "log_odds_chosen": 1.0037720203399658,
      "log_odds_ratio": -0.484130859375,
      "logits/chosen": -0.971484363079071,
      "logits/rejected": -0.8583984375,
      "logps/chosen": -0.49101561307907104,
      "logps/rejected": -1.073632836341858,
      "loss": 0.7063,
      "nll_loss": 0.643261730670929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.024566650390625,
      "rewards/margins": 0.0290985107421875,
      "rewards/rejected": -0.05367431789636612,
      "step": 11090
    },
    {
      "epoch": 0.80865479182603,
      "grad_norm": 2.2046308380655697,
      "learning_rate": 4.745789978762495e-07,
      "log_odds_chosen": 1.078820824623108,
      "log_odds_ratio": -0.4661621153354645,
      "logits/chosen": -0.98046875,
      "logits/rejected": -0.8824218511581421,
      "logps/chosen": -0.5702148675918579,
      "logps/rejected": -1.214257836341858,
      "loss": 0.7198,
      "nll_loss": 0.746874988079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.028564453125,
      "rewards/margins": 0.032123565673828125,
      "rewards/rejected": -0.0606689453125,
      "step": 11100
    },
    {
      "epoch": 0.8093833096565038,
      "grad_norm": 2.3303145409218913,
      "learning_rate": 4.743653678866376e-07,
      "log_odds_chosen": 1.2373046875,
      "log_odds_ratio": -0.4039550721645355,
      "logits/chosen": -0.990039050579071,
      "logits/rejected": -0.886914074420929,
      "logps/chosen": -0.5562499761581421,
      "logps/rejected": -1.3230469226837158,
      "loss": 0.721,
      "nll_loss": 0.7505859136581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.02781982347369194,
      "rewards/margins": 0.03832092136144638,
      "rewards/rejected": -0.06618652492761612,
      "step": 11110
    },
    {
      "epoch": 0.8101118274869777,
      "grad_norm": 2.432601076925892,
      "learning_rate": 4.7415202613180106e-07,
      "log_odds_chosen": 1.1779296398162842,
      "log_odds_ratio": -0.43657225370407104,
      "logits/chosen": -0.97265625,
      "logits/rejected": -0.879101574420929,
      "logps/chosen": -0.5259765386581421,
      "logps/rejected": -1.232031226158142,
      "loss": 0.7202,
      "nll_loss": 0.686718761920929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02628173865377903,
      "rewards/margins": 0.03533325344324112,
      "rewards/rejected": -0.06159668043255806,
      "step": 11120
    },
    {
      "epoch": 0.8108403453174516,
      "grad_norm": 2.351305715421037,
      "learning_rate": 4.7393897196416653e-07,
      "log_odds_chosen": 1.254785180091858,
      "log_odds_ratio": -0.43876951932907104,
      "logits/chosen": -0.989062488079071,
      "logits/rejected": -0.842578113079071,
      "logps/chosen": -0.5365234613418579,
      "logps/rejected": -1.271093726158142,
      "loss": 0.7142,
      "nll_loss": 0.7291015386581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02681884728372097,
      "rewards/margins": 0.03670043870806694,
      "rewards/rejected": -0.06350097805261612,
      "step": 11130
    },
    {
      "epoch": 0.8115688631479255,
      "grad_norm": 2.441356398544974,
      "learning_rate": 4.737262047381962e-07,
      "log_odds_chosen": 0.953320324420929,
      "log_odds_ratio": -0.5057617425918579,
      "logits/chosen": -0.9404296875,
      "logits/rejected": -0.8453124761581421,
      "logps/chosen": -0.56201171875,
      "logps/rejected": -1.1220703125,
      "loss": 0.7186,
      "nll_loss": 0.7412109375,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02809448167681694,
      "rewards/margins": 0.0279998779296875,
      "rewards/rejected": -0.05611572414636612,
      "step": 11140
    },
    {
      "epoch": 0.8122973809783994,
      "grad_norm": 2.3839874099010694,
      "learning_rate": 4.7351372381037846e-07,
      "log_odds_chosen": 1.0146484375,
      "log_odds_ratio": -0.47685545682907104,
      "logits/chosen": -0.981249988079071,
      "logits/rejected": -0.8861328363418579,
      "logps/chosen": -0.548632800579071,
      "logps/rejected": -1.1769530773162842,
      "loss": 0.7168,
      "nll_loss": 0.6982421875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02742309495806694,
      "rewards/margins": 0.03142394870519638,
      "rewards/rejected": -0.05888671800494194,
      "step": 11150
    },
    {
      "epoch": 0.8130258988088733,
      "grad_norm": 2.4728853903399846,
      "learning_rate": 4.733015285392207e-07,
      "log_odds_chosen": 1.2454102039337158,
      "log_odds_ratio": -0.4141845703125,
      "logits/chosen": -1.0193359851837158,
      "logits/rejected": -0.944531261920929,
      "logps/chosen": -0.547167956829071,
      "logps/rejected": -1.284765601158142,
      "loss": 0.7066,
      "nll_loss": 0.662890613079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.027374267578125,
      "rewards/margins": 0.03689880296587944,
      "rewards/rejected": -0.06425781548023224,
      "step": 11160
    },
    {
      "epoch": 0.8137544166393472,
      "grad_norm": 2.6844101311646895,
      "learning_rate": 4.7308961828524083e-07,
      "log_odds_chosen": 1.2771484851837158,
      "log_odds_ratio": -0.44091796875,
      "logits/chosen": -0.9677734375,
      "logits/rejected": -0.8541015386581421,
      "logps/chosen": -0.5689452886581421,
      "logps/rejected": -1.376367211341858,
      "loss": 0.7133,
      "nll_loss": 0.6968749761581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0284423828125,
      "rewards/margins": 0.04032745212316513,
      "rewards/rejected": -0.06884765625,
      "step": 11170
    },
    {
      "epoch": 0.8144829344698211,
      "grad_norm": 2.5180128531187873,
      "learning_rate": 4.72877992410959e-07,
      "log_odds_chosen": 1.0026366710662842,
      "log_odds_ratio": -0.505175769329071,
      "logits/chosen": -1.009179711341858,
      "logits/rejected": -0.8863281011581421,
      "logps/chosen": -0.554980456829071,
      "logps/rejected": -1.1814453601837158,
      "loss": 0.7229,
      "nll_loss": 0.688671886920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02772827073931694,
      "rewards/margins": 0.031307220458984375,
      "rewards/rejected": -0.05902099609375,
      "step": 11180
    },
    {
      "epoch": 0.815211452300295,
      "grad_norm": 2.2057202107636944,
      "learning_rate": 4.7266665028088976e-07,
      "log_odds_chosen": 1.02587890625,
      "log_odds_ratio": -0.4764648377895355,
      "logits/chosen": -0.9212890863418579,
      "logits/rejected": -0.8384765386581421,
      "logps/chosen": -0.54150390625,
      "logps/rejected": -1.129492163658142,
      "loss": 0.7191,
      "nll_loss": 0.723828136920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0270538330078125,
      "rewards/margins": 0.02939758263528347,
      "rewards/rejected": -0.05642089992761612,
      "step": 11190
    },
    {
      "epoch": 0.8159399701307689,
      "grad_norm": 2.612461234828524,
      "learning_rate": 4.7245559126153405e-07,
      "log_odds_chosen": 0.949951171875,
      "log_odds_ratio": -0.5174804925918579,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.8988281488418579,
      "logps/chosen": -0.55078125,
      "logps/rejected": -1.12109375,
      "loss": 0.7234,
      "nll_loss": 0.7177734375,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02753295935690403,
      "rewards/margins": 0.028501510620117188,
      "rewards/rejected": -0.05610961839556694,
      "step": 11200
    },
    {
      "epoch": 0.8166684879612428,
      "grad_norm": 2.517261970395586,
      "learning_rate": 4.7224481472137115e-07,
      "log_odds_chosen": 1.130273461341858,
      "log_odds_ratio": -0.4512695372104645,
      "logits/chosen": -1.016015648841858,
      "logits/rejected": -0.892773449420929,
      "logps/chosen": -0.568652331829071,
      "logps/rejected": -1.250390648841858,
      "loss": 0.7095,
      "nll_loss": 0.6524413824081421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02844848670065403,
      "rewards/margins": 0.03409423679113388,
      "rewards/rejected": -0.06258545070886612,
      "step": 11210
    },
    {
      "epoch": 0.8173970057917167,
      "grad_norm": 2.460788968751757,
      "learning_rate": 4.720343200308506e-07,
      "log_odds_chosen": 1.1305663585662842,
      "log_odds_ratio": -0.424560546875,
      "logits/chosen": -1.0363280773162842,
      "logits/rejected": -0.929882824420929,
      "logps/chosen": -0.530468761920929,
      "logps/rejected": -1.1755859851837158,
      "loss": 0.7183,
      "nll_loss": 0.6802734136581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02650756761431694,
      "rewards/margins": 0.03223266452550888,
      "rewards/rejected": -0.05874023586511612,
      "step": 11220
    },
    {
      "epoch": 0.8181255236221906,
      "grad_norm": 2.2354246848083,
      "learning_rate": 4.718241065623849e-07,
      "log_odds_chosen": 0.971240222454071,
      "log_odds_ratio": -0.47504884004592896,
      "logits/chosen": -1.011132836341858,
      "logits/rejected": -0.890820324420929,
      "logps/chosen": -0.526074230670929,
      "logps/rejected": -1.113671898841858,
      "loss": 0.7167,
      "nll_loss": 0.6982421875,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02630004845559597,
      "rewards/margins": 0.02941436693072319,
      "rewards/rejected": -0.05568237230181694,
      "step": 11230
    },
    {
      "epoch": 0.8188540414526646,
      "grad_norm": 2.7135706431601374,
      "learning_rate": 4.716141736903406e-07,
      "log_odds_chosen": 1.195410132408142,
      "log_odds_ratio": -0.443359375,
      "logits/chosen": -0.9458984136581421,
      "logits/rejected": -0.85546875,
      "logps/chosen": -0.563183605670929,
      "logps/rejected": -1.2941405773162842,
      "loss": 0.7152,
      "nll_loss": 0.651171863079071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02814331091940403,
      "rewards/margins": 0.03658141940832138,
      "rewards/rejected": -0.06470946967601776,
      "step": 11240
    },
    {
      "epoch": 0.8195825592831385,
      "grad_norm": 2.177846247539774,
      "learning_rate": 4.714045207910317e-07,
      "log_odds_chosen": 1.1015136241912842,
      "log_odds_ratio": -0.4720214903354645,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.8814452886581421,
      "logps/chosen": -0.547558605670929,
      "logps/rejected": -1.2800781726837158,
      "loss": 0.7193,
      "nll_loss": 0.709765613079071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02738037146627903,
      "rewards/margins": 0.03660430759191513,
      "rewards/rejected": -0.063995361328125,
      "step": 11250
    },
    {
      "epoch": 0.8203110771136124,
      "grad_norm": 2.401384926143133,
      "learning_rate": 4.7119514724271095e-07,
      "log_odds_chosen": 1.516992211341858,
      "log_odds_ratio": -0.35624998807907104,
      "logits/chosen": -1.004296898841858,
      "logits/rejected": -0.8607422113418579,
      "logps/chosen": -0.45976561307907104,
      "logps/rejected": -1.3332030773162842,
      "loss": 0.7231,
      "nll_loss": 0.66796875,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.02299804612994194,
      "rewards/margins": 0.04369506984949112,
      "rewards/rejected": -0.066650390625,
      "step": 11260
    },
    {
      "epoch": 0.8210395949440863,
      "grad_norm": 2.6235950055689767,
      "learning_rate": 4.7098605242556225e-07,
      "log_odds_chosen": 1.20166015625,
      "log_odds_ratio": -0.45234376192092896,
      "logits/chosen": -0.9351562261581421,
      "logits/rejected": -0.8402343988418579,
      "logps/chosen": -0.529101550579071,
      "logps/rejected": -1.1970703601837158,
      "loss": 0.7232,
      "nll_loss": 0.6495116949081421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02644043043255806,
      "rewards/margins": 0.03338928148150444,
      "rewards/rejected": -0.059814453125,
      "step": 11270
    },
    {
      "epoch": 0.8217681127745602,
      "grad_norm": 2.159875944112355,
      "learning_rate": 4.7077723572169347e-07,
      "log_odds_chosen": 0.972399890422821,
      "log_odds_ratio": -0.49882811307907104,
      "logits/chosen": -0.968554675579071,
      "logits/rejected": -0.837890625,
      "logps/chosen": -0.54638671875,
      "logps/rejected": -1.1398437023162842,
      "loss": 0.7178,
      "nll_loss": 0.6968749761581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02733154222369194,
      "rewards/margins": 0.029631424695253372,
      "rewards/rejected": -0.05698242038488388,
      "step": 11280
    },
    {
      "epoch": 0.8224966306050341,
      "grad_norm": 1.9706825141479332,
      "learning_rate": 4.705686965151282e-07,
      "log_odds_chosen": 1.276953101158142,
      "log_odds_ratio": -0.4078613221645355,
      "logits/chosen": -0.9857422113418579,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.5254882574081421,
      "logps/rejected": -1.299414038658142,
      "loss": 0.7236,
      "nll_loss": 0.6963866949081421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02628173865377903,
      "rewards/margins": 0.03864746168255806,
      "rewards/rejected": -0.06494140625,
      "step": 11290
    },
    {
      "epoch": 0.823225148435508,
      "grad_norm": 2.5345282961995106,
      "learning_rate": 4.7036043419179867e-07,
      "log_odds_chosen": 1.228906273841858,
      "log_odds_ratio": -0.441162109375,
      "logits/chosen": -0.951171875,
      "logits/rejected": -0.8373047113418579,
      "logps/chosen": -0.5150390863418579,
      "logps/rejected": -1.273828148841858,
      "loss": 0.7296,
      "nll_loss": 0.745898425579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02574462816119194,
      "rewards/margins": 0.03800048679113388,
      "rewards/rejected": -0.06373290717601776,
      "step": 11300
    },
    {
      "epoch": 0.8239536662659819,
      "grad_norm": 2.4119825257012923,
      "learning_rate": 4.701524481395374e-07,
      "log_odds_chosen": 1.0905272960662842,
      "log_odds_ratio": -0.44169920682907104,
      "logits/chosen": -0.9771484136581421,
      "logits/rejected": -0.8255859613418579,
      "logps/chosen": -0.55224609375,
      "logps/rejected": -1.193750023841858,
      "loss": 0.7277,
      "nll_loss": 0.7337890863418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02761230431497097,
      "rewards/margins": 0.03201599046587944,
      "rewards/rejected": -0.05963134765625,
      "step": 11310
    },
    {
      "epoch": 0.8246821840964558,
      "grad_norm": 5.335492353615694,
      "learning_rate": 4.699447377480703e-07,
      "log_odds_chosen": 1.014892578125,
      "log_odds_ratio": -0.486328125,
      "logits/chosen": -0.9849609136581421,
      "logits/rejected": -0.8876953125,
      "logps/chosen": -0.580761730670929,
      "logps/rejected": -1.2117187976837158,
      "loss": 0.7246,
      "nll_loss": 0.7001953125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02904052659869194,
      "rewards/margins": 0.03153686597943306,
      "rewards/rejected": -0.06053466722369194,
      "step": 11320
    },
    {
      "epoch": 0.8254107019269297,
      "grad_norm": 2.2477109790067757,
      "learning_rate": 4.697373024090088e-07,
      "log_odds_chosen": 1.142553687095642,
      "log_odds_ratio": -0.4471191465854645,
      "logits/chosen": -0.9996093511581421,
      "logits/rejected": -0.8812500238418579,
      "logps/chosen": -0.530566394329071,
      "logps/rejected": -1.1980469226837158,
      "loss": 0.7051,
      "nll_loss": 0.6664062738418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.026519775390625,
      "rewards/margins": 0.03335227817296982,
      "rewards/rejected": -0.05986328050494194,
      "step": 11330
    },
    {
      "epoch": 0.8261392197574036,
      "grad_norm": 2.762610742649714,
      "learning_rate": 4.6953014151584257e-07,
      "log_odds_chosen": 1.14501953125,
      "log_odds_ratio": -0.4217773377895355,
      "logits/chosen": -0.9703124761581421,
      "logits/rejected": -0.858593761920929,
      "logps/chosen": -0.53515625,
      "logps/rejected": -1.235937476158142,
      "loss": 0.7064,
      "nll_loss": 0.6644531488418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02678832970559597,
      "rewards/margins": 0.03509216383099556,
      "rewards/rejected": -0.06187744066119194,
      "step": 11340
    },
    {
      "epoch": 0.8268677375878775,
      "grad_norm": 2.1424776516144,
      "learning_rate": 4.6932325446393214e-07,
      "log_odds_chosen": 1.0377929210662842,
      "log_odds_ratio": -0.4698730409145355,
      "logits/chosen": -0.951171875,
      "logits/rejected": -0.8548828363418579,
      "logps/chosen": -0.5462890863418579,
      "logps/rejected": -1.179101586341858,
      "loss": 0.7122,
      "nll_loss": 0.698046863079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02732544019818306,
      "rewards/margins": 0.03163452073931694,
      "rewards/rejected": -0.05894775316119194,
      "step": 11350
    },
    {
      "epoch": 0.8275962554183514,
      "grad_norm": 2.5851907040040576,
      "learning_rate": 4.69116640650501e-07,
      "log_odds_chosen": 1.053808569908142,
      "log_odds_ratio": -0.4458984434604645,
      "logits/chosen": -0.9720703363418579,
      "logits/rejected": -0.888671875,
      "logps/chosen": -0.57177734375,
      "logps/rejected": -1.2058594226837158,
      "loss": 0.7173,
      "nll_loss": 0.7079101800918579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02858276292681694,
      "rewards/margins": 0.03173828125,
      "rewards/rejected": -0.06029052659869194,
      "step": 11360
    },
    {
      "epoch": 0.8283247732488253,
      "grad_norm": 2.2002279939142677,
      "learning_rate": 4.6891029947462883e-07,
      "log_odds_chosen": 1.1199219226837158,
      "log_odds_ratio": -0.46586912870407104,
      "logits/chosen": -0.9488281011581421,
      "logits/rejected": -0.855273425579071,
      "logps/chosen": -0.5611327886581421,
      "logps/rejected": -1.260351538658142,
      "loss": 0.7071,
      "nll_loss": 0.7412109375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.028045654296875,
      "rewards/margins": 0.03496704250574112,
      "rewards/rejected": -0.06306152045726776,
      "step": 11370
    },
    {
      "epoch": 0.8290532910792991,
      "grad_norm": 2.433230555155692,
      "learning_rate": 4.687042303372439e-07,
      "log_odds_chosen": 0.97021484375,
      "log_odds_ratio": -0.4776855409145355,
      "logits/chosen": -0.974609375,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.571484386920929,
      "logps/rejected": -1.156640648841858,
      "loss": 0.7151,
      "nll_loss": 0.701855480670929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02855834923684597,
      "rewards/margins": 0.02920074388384819,
      "rewards/rejected": -0.05778808519244194,
      "step": 11380
    },
    {
      "epoch": 0.829781808909773,
      "grad_norm": 2.085959532003636,
      "learning_rate": 4.684984326411154e-07,
      "log_odds_chosen": 1.2404906749725342,
      "log_odds_ratio": -0.41240233182907104,
      "logits/chosen": -0.9791015386581421,
      "logits/rejected": -0.8501952886581421,
      "logps/chosen": -0.5127929449081421,
      "logps/rejected": -1.230078101158142,
      "loss": 0.7221,
      "nll_loss": 0.704296886920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02562255784869194,
      "rewards/margins": 0.03587799146771431,
      "rewards/rejected": -0.06157226487994194,
      "step": 11390
    },
    {
      "epoch": 0.8305103267402469,
      "grad_norm": 2.3601466152285213,
      "learning_rate": 4.6829290579084705e-07,
      "log_odds_chosen": 1.229467749595642,
      "log_odds_ratio": -0.45087891817092896,
      "logits/chosen": -0.924023449420929,
      "logits/rejected": -0.850781261920929,
      "logps/chosen": -0.514843761920929,
      "logps/rejected": -1.2373046875,
      "loss": 0.726,
      "nll_loss": 0.755078136920929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02571411058306694,
      "rewards/margins": 0.03614959865808487,
      "rewards/rejected": -0.06192626804113388,
      "step": 11400
    },
    {
      "epoch": 0.8312388445707208,
      "grad_norm": 2.0066853053458598,
      "learning_rate": 4.6808764919286887e-07,
      "log_odds_chosen": 0.79248046875,
      "log_odds_ratio": -0.52099609375,
      "logits/chosen": -0.9593750238418579,
      "logits/rejected": -0.8755859136581421,
      "logps/chosen": -0.49658203125,
      "logps/rejected": -0.918164074420929,
      "loss": 0.7115,
      "nll_loss": 0.6739257574081421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02484741248190403,
      "rewards/margins": 0.02107391320168972,
      "rewards/rejected": -0.04595947265625,
      "step": 11410
    },
    {
      "epoch": 0.8319673624011947,
      "grad_norm": 2.5144845381192553,
      "learning_rate": 4.678826622554308e-07,
      "log_odds_chosen": 1.1135742664337158,
      "log_odds_ratio": -0.463623046875,
      "logits/chosen": -1.006445288658142,
      "logits/rejected": -0.908007800579071,
      "logps/chosen": -0.5833984613418579,
      "logps/rejected": -1.2853515148162842,
      "loss": 0.7205,
      "nll_loss": 0.70361328125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02916259691119194,
      "rewards/margins": 0.03509826585650444,
      "rewards/rejected": -0.06428222358226776,
      "step": 11420
    },
    {
      "epoch": 0.8326958802316686,
      "grad_norm": 2.3468903605801157,
      "learning_rate": 4.676779443885948e-07,
      "log_odds_chosen": 1.166894555091858,
      "log_odds_ratio": -0.46088868379592896,
      "logits/chosen": -1.025390625,
      "logits/rejected": -0.9072265625,
      "logps/chosen": -0.5458008050918579,
      "logps/rejected": -1.2498047351837158,
      "loss": 0.7129,
      "nll_loss": 0.7481445074081421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02726440504193306,
      "rewards/margins": 0.03523864597082138,
      "rewards/rejected": -0.06252441555261612,
      "step": 11430
    },
    {
      "epoch": 0.8334243980621425,
      "grad_norm": 2.380965582228321,
      "learning_rate": 4.674734950042286e-07,
      "log_odds_chosen": 1.179589867591858,
      "log_odds_ratio": -0.4334716796875,
      "logits/chosen": -0.9970703125,
      "logits/rejected": -0.8955078125,
      "logps/chosen": -0.507519543170929,
      "logps/rejected": -1.1550781726837158,
      "loss": 0.7088,
      "nll_loss": 0.712109386920929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.025360107421875,
      "rewards/margins": 0.032366178929805756,
      "rewards/rejected": -0.05772094801068306,
      "step": 11440
    },
    {
      "epoch": 0.8341529158926164,
      "grad_norm": 2.158743291121844,
      "learning_rate": 4.672693135159978e-07,
      "log_odds_chosen": 1.0896484851837158,
      "log_odds_ratio": -0.466796875,
      "logits/chosen": -0.9541015625,
      "logits/rejected": -0.8363281488418579,
      "logps/chosen": -0.5489257574081421,
      "logps/rejected": -1.199804663658142,
      "loss": 0.7153,
      "nll_loss": 0.700878918170929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02744140662252903,
      "rewards/margins": 0.03258361667394638,
      "rewards/rejected": -0.06000976637005806,
      "step": 11450
    },
    {
      "epoch": 0.8348814337230904,
      "grad_norm": 2.6394414551012364,
      "learning_rate": 4.6706539933935904e-07,
      "log_odds_chosen": 1.053808569908142,
      "log_odds_ratio": -0.46625977754592896,
      "logits/chosen": -0.983593761920929,
      "logits/rejected": -0.894335925579071,
      "logps/chosen": -0.51220703125,
      "logps/rejected": -1.1435546875,
      "loss": 0.7044,
      "nll_loss": 0.630175769329071,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02561035193502903,
      "rewards/margins": 0.03149986267089844,
      "rewards/rejected": -0.05714721605181694,
      "step": 11460
    },
    {
      "epoch": 0.8356099515535643,
      "grad_norm": 2.721525062143482,
      "learning_rate": 4.668617518915533e-07,
      "log_odds_chosen": 1.1149413585662842,
      "log_odds_ratio": -0.442626953125,
      "logits/chosen": -0.9798828363418579,
      "logits/rejected": -0.849609375,
      "logps/chosen": -0.525195300579071,
      "logps/rejected": -1.1408202648162842,
      "loss": 0.7076,
      "nll_loss": 0.675488293170929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02623901329934597,
      "rewards/margins": 0.03076782263815403,
      "rewards/rejected": -0.0570068359375,
      "step": 11470
    },
    {
      "epoch": 0.8363384693840382,
      "grad_norm": 2.265810081711213,
      "learning_rate": 4.666583705915984e-07,
      "log_odds_chosen": 1.2482421398162842,
      "log_odds_ratio": -0.4410644471645355,
      "logits/chosen": -0.9791015386581421,
      "logits/rejected": -0.8695312738418579,
      "logps/chosen": -0.526562511920929,
      "logps/rejected": -1.252343773841858,
      "loss": 0.7132,
      "nll_loss": 0.7025390863418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.026336669921875,
      "rewards/margins": 0.03624420240521431,
      "rewards/rejected": -0.0626220703125,
      "step": 11480
    },
    {
      "epoch": 0.8370669872145121,
      "grad_norm": 3.176865530050265,
      "learning_rate": 4.664552548602825e-07,
      "log_odds_chosen": 1.17822265625,
      "log_odds_ratio": -0.458984375,
      "logits/chosen": -0.982617199420929,
      "logits/rejected": -0.870898425579071,
      "logps/chosen": -0.5660156011581421,
      "logps/rejected": -1.31640625,
      "loss": 0.7323,
      "nll_loss": 0.7212890386581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02833252027630806,
      "rewards/margins": 0.03747253492474556,
      "rewards/rejected": -0.06574706733226776,
      "step": 11490
    },
    {
      "epoch": 0.837795505044986,
      "grad_norm": 2.4374593369572986,
      "learning_rate": 4.6625240412015694e-07,
      "log_odds_chosen": 1.2996094226837158,
      "log_odds_ratio": -0.4134765565395355,
      "logits/chosen": -1.0251953601837158,
      "logits/rejected": -0.9027343988418579,
      "logps/chosen": -0.541796863079071,
      "logps/rejected": -1.363867163658142,
      "loss": 0.7053,
      "nll_loss": 0.6416015625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02709350548684597,
      "rewards/margins": 0.04105224460363388,
      "rewards/rejected": -0.06816406548023224,
      "step": 11500
    },
    {
      "epoch": 0.8385240228754599,
      "grad_norm": 2.3539088740596763,
      "learning_rate": 4.6604981779552915e-07,
      "log_odds_chosen": 1.2616088390350342,
      "log_odds_ratio": -0.4203124940395355,
      "logits/chosen": -0.9203125238418579,
      "logits/rejected": -0.8304687738418579,
      "logps/chosen": -0.48505860567092896,
      "logps/rejected": -1.224023461341858,
      "loss": 0.7238,
      "nll_loss": 0.6885741949081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02424926683306694,
      "rewards/margins": 0.036949921399354935,
      "rewards/rejected": -0.0611572265625,
      "step": 11510
    },
    {
      "epoch": 0.8392525407059338,
      "grad_norm": 2.54410432613032,
      "learning_rate": 4.6584749531245624e-07,
      "log_odds_chosen": 1.206445336341858,
      "log_odds_ratio": -0.40673828125,
      "logits/chosen": -0.985546886920929,
      "logits/rejected": -0.8609374761581421,
      "logps/chosen": -0.542773425579071,
      "logps/rejected": -1.2822265625,
      "loss": 0.7,
      "nll_loss": 0.7193359136581421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.02713623084127903,
      "rewards/margins": 0.03694457933306694,
      "rewards/rejected": -0.0640869140625,
      "step": 11520
    },
    {
      "epoch": 0.8399810585364077,
      "grad_norm": 2.5732539600708813,
      "learning_rate": 4.656454360987378e-07,
      "log_odds_chosen": 1.1179687976837158,
      "log_odds_ratio": -0.46611326932907104,
      "logits/chosen": -0.958203136920929,
      "logits/rejected": -0.881640613079071,
      "logps/chosen": -0.5040038824081421,
      "logps/rejected": -1.1533203125,
      "loss": 0.7211,
      "nll_loss": 0.701171875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02520446851849556,
      "rewards/margins": 0.032440949231386185,
      "rewards/rejected": -0.05762328952550888,
      "step": 11530
    },
    {
      "epoch": 0.8407095763668816,
      "grad_norm": 2.255255403498335,
      "learning_rate": 4.654436395839094e-07,
      "log_odds_chosen": 1.0232422351837158,
      "log_odds_ratio": -0.4984374940395355,
      "logits/chosen": -0.9673827886581421,
      "logits/rejected": -0.892382800579071,
      "logps/chosen": -0.5655273199081421,
      "logps/rejected": -1.1462891101837158,
      "loss": 0.7349,
      "nll_loss": 0.7303711175918579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.0282745361328125,
      "rewards/margins": 0.02909240685403347,
      "rewards/rejected": -0.05729980394244194,
      "step": 11540
    },
    {
      "epoch": 0.8414380941973555,
      "grad_norm": 2.6453318805159634,
      "learning_rate": 4.652421051992355e-07,
      "log_odds_chosen": 1.001855492591858,
      "log_odds_ratio": -0.462890625,
      "logits/chosen": -0.986328125,
      "logits/rejected": -0.856640636920929,
      "logps/chosen": -0.5536133050918579,
      "logps/rejected": -1.1318359375,
      "loss": 0.7176,
      "nll_loss": 0.731640636920929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0277099609375,
      "rewards/margins": 0.028928374871611595,
      "rewards/rejected": -0.05660400539636612,
      "step": 11550
    },
    {
      "epoch": 0.8421666120278294,
      "grad_norm": 2.339912899312864,
      "learning_rate": 4.650408323777029e-07,
      "log_odds_chosen": 1.0129883289337158,
      "log_odds_ratio": -0.4930175840854645,
      "logits/chosen": -0.9750000238418579,
      "logits/rejected": -0.8857421875,
      "logps/chosen": -0.5614258050918579,
      "logps/rejected": -1.1951172351837158,
      "loss": 0.7098,
      "nll_loss": 0.674121081829071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02807006798684597,
      "rewards/margins": 0.03168640285730362,
      "rewards/rejected": -0.059814453125,
      "step": 11560
    },
    {
      "epoch": 0.8428951298583033,
      "grad_norm": 2.5454795465076074,
      "learning_rate": 4.648398205540141e-07,
      "log_odds_chosen": 1.2765624523162842,
      "log_odds_ratio": -0.4150390625,
      "logits/chosen": -0.9800781011581421,
      "logits/rejected": -0.8519531488418579,
      "logps/chosen": -0.554882824420929,
      "logps/rejected": -1.331445336341858,
      "loss": 0.7102,
      "nll_loss": 0.7208007574081421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.027740478515625,
      "rewards/margins": 0.03885497897863388,
      "rewards/rejected": -0.06657715141773224,
      "step": 11570
    },
    {
      "epoch": 0.8436236476887772,
      "grad_norm": 2.8986808426053625,
      "learning_rate": 4.6463906916458053e-07,
      "log_odds_chosen": 1.335107445716858,
      "log_odds_ratio": -0.375732421875,
      "logits/chosen": -0.9789062738418579,
      "logits/rejected": -0.8599609136581421,
      "logps/chosen": -0.5303710699081421,
      "logps/rejected": -1.30859375,
      "loss": 0.7047,
      "nll_loss": 0.6553710699081421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02653198316693306,
      "rewards/margins": 0.03887939453125,
      "rewards/rejected": -0.06546630710363388,
      "step": 11580
    },
    {
      "epoch": 0.8443521655192511,
      "grad_norm": 2.335590008141301,
      "learning_rate": 4.644385776475159e-07,
      "log_odds_chosen": 0.867236316204071,
      "log_odds_ratio": -0.525585949420929,
      "logits/chosen": -1.0167968273162842,
      "logits/rejected": -0.904101550579071,
      "logps/chosen": -0.552539050579071,
      "logps/rejected": -1.0986328125,
      "loss": 0.6974,
      "nll_loss": 0.716015636920929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02763671800494194,
      "rewards/margins": 0.02727355994284153,
      "rewards/rejected": -0.05491943284869194,
      "step": 11590
    },
    {
      "epoch": 0.845080683349725,
      "grad_norm": 2.643231572921353,
      "learning_rate": 4.642383454426297e-07,
      "log_odds_chosen": 1.10791015625,
      "log_odds_ratio": -0.4632324278354645,
      "logits/chosen": -0.935351550579071,
      "logits/rejected": -0.8744140863418579,
      "logps/chosen": -0.5586913824081421,
      "logps/rejected": -1.1701171398162842,
      "loss": 0.7364,
      "nll_loss": 0.7958984375,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02796630933880806,
      "rewards/margins": 0.030579376965761185,
      "rewards/rejected": -0.05849609524011612,
      "step": 11600
    },
    {
      "epoch": 0.8458092011801989,
      "grad_norm": 2.8062886529640623,
      "learning_rate": 4.640383719914205e-07,
      "log_odds_chosen": 1.120019555091858,
      "log_odds_ratio": -0.4501953125,
      "logits/chosen": -1.004296898841858,
      "logits/rejected": -0.88671875,
      "logps/chosen": -0.5282226800918579,
      "logps/rejected": -1.2136719226837158,
      "loss": 0.7011,
      "nll_loss": 0.713574230670929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02641601487994194,
      "rewards/margins": 0.034268952906131744,
      "rewards/rejected": -0.06064452975988388,
      "step": 11610
    },
    {
      "epoch": 0.8465377190106728,
      "grad_norm": 2.183074127391095,
      "learning_rate": 4.6383865673706943e-07,
      "log_odds_chosen": 0.997851550579071,
      "log_odds_ratio": -0.473876953125,
      "logits/chosen": -1.013281226158142,
      "logits/rejected": -0.9095703363418579,
      "logps/chosen": -0.560351550579071,
      "logps/rejected": -1.1658203601837158,
      "loss": 0.7065,
      "nll_loss": 0.6895507574081421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02801818773150444,
      "rewards/margins": 0.03027038648724556,
      "rewards/rejected": -0.05826415866613388,
      "step": 11620
    },
    {
      "epoch": 0.8472662368411467,
      "grad_norm": 2.2188621582117944,
      "learning_rate": 4.6363919912443385e-07,
      "log_odds_chosen": 1.0216796398162842,
      "log_odds_ratio": -0.4955078065395355,
      "logits/chosen": -0.9501953125,
      "logits/rejected": -0.8740234375,
      "logps/chosen": -0.541308581829071,
      "logps/rejected": -1.1443359851837158,
      "loss": 0.7041,
      "nll_loss": 0.68603515625,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02706298790872097,
      "rewards/margins": 0.03019103966653347,
      "rewards/rejected": -0.05726318433880806,
      "step": 11630
    },
    {
      "epoch": 0.8479947546716206,
      "grad_norm": 2.2166801274219403,
      "learning_rate": 4.634399986000405e-07,
      "log_odds_chosen": 1.0052611827850342,
      "log_odds_ratio": -0.5049804449081421,
      "logits/chosen": -0.963671863079071,
      "logits/rejected": -0.896679699420929,
      "logps/chosen": -0.5455077886581421,
      "logps/rejected": -1.175390601158142,
      "loss": 0.7098,
      "nll_loss": 0.7061523199081421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02727661095559597,
      "rewards/margins": 0.03149108961224556,
      "rewards/rejected": -0.05875243991613388,
      "step": 11640
    },
    {
      "epoch": 0.8487232725020945,
      "grad_norm": 2.513069888454587,
      "learning_rate": 4.6324105461207953e-07,
      "log_odds_chosen": 1.0976073741912842,
      "log_odds_ratio": -0.45854490995407104,
      "logits/chosen": -0.9857422113418579,
      "logits/rejected": -0.892382800579071,
      "logps/chosen": -0.5381835699081421,
      "logps/rejected": -1.1650390625,
      "loss": 0.7078,
      "nll_loss": 0.735156238079071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02692871168255806,
      "rewards/margins": 0.031316377222537994,
      "rewards/rejected": -0.0582275390625,
      "step": 11650
    },
    {
      "epoch": 0.8494517903325683,
      "grad_norm": 2.0624282053419876,
      "learning_rate": 4.630423666103976e-07,
      "log_odds_chosen": 1.2724609375,
      "log_odds_ratio": -0.41743165254592896,
      "logits/chosen": -1.0066406726837158,
      "logits/rejected": -0.900195300579071,
      "logps/chosen": -0.5142577886581421,
      "logps/rejected": -1.257421851158142,
      "loss": 0.7047,
      "nll_loss": 0.6490234136581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02568664588034153,
      "rewards/margins": 0.03717346116900444,
      "rewards/rejected": -0.06285400688648224,
      "step": 11660
    },
    {
      "epoch": 0.8501803081630422,
      "grad_norm": 2.513231290183183,
      "learning_rate": 4.628439340464918e-07,
      "log_odds_chosen": 1.078369140625,
      "log_odds_ratio": -0.4911132752895355,
      "logits/chosen": -0.9478515386581421,
      "logits/rejected": -0.873242199420929,
      "logps/chosen": -0.596484363079071,
      "logps/rejected": -1.2976562976837158,
      "loss": 0.7019,
      "nll_loss": 0.7032226324081421,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02984008751809597,
      "rewards/margins": 0.03503532335162163,
      "rewards/rejected": -0.06488037109375,
      "step": 11670
    },
    {
      "epoch": 0.8509088259935161,
      "grad_norm": 2.9330589393668096,
      "learning_rate": 4.626457563735033e-07,
      "log_odds_chosen": 1.294042944908142,
      "log_odds_ratio": -0.393310546875,
      "logits/chosen": -0.964648425579071,
      "logits/rejected": -0.8550781011581421,
      "logps/chosen": -0.5213867425918579,
      "logps/rejected": -1.341210961341858,
      "loss": 0.7141,
      "nll_loss": 0.6607421636581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.026092529296875,
      "rewards/margins": 0.04098205640912056,
      "rewards/rejected": -0.06708984076976776,
      "step": 11680
    },
    {
      "epoch": 0.8516373438239901,
      "grad_norm": 2.758059701418182,
      "learning_rate": 4.6244783304621077e-07,
      "log_odds_chosen": 1.217529296875,
      "log_odds_ratio": -0.4426025450229645,
      "logits/chosen": -1.0322265625,
      "logits/rejected": -0.881640613079071,
      "logps/chosen": -0.538281261920929,
      "logps/rejected": -1.277929663658142,
      "loss": 0.7096,
      "nll_loss": 0.6592773199081421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02692871168255806,
      "rewards/margins": 0.03691711276769638,
      "rewards/rejected": -0.06390380859375,
      "step": 11690
    },
    {
      "epoch": 0.852365861654464,
      "grad_norm": 2.060653989445209,
      "learning_rate": 4.6225016352102427e-07,
      "log_odds_chosen": 1.1110656261444092,
      "log_odds_ratio": -0.427978515625,
      "logits/chosen": -1.030859351158142,
      "logits/rejected": -0.9136718511581421,
      "logps/chosen": -0.519335925579071,
      "logps/rejected": -1.207421898841858,
      "loss": 0.7094,
      "nll_loss": 0.6553710699081421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.025970458984375,
      "rewards/margins": 0.03439369052648544,
      "rewards/rejected": -0.06037597730755806,
      "step": 11700
    },
    {
      "epoch": 0.8530943794849379,
      "grad_norm": 2.4974475667830536,
      "learning_rate": 4.62052747255979e-07,
      "log_odds_chosen": 1.0679442882537842,
      "log_odds_ratio": -0.4579101502895355,
      "logits/chosen": -1.0353515148162842,
      "logits/rejected": -0.9175781011581421,
      "logps/chosen": -0.5550781488418579,
      "logps/rejected": -1.1767578125,
      "loss": 0.7028,
      "nll_loss": 0.7216796875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02775878831744194,
      "rewards/margins": 0.03106384351849556,
      "rewards/rejected": -0.05880127102136612,
      "step": 11710
    },
    {
      "epoch": 0.8538228973154118,
      "grad_norm": 2.604220106189069,
      "learning_rate": 4.61855583710729e-07,
      "log_odds_chosen": 1.07452392578125,
      "log_odds_ratio": -0.45219725370407104,
      "logits/chosen": -0.9937499761581421,
      "logits/rejected": -0.8822265863418579,
      "logps/chosen": -0.5645507574081421,
      "logps/rejected": -1.222265601158142,
      "loss": 0.7095,
      "nll_loss": 0.6885741949081421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.028228759765625,
      "rewards/margins": 0.03290100023150444,
      "rewards/rejected": -0.06109619140625,
      "step": 11720
    },
    {
      "epoch": 0.8545514151458857,
      "grad_norm": 2.2218550359572533,
      "learning_rate": 4.6165867234654084e-07,
      "log_odds_chosen": 1.2041015625,
      "log_odds_ratio": -0.404296875,
      "logits/chosen": -0.9853515625,
      "logits/rejected": -0.8583984375,
      "logps/chosen": -0.531933605670929,
      "logps/rejected": -1.2501952648162842,
      "loss": 0.7058,
      "nll_loss": 0.669238269329071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02660522423684597,
      "rewards/margins": 0.03597412258386612,
      "rewards/rejected": -0.06256103515625,
      "step": 11730
    },
    {
      "epoch": 0.8552799329763596,
      "grad_norm": 2.70430224345114,
      "learning_rate": 4.6146201262628753e-07,
      "log_odds_chosen": 1.0808594226837158,
      "log_odds_ratio": -0.4541015625,
      "logits/chosen": -0.9906250238418579,
      "logits/rejected": -0.847851574420929,
      "logps/chosen": -0.49882811307907104,
      "logps/rejected": -1.127343773841858,
      "loss": 0.7011,
      "nll_loss": 0.6552734375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02493896521627903,
      "rewards/margins": 0.03142394870519638,
      "rewards/rejected": -0.05635986477136612,
      "step": 11740
    },
    {
      "epoch": 0.8560084508068335,
      "grad_norm": 2.3607641396251786,
      "learning_rate": 4.6126560401444254e-07,
      "log_odds_chosen": 1.028778076171875,
      "log_odds_ratio": -0.48369139432907104,
      "logits/chosen": -0.983203113079071,
      "logits/rejected": -0.8837890625,
      "logps/chosen": -0.5625,
      "logps/rejected": -1.20458984375,
      "loss": 0.7141,
      "nll_loss": 0.6900390386581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02813110314309597,
      "rewards/margins": 0.032115936279296875,
      "rewards/rejected": -0.06021728366613388,
      "step": 11750
    },
    {
      "epoch": 0.8567369686373074,
      "grad_norm": 3.4857322333418,
      "learning_rate": 4.6106944597707345e-07,
      "log_odds_chosen": 1.079199194908142,
      "log_odds_ratio": -0.42851561307907104,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.837695300579071,
      "logps/chosen": -0.5293945074081421,
      "logps/rejected": -1.11328125,
      "loss": 0.6966,
      "nll_loss": 0.697265625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02646484412252903,
      "rewards/margins": 0.02923889085650444,
      "rewards/rejected": -0.05571288987994194,
      "step": 11760
    },
    {
      "epoch": 0.8574654864677813,
      "grad_norm": 2.344626925086774,
      "learning_rate": 4.6087353798183585e-07,
      "log_odds_chosen": 0.8582397699356079,
      "log_odds_ratio": -0.51171875,
      "logits/chosen": -0.9683593511581421,
      "logits/rejected": -0.8265625238418579,
      "logps/chosen": -0.5575195550918579,
      "logps/rejected": -1.0832030773162842,
      "loss": 0.7082,
      "nll_loss": 0.693359375,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02786865271627903,
      "rewards/margins": 0.02627868577837944,
      "rewards/rejected": -0.05412597581744194,
      "step": 11770
    },
    {
      "epoch": 0.8581940042982552,
      "grad_norm": 2.1996549323623884,
      "learning_rate": 4.606778794979673e-07,
      "log_odds_chosen": 1.0495483875274658,
      "log_odds_ratio": -0.4922851622104645,
      "logits/chosen": -1.0205078125,
      "logits/rejected": -0.906445324420929,
      "logps/chosen": -0.545117199420929,
      "logps/rejected": -1.138085961341858,
      "loss": 0.6964,
      "nll_loss": 0.710644543170929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02726135216653347,
      "rewards/margins": 0.029657745733857155,
      "rewards/rejected": -0.05694580078125,
      "step": 11780
    },
    {
      "epoch": 0.8589225221287291,
      "grad_norm": 2.081457064532829,
      "learning_rate": 4.6048246999628133e-07,
      "log_odds_chosen": 0.935717761516571,
      "log_odds_ratio": -0.511523425579071,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.8603515625,
      "logps/chosen": -0.569628894329071,
      "logps/rejected": -1.146484375,
      "loss": 0.6962,
      "nll_loss": 0.6908203363418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02848510816693306,
      "rewards/margins": 0.02882995642721653,
      "rewards/rejected": -0.05733642727136612,
      "step": 11790
    },
    {
      "epoch": 0.859651039959203,
      "grad_norm": 2.2082836614546024,
      "learning_rate": 4.6028730894916175e-07,
      "log_odds_chosen": 1.079687476158142,
      "log_odds_ratio": -0.458984375,
      "logits/chosen": -0.958789050579071,
      "logits/rejected": -0.875781238079071,
      "logps/chosen": -0.5038086175918579,
      "logps/rejected": -1.118554711341858,
      "loss": 0.6977,
      "nll_loss": 0.625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02519531175494194,
      "rewards/margins": 0.03070068359375,
      "rewards/rejected": -0.05587158352136612,
      "step": 11800
    },
    {
      "epoch": 0.8603795577896769,
      "grad_norm": 2.0292850201990893,
      "learning_rate": 4.600923958305557e-07,
      "log_odds_chosen": 1.265527367591858,
      "log_odds_ratio": -0.40605467557907104,
      "logits/chosen": -0.9525390863418579,
      "logits/rejected": -0.8511718511581421,
      "logps/chosen": -0.503222644329071,
      "logps/rejected": -1.2595703601837158,
      "loss": 0.7112,
      "nll_loss": 0.7149413824081421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02513427659869194,
      "rewards/margins": 0.03785400465130806,
      "rewards/rejected": -0.06297607719898224,
      "step": 11810
    },
    {
      "epoch": 0.8611080756201508,
      "grad_norm": 2.1130668277185434,
      "learning_rate": 4.59897730115969e-07,
      "log_odds_chosen": 1.0583984851837158,
      "log_odds_ratio": -0.48457032442092896,
      "logits/chosen": -0.9521484375,
      "logits/rejected": -0.8775390386581421,
      "logps/chosen": -0.5665038824081421,
      "logps/rejected": -1.1845703125,
      "loss": 0.7071,
      "nll_loss": 0.71875,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02831420861184597,
      "rewards/margins": 0.03093872033059597,
      "rewards/rejected": -0.05925292894244194,
      "step": 11820
    },
    {
      "epoch": 0.8618365934506247,
      "grad_norm": 2.8359999265927796,
      "learning_rate": 4.5970331128245914e-07,
      "log_odds_chosen": 1.0012695789337158,
      "log_odds_ratio": -0.47773438692092896,
      "logits/chosen": -0.9800781011581421,
      "logits/rejected": -0.8675781488418579,
      "logps/chosen": -0.575390636920929,
      "logps/rejected": -1.2208983898162842,
      "loss": 0.6988,
      "nll_loss": 0.665332019329071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02878418006002903,
      "rewards/margins": 0.03230895847082138,
      "rewards/rejected": -0.06105346605181694,
      "step": 11830
    },
    {
      "epoch": 0.8625651112810986,
      "grad_norm": 2.8648954139619134,
      "learning_rate": 4.595091388086299e-07,
      "log_odds_chosen": 1.1025390625,
      "log_odds_ratio": -0.44921875,
      "logits/chosen": -0.993359386920929,
      "logits/rejected": -0.8724609613418579,
      "logps/chosen": -0.53955078125,
      "logps/rejected": -1.1916015148162842,
      "loss": 0.7061,
      "nll_loss": 0.6771484613418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02697143517434597,
      "rewards/margins": 0.03259735181927681,
      "rewards/rejected": -0.05958252027630806,
      "step": 11840
    },
    {
      "epoch": 0.8632936291115725,
      "grad_norm": 1.9938577975432894,
      "learning_rate": 4.5931521217462543e-07,
      "log_odds_chosen": 1.1806640625,
      "log_odds_ratio": -0.4535156190395355,
      "logits/chosen": -0.9691406488418579,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.529492199420929,
      "logps/rejected": -1.1902344226837158,
      "loss": 0.6945,
      "nll_loss": 0.645703136920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.0264739990234375,
      "rewards/margins": 0.0330810546875,
      "rewards/rejected": -0.05959472805261612,
      "step": 11850
    },
    {
      "epoch": 0.8640221469420464,
      "grad_norm": 1.8902617802451827,
      "learning_rate": 4.591215308621242e-07,
      "log_odds_chosen": 1.23095703125,
      "log_odds_ratio": -0.3948730528354645,
      "logits/chosen": -0.9957031011581421,
      "logits/rejected": -0.8628906011581421,
      "logps/chosen": -0.506640613079071,
      "logps/rejected": -1.2021484375,
      "loss": 0.6855,
      "nll_loss": 0.648632824420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02532348595559597,
      "rewards/margins": 0.03476715087890625,
      "rewards/rejected": -0.06007080152630806,
      "step": 11860
    },
    {
      "epoch": 0.8647506647725203,
      "grad_norm": 3.0957268867549583,
      "learning_rate": 4.589280943543336e-07,
      "log_odds_chosen": 1.193945288658142,
      "log_odds_ratio": -0.4302734434604645,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.8857421875,
      "logps/chosen": -0.53857421875,
      "logps/rejected": -1.2707030773162842,
      "loss": 0.6953,
      "nll_loss": 0.656933605670929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.02691650390625,
      "rewards/margins": 0.03662719577550888,
      "rewards/rejected": -0.06358642876148224,
      "step": 11870
    },
    {
      "epoch": 0.8654791826029942,
      "grad_norm": 2.153280538410634,
      "learning_rate": 4.587349021359836e-07,
      "log_odds_chosen": 1.202050805091858,
      "log_odds_ratio": -0.40869140625,
      "logits/chosen": -1.008203148841858,
      "logits/rejected": -0.860546886920929,
      "logps/chosen": -0.49951171875,
      "logps/rejected": -1.2273437976837158,
      "loss": 0.6738,
      "nll_loss": 0.642578125,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02496948279440403,
      "rewards/margins": 0.036449432373046875,
      "rewards/rejected": -0.06140746921300888,
      "step": 11880
    },
    {
      "epoch": 0.8662077004334681,
      "grad_norm": 2.359957835936585,
      "learning_rate": 4.585419536933215e-07,
      "log_odds_chosen": 1.193750023841858,
      "log_odds_ratio": -0.41083985567092896,
      "logits/chosen": -1.0109374523162842,
      "logits/rejected": -0.8814452886581421,
      "logps/chosen": -0.5107421875,
      "logps/rejected": -1.1369140148162842,
      "loss": 0.7163,
      "nll_loss": 0.637011706829071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.02553710900247097,
      "rewards/margins": 0.03128356859087944,
      "rewards/rejected": -0.05679931491613388,
      "step": 11890
    },
    {
      "epoch": 0.866936218263942,
      "grad_norm": 2.060807018114865,
      "learning_rate": 4.583492485141057e-07,
      "log_odds_chosen": 1.116064429283142,
      "log_odds_ratio": -0.4544921815395355,
      "logits/chosen": -1.019140601158142,
      "logits/rejected": -0.9095703363418579,
      "logps/chosen": -0.55419921875,
      "logps/rejected": -1.2335937023162842,
      "loss": 0.7108,
      "nll_loss": 0.70703125,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0277099609375,
      "rewards/margins": 0.03395233303308487,
      "rewards/rejected": -0.06166992336511612,
      "step": 11900
    },
    {
      "epoch": 0.8676647360944159,
      "grad_norm": 3.3840007933188447,
      "learning_rate": 4.581567860876004e-07,
      "log_odds_chosen": 1.078125,
      "log_odds_ratio": -0.4601074159145355,
      "logits/chosen": -0.980273425579071,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.546191394329071,
      "logps/rejected": -1.219140648841858,
      "loss": 0.6907,
      "nll_loss": 0.6455078125,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02729492262005806,
      "rewards/margins": 0.03365783765912056,
      "rewards/rejected": -0.06101074069738388,
      "step": 11910
    },
    {
      "epoch": 0.8683932539248899,
      "grad_norm": 2.8887117671366327,
      "learning_rate": 4.579645659045698e-07,
      "log_odds_chosen": 1.1769530773162842,
      "log_odds_ratio": -0.4369140565395355,
      "logits/chosen": -1.012304663658142,
      "logits/rejected": -0.903515636920929,
      "logps/chosen": -0.538867175579071,
      "logps/rejected": -1.2800781726837158,
      "loss": 0.6899,
      "nll_loss": 0.670703113079071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02693481370806694,
      "rewards/margins": 0.03711242601275444,
      "rewards/rejected": -0.06405029445886612,
      "step": 11920
    },
    {
      "epoch": 0.8691217717553638,
      "grad_norm": 3.0261907310293417,
      "learning_rate": 4.577725874572724e-07,
      "log_odds_chosen": 1.18701171875,
      "log_odds_ratio": -0.44243162870407104,
      "logits/chosen": -1.000390648841858,
      "logits/rejected": -0.8998047113418579,
      "logps/chosen": -0.524609386920929,
      "logps/rejected": -1.214453101158142,
      "loss": 0.7073,
      "nll_loss": 0.66650390625,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02622680738568306,
      "rewards/margins": 0.03448181226849556,
      "rewards/rejected": -0.06069336086511612,
      "step": 11930
    },
    {
      "epoch": 0.8698502895858377,
      "grad_norm": 2.661889840717283,
      "learning_rate": 4.575808502394551e-07,
      "log_odds_chosen": 1.335351586341858,
      "log_odds_ratio": -0.4109863340854645,
      "logits/chosen": -1.0363280773162842,
      "logits/rejected": -0.91796875,
      "logps/chosen": -0.575488269329071,
      "logps/rejected": -1.3984375,
      "loss": 0.7132,
      "nll_loss": 0.7109375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.028778076171875,
      "rewards/margins": 0.04114379733800888,
      "rewards/rejected": -0.0699462890625,
      "step": 11940
    },
    {
      "epoch": 0.8705788074163116,
      "grad_norm": 2.748203787383127,
      "learning_rate": 4.573893537463483e-07,
      "log_odds_chosen": 1.2276611328125,
      "log_odds_ratio": -0.43583983182907104,
      "logits/chosen": -0.997851550579071,
      "logits/rejected": -0.879687488079071,
      "logps/chosen": -0.5643554925918579,
      "logps/rejected": -1.3660156726837158,
      "loss": 0.6866,
      "nll_loss": 0.7037109136581421,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02818603441119194,
      "rewards/margins": 0.04011230543255806,
      "rewards/rejected": -0.06831054389476776,
      "step": 11950
    },
    {
      "epoch": 0.8713073252467854,
      "grad_norm": 2.041345678071257,
      "learning_rate": 4.5719809747465945e-07,
      "log_odds_chosen": 1.120507836341858,
      "log_odds_ratio": -0.42744141817092896,
      "logits/chosen": -0.9771484136581421,
      "logits/rejected": -0.8550781011581421,
      "logps/chosen": -0.529980480670929,
      "logps/rejected": -1.1960937976837158,
      "loss": 0.7052,
      "nll_loss": 0.632519543170929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.02649536170065403,
      "rewards/margins": 0.03328552097082138,
      "rewards/rejected": -0.05982666090130806,
      "step": 11960
    },
    {
      "epoch": 0.8720358430772593,
      "grad_norm": 2.217485183041776,
      "learning_rate": 4.5700708092256815e-07,
      "log_odds_chosen": 1.3458983898162842,
      "log_odds_ratio": -0.3860839903354645,
      "logits/chosen": -0.968554675579071,
      "logits/rejected": -0.853710949420929,
      "logps/chosen": -0.5384765863418579,
      "logps/rejected": -1.369531273841858,
      "loss": 0.7207,
      "nll_loss": 0.685351550579071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.02692871168255806,
      "rewards/margins": 0.04152832180261612,
      "rewards/rejected": -0.06850586086511612,
      "step": 11970
    },
    {
      "epoch": 0.8727643609077332,
      "grad_norm": 1.9975378621520559,
      "learning_rate": 4.5681630358972045e-07,
      "log_odds_chosen": 1.221948266029358,
      "log_odds_ratio": -0.41508787870407104,
      "logits/chosen": -0.9599609375,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.4989257752895355,
      "logps/rejected": -1.188574194908142,
      "loss": 0.6955,
      "nll_loss": 0.658886730670929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.024932861328125,
      "rewards/margins": 0.034548185765743256,
      "rewards/rejected": -0.05953369289636612,
      "step": 11980
    },
    {
      "epoch": 0.8734928787382071,
      "grad_norm": 2.2261866054549677,
      "learning_rate": 4.5662576497722315e-07,
      "log_odds_chosen": 1.1337890625,
      "log_odds_ratio": -0.4410156309604645,
      "logits/chosen": -0.995312511920929,
      "logits/rejected": -0.912304699420929,
      "logps/chosen": -0.4837890565395355,
      "logps/rejected": -1.1115233898162842,
      "loss": 0.6933,
      "nll_loss": 0.6761718988418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.024200439453125,
      "rewards/margins": 0.03134269639849663,
      "rewards/rejected": -0.05558471754193306,
      "step": 11990
    },
    {
      "epoch": 0.874221396568681,
      "grad_norm": 2.4818526848428535,
      "learning_rate": 4.564354645876384e-07,
      "log_odds_chosen": 1.0205078125,
      "log_odds_ratio": -0.49101561307907104,
      "logits/chosen": -1.0185546875,
      "logits/rejected": -0.937695324420929,
      "logps/chosen": -0.546679675579071,
      "logps/rejected": -1.156640648841858,
      "loss": 0.688,
      "nll_loss": 0.6590820550918579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02736816368997097,
      "rewards/margins": 0.03043212927877903,
      "rewards/rejected": -0.05777587741613388,
      "step": 12000
    },
    {
      "epoch": 0.8749499143991549,
      "grad_norm": 2.635365690474959,
      "learning_rate": 4.562454019249787e-07,
      "log_odds_chosen": 1.0849120616912842,
      "log_odds_ratio": -0.4725097715854645,
      "logits/chosen": -1.0041015148162842,
      "logits/rejected": -0.9232422113418579,
      "logps/chosen": -0.517871081829071,
      "logps/rejected": -1.079687476158142,
      "loss": 0.6941,
      "nll_loss": 0.7392578125,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02586669847369194,
      "rewards/margins": 0.02811736986041069,
      "rewards/rejected": -0.05400390550494194,
      "step": 12010
    },
    {
      "epoch": 0.8756784322296288,
      "grad_norm": 2.166018635995961,
      "learning_rate": 4.5605557649470034e-07,
      "log_odds_chosen": 1.248046875,
      "log_odds_ratio": -0.4083496034145355,
      "logits/chosen": -0.9878906011581421,
      "logits/rejected": -0.8560546636581421,
      "logps/chosen": -0.5562499761581421,
      "logps/rejected": -1.3175780773162842,
      "loss": 0.7181,
      "nll_loss": 0.71044921875,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02784423902630806,
      "rewards/margins": 0.03806152194738388,
      "rewards/rejected": -0.06586913764476776,
      "step": 12020
    },
    {
      "epoch": 0.8764069500601027,
      "grad_norm": 2.328197247489465,
      "learning_rate": 4.558659878036996e-07,
      "log_odds_chosen": 1.4248046875,
      "log_odds_ratio": -0.3795410096645355,
      "logits/chosen": -0.9623047113418579,
      "logits/rejected": -0.8388671875,
      "logps/chosen": -0.4585937559604645,
      "logps/rejected": -1.3048827648162842,
      "loss": 0.6997,
      "nll_loss": 0.5868164300918579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.02293090894818306,
      "rewards/margins": 0.04230957105755806,
      "rewards/rejected": -0.065185546875,
      "step": 12030
    },
    {
      "epoch": 0.8771354678905766,
      "grad_norm": 2.3152228073685293,
      "learning_rate": 4.556766353603058e-07,
      "log_odds_chosen": 1.37744140625,
      "log_odds_ratio": -0.40473634004592896,
      "logits/chosen": -0.9736328125,
      "logits/rejected": -0.881054699420929,
      "logps/chosen": -0.52099609375,
      "logps/rejected": -1.3058593273162842,
      "loss": 0.7097,
      "nll_loss": 0.641894519329071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02607116661965847,
      "rewards/margins": 0.03916320949792862,
      "rewards/rejected": -0.06528930366039276,
      "step": 12040
    },
    {
      "epoch": 0.8778639857210505,
      "grad_norm": 2.441198222637589,
      "learning_rate": 4.5548751867427693e-07,
      "log_odds_chosen": 1.269677758216858,
      "log_odds_ratio": -0.43657225370407104,
      "logits/chosen": -0.9751952886581421,
      "logits/rejected": -0.8783203363418579,
      "logps/chosen": -0.534863293170929,
      "logps/rejected": -1.300195336341858,
      "loss": 0.6968,
      "nll_loss": 0.7289062738418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0267333984375,
      "rewards/margins": 0.03828735277056694,
      "rewards/rejected": -0.06502075493335724,
      "step": 12050
    },
    {
      "epoch": 0.8785925035515244,
      "grad_norm": 2.2039561185232657,
      "learning_rate": 4.552986372567942e-07,
      "log_odds_chosen": 1.140625,
      "log_odds_ratio": -0.42548829317092896,
      "logits/chosen": -0.9429687261581421,
      "logits/rejected": -0.8511718511581421,
      "logps/chosen": -0.517285168170929,
      "logps/rejected": -1.1666991710662842,
      "loss": 0.699,
      "nll_loss": 0.6591796875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02585449256002903,
      "rewards/margins": 0.03246307373046875,
      "rewards/rejected": -0.05833740159869194,
      "step": 12060
    },
    {
      "epoch": 0.8793210213819983,
      "grad_norm": 2.3537525505978323,
      "learning_rate": 4.5510999062045626e-07,
      "log_odds_chosen": 0.954968273639679,
      "log_odds_ratio": -0.45317381620407104,
      "logits/chosen": -0.9662109613418579,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -0.544921875,
      "logps/rejected": -1.0792968273162842,
      "loss": 0.714,
      "nll_loss": 0.6405273675918579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.027252197265625,
      "rewards/margins": 0.02669677697122097,
      "rewards/rejected": -0.05399169772863388,
      "step": 12070
    },
    {
      "epoch": 0.8800495392124722,
      "grad_norm": 3.7984842990175127,
      "learning_rate": 4.549215782792744e-07,
      "log_odds_chosen": 1.026611328125,
      "log_odds_ratio": -0.48486328125,
      "logits/chosen": -1.000585913658142,
      "logits/rejected": -0.8697265386581421,
      "logps/chosen": -0.560742199420929,
      "logps/rejected": -1.203710913658142,
      "loss": 0.6898,
      "nll_loss": 0.7281249761581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02802734449505806,
      "rewards/margins": 0.03217773512005806,
      "rewards/rejected": -0.06015624850988388,
      "step": 12080
    },
    {
      "epoch": 0.8807780570429461,
      "grad_norm": 2.3873224518554985,
      "learning_rate": 4.5473339974866706e-07,
      "log_odds_chosen": 1.046972632408142,
      "log_odds_ratio": -0.44990235567092896,
      "logits/chosen": -0.986132800579071,
      "logits/rejected": -0.8531249761581421,
      "logps/chosen": -0.5458008050918579,
      "logps/rejected": -1.1630859375,
      "loss": 0.7182,
      "nll_loss": 0.6922851800918579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02730102464556694,
      "rewards/margins": 0.03085632249712944,
      "rewards/rejected": -0.05812988430261612,
      "step": 12090
    },
    {
      "epoch": 0.88150657487342,
      "grad_norm": 2.9713975988440073,
      "learning_rate": 4.5454545454545457e-07,
      "log_odds_chosen": 1.0984375476837158,
      "log_odds_ratio": -0.44208985567092896,
      "logits/chosen": -0.94921875,
      "logits/rejected": -0.886523425579071,
      "logps/chosen": -0.5570312738418579,
      "logps/rejected": -1.218359351158142,
      "loss": 0.6999,
      "nll_loss": 0.716503918170929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.027862548828125,
      "rewards/margins": 0.03311462327837944,
      "rewards/rejected": -0.06097412109375,
      "step": 12100
    },
    {
      "epoch": 0.8822350927038939,
      "grad_norm": 2.616001928879527,
      "learning_rate": 4.543577421878543e-07,
      "log_odds_chosen": 1.08154296875,
      "log_odds_ratio": -0.497314453125,
      "logits/chosen": -0.9404296875,
      "logits/rejected": -0.8720703125,
      "logps/chosen": -0.541308581829071,
      "logps/rejected": -1.1896483898162842,
      "loss": 0.7064,
      "nll_loss": 0.7105468511581421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.027069091796875,
      "rewards/margins": 0.03248481824994087,
      "rewards/rejected": -0.05946044996380806,
      "step": 12110
    },
    {
      "epoch": 0.8829636105343678,
      "grad_norm": 2.622926707813214,
      "learning_rate": 4.541702621954747e-07,
      "log_odds_chosen": 1.1501953601837158,
      "log_odds_ratio": -0.438232421875,
      "logits/chosen": -0.942578136920929,
      "logits/rejected": -0.8734375238418579,
      "logps/chosen": -0.511035144329071,
      "logps/rejected": -1.1824219226837158,
      "loss": 0.7135,
      "nll_loss": 0.654492199420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.025543212890625,
      "rewards/margins": 0.03358917310833931,
      "rewards/rejected": -0.05913696438074112,
      "step": 12120
    },
    {
      "epoch": 0.8836921283648417,
      "grad_norm": 2.2825805211024672,
      "learning_rate": 4.5398301408931133e-07,
      "log_odds_chosen": 1.1812012195587158,
      "log_odds_ratio": -0.443359375,
      "logits/chosen": -1.011132836341858,
      "logits/rejected": -0.875195324420929,
      "logps/chosen": -0.51171875,
      "logps/rejected": -1.212499976158142,
      "loss": 0.7063,
      "nll_loss": 0.6756836175918579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02557983435690403,
      "rewards/margins": 0.03507671505212784,
      "rewards/rejected": -0.06061401218175888,
      "step": 12130
    },
    {
      "epoch": 0.8844206461953157,
      "grad_norm": 2.391312774137757,
      "learning_rate": 4.537959973917404e-07,
      "log_odds_chosen": 1.167688012123108,
      "log_odds_ratio": -0.41425782442092896,
      "logits/chosen": -1.003320336341858,
      "logits/rejected": -0.8955078125,
      "logps/chosen": -0.520800769329071,
      "logps/rejected": -1.235742211341858,
      "loss": 0.6707,
      "nll_loss": 0.635937511920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.026031494140625,
      "rewards/margins": 0.03576507419347763,
      "rewards/rejected": -0.06175537034869194,
      "step": 12140
    },
    {
      "epoch": 0.8851491640257896,
      "grad_norm": 2.4929851259971927,
      "learning_rate": 4.536092116265145e-07,
      "log_odds_chosen": 1.2439453601837158,
      "log_odds_ratio": -0.41743165254592896,
      "logits/chosen": -0.982617199420929,
      "logits/rejected": -0.865429699420929,
      "logps/chosen": -0.506152331829071,
      "logps/rejected": -1.2685546875,
      "loss": 0.7053,
      "nll_loss": 0.6722656488418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02531738206744194,
      "rewards/margins": 0.03813781589269638,
      "rewards/rejected": -0.06346435844898224,
      "step": 12150
    },
    {
      "epoch": 0.8858776818562635,
      "grad_norm": 2.5937725048543503,
      "learning_rate": 4.5342265631875734e-07,
      "log_odds_chosen": 1.140722632408142,
      "log_odds_ratio": -0.4659179747104645,
      "logits/chosen": -1.005468726158142,
      "logits/rejected": -0.911914050579071,
      "logps/chosen": -0.5501953363418579,
      "logps/rejected": -1.236328125,
      "loss": 0.7,
      "nll_loss": 0.6534179449081421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02752990648150444,
      "rewards/margins": 0.034360505640506744,
      "rewards/rejected": -0.06185302883386612,
      "step": 12160
    },
    {
      "epoch": 0.8866061996867374,
      "grad_norm": 2.7317567601990076,
      "learning_rate": 4.5323633099495855e-07,
      "log_odds_chosen": 1.0976073741912842,
      "log_odds_ratio": -0.435302734375,
      "logits/chosen": -0.9888671636581421,
      "logits/rejected": -0.8744140863418579,
      "logps/chosen": -0.5396484136581421,
      "logps/rejected": -1.1902344226837158,
      "loss": 0.6773,
      "nll_loss": 0.672070324420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02696533128619194,
      "rewards/margins": 0.03254546970129013,
      "rewards/rejected": -0.059539794921875,
      "step": 12170
    },
    {
      "epoch": 0.8873347175172113,
      "grad_norm": 2.2141110350013493,
      "learning_rate": 4.5305023518296865e-07,
      "log_odds_chosen": 0.8709961175918579,
      "log_odds_ratio": -0.5057617425918579,
      "logits/chosen": -1.003320336341858,
      "logits/rejected": -0.915820300579071,
      "logps/chosen": -0.5068359375,
      "logps/rejected": -0.9857422113418579,
      "loss": 0.7241,
      "nll_loss": 0.6387695074081421,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02532348595559597,
      "rewards/margins": 0.02397308312356472,
      "rewards/rejected": -0.04932861402630806,
      "step": 12180
    },
    {
      "epoch": 0.8880632353476852,
      "grad_norm": 2.2807967567470206,
      "learning_rate": 4.5286436841199424e-07,
      "log_odds_chosen": 1.0136840343475342,
      "log_odds_ratio": -0.466796875,
      "logits/chosen": -0.9755859375,
      "logits/rejected": -0.840039074420929,
      "logps/chosen": -0.53515625,
      "logps/rejected": -1.1531250476837158,
      "loss": 0.7043,
      "nll_loss": 0.7437499761581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02675781212747097,
      "rewards/margins": 0.03087158128619194,
      "rewards/rejected": -0.05764160305261612,
      "step": 12190
    },
    {
      "epoch": 0.8887917531781591,
      "grad_norm": 3.751913304853758,
      "learning_rate": 4.5267873021259267e-07,
      "log_odds_chosen": 0.8643554449081421,
      "log_odds_ratio": -0.4813476502895355,
      "logits/chosen": -1.044921875,
      "logits/rejected": -0.91015625,
      "logps/chosen": -0.5767577886581421,
      "logps/rejected": -1.09619140625,
      "loss": 0.7065,
      "nll_loss": 0.7177734375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02881469763815403,
      "rewards/margins": 0.02601470984518528,
      "rewards/rejected": -0.05477295070886612,
      "step": 12200
    },
    {
      "epoch": 0.889520271008633,
      "grad_norm": 2.7101750220915237,
      "learning_rate": 4.524933201166674e-07,
      "log_odds_chosen": 1.09375,
      "log_odds_ratio": -0.4330078065395355,
      "logits/chosen": -0.9947265386581421,
      "logits/rejected": -0.862500011920929,
      "logps/chosen": -0.5703125,
      "logps/rejected": -1.2185547351837158,
      "loss": 0.7229,
      "nll_loss": 0.7109375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02849121019244194,
      "rewards/margins": 0.03244934231042862,
      "rewards/rejected": -0.06096191331744194,
      "step": 12210
    },
    {
      "epoch": 0.8902487888391069,
      "grad_norm": 2.263266865706158,
      "learning_rate": 4.523081376574625e-07,
      "log_odds_chosen": 0.9583984613418579,
      "log_odds_ratio": -0.48486328125,
      "logits/chosen": -0.9833984375,
      "logits/rejected": -0.860546886920929,
      "logps/chosen": -0.56884765625,
      "logps/rejected": -1.149804711341858,
      "loss": 0.6886,
      "nll_loss": 0.6698242425918579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02843322791159153,
      "rewards/margins": 0.02909545972943306,
      "rewards/rejected": -0.05751953274011612,
      "step": 12220
    },
    {
      "epoch": 0.8909773066695807,
      "grad_norm": 2.083383440516753,
      "learning_rate": 4.5212318236955857e-07,
      "log_odds_chosen": 1.035180687904358,
      "log_odds_ratio": -0.49492186307907104,
      "logits/chosen": -0.9638671875,
      "logits/rejected": -0.8570312261581421,
      "logps/chosen": -0.595507800579071,
      "logps/rejected": -1.236328125,
      "loss": 0.6858,
      "nll_loss": 0.711718738079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02977294847369194,
      "rewards/margins": 0.03208465501666069,
      "rewards/rejected": -0.06184692308306694,
      "step": 12230
    },
    {
      "epoch": 0.8917058245000546,
      "grad_norm": 2.523867211494533,
      "learning_rate": 4.51938453788867e-07,
      "log_odds_chosen": 1.16363525390625,
      "log_odds_ratio": -0.43476563692092896,
      "logits/chosen": -1.0412108898162842,
      "logits/rejected": -0.8902343511581421,
      "logps/chosen": -0.535449206829071,
      "logps/rejected": -1.2488281726837158,
      "loss": 0.7031,
      "nll_loss": 0.668164074420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02677612379193306,
      "rewards/margins": 0.03560333326458931,
      "rewards/rejected": -0.06240234524011612,
      "step": 12240
    },
    {
      "epoch": 0.8924343423305285,
      "grad_norm": 2.2324273349241017,
      "learning_rate": 4.517539514526257e-07,
      "log_odds_chosen": 1.131445288658142,
      "log_odds_ratio": -0.451416015625,
      "logits/chosen": -0.984179675579071,
      "logits/rejected": -0.9105468988418579,
      "logps/chosen": -0.5693359375,
      "logps/rejected": -1.2257812023162842,
      "loss": 0.7119,
      "nll_loss": 0.687792956829071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02846679650247097,
      "rewards/margins": 0.03277282789349556,
      "rewards/rejected": -0.06129760667681694,
      "step": 12250
    },
    {
      "epoch": 0.8931628601610024,
      "grad_norm": 2.827443908633277,
      "learning_rate": 4.515696748993935e-07,
      "log_odds_chosen": 1.2550780773162842,
      "log_odds_ratio": -0.40668946504592896,
      "logits/chosen": -0.969531238079071,
      "logits/rejected": -0.846875011920929,
      "logps/chosen": -0.5130859613418579,
      "logps/rejected": -1.210351586341858,
      "loss": 0.6853,
      "nll_loss": 0.674511730670929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02564697340130806,
      "rewards/margins": 0.03491516038775444,
      "rewards/rejected": -0.06053466722369194,
      "step": 12260
    },
    {
      "epoch": 0.8938913779914763,
      "grad_norm": 2.2582048062993483,
      "learning_rate": 4.5138562366904615e-07,
      "log_odds_chosen": 1.2332031726837158,
      "log_odds_ratio": -0.41401368379592896,
      "logits/chosen": -1.0099608898162842,
      "logits/rejected": -0.876171886920929,
      "logps/chosen": -0.537402331829071,
      "logps/rejected": -1.2900390625,
      "loss": 0.7185,
      "nll_loss": 0.651660144329071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02687377855181694,
      "rewards/margins": 0.03763427585363388,
      "rewards/rejected": -0.06451416015625,
      "step": 12270
    },
    {
      "epoch": 0.8946198958219502,
      "grad_norm": 2.933876294017418,
      "learning_rate": 4.5120179730277105e-07,
      "log_odds_chosen": 1.0167968273162842,
      "log_odds_ratio": -0.49921876192092896,
      "logits/chosen": -0.983593761920929,
      "logits/rejected": -0.8619140386581421,
      "logps/chosen": -0.4825195372104645,
      "logps/rejected": -1.0886719226837158,
      "loss": 0.6939,
      "nll_loss": 0.6221679449081421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0241241455078125,
      "rewards/margins": 0.03026580810546875,
      "rewards/rejected": -0.054351806640625,
      "step": 12280
    },
    {
      "epoch": 0.8953484136524241,
      "grad_norm": 2.951437738324574,
      "learning_rate": 4.510181953430622e-07,
      "log_odds_chosen": 1.162695288658142,
      "log_odds_ratio": -0.41943359375,
      "logits/chosen": -0.9814453125,
      "logits/rejected": -0.8335937261581421,
      "logps/chosen": -0.513476550579071,
      "logps/rejected": -1.1964843273162842,
      "loss": 0.6902,
      "nll_loss": 0.664257824420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02567749097943306,
      "rewards/margins": 0.03414306789636612,
      "rewards/rejected": -0.05980224534869194,
      "step": 12290
    },
    {
      "epoch": 0.896076931482898,
      "grad_norm": 2.749522027961045,
      "learning_rate": 4.508348173337162e-07,
      "log_odds_chosen": 1.2165038585662842,
      "log_odds_ratio": -0.41728514432907104,
      "logits/chosen": -1.0154297351837158,
      "logits/rejected": -0.8714843988418579,
      "logps/chosen": -0.50146484375,
      "logps/rejected": -1.2478516101837158,
      "loss": 0.6913,
      "nll_loss": 0.584765613079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02509155310690403,
      "rewards/margins": 0.037293244153261185,
      "rewards/rejected": -0.06238403171300888,
      "step": 12300
    },
    {
      "epoch": 0.8968054493133719,
      "grad_norm": 2.91143277037206,
      "learning_rate": 4.5065166281982654e-07,
      "log_odds_chosen": 1.2722656726837158,
      "log_odds_ratio": -0.4154296815395355,
      "logits/chosen": -1.001953125,
      "logits/rejected": -0.876757800579071,
      "logps/chosen": -0.509082019329071,
      "logps/rejected": -1.298828125,
      "loss": 0.6839,
      "nll_loss": 0.605273425579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02544555626809597,
      "rewards/margins": 0.03944854810833931,
      "rewards/rejected": -0.06491699069738388,
      "step": 12310
    },
    {
      "epoch": 0.8975339671438458,
      "grad_norm": 2.687780211057373,
      "learning_rate": 4.504687313477795e-07,
      "log_odds_chosen": 1.1744384765625,
      "log_odds_ratio": -0.43437498807907104,
      "logits/chosen": -0.9917968511581421,
      "logits/rejected": -0.890820324420929,
      "logps/chosen": -0.555859386920929,
      "logps/rejected": -1.2302734851837158,
      "loss": 0.6986,
      "nll_loss": 0.7265625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02781372144818306,
      "rewards/margins": 0.03372497484087944,
      "rewards/rejected": -0.06154785305261612,
      "step": 12320
    },
    {
      "epoch": 0.8982624849743197,
      "grad_norm": 2.8629893174639682,
      "learning_rate": 4.502860224652494e-07,
      "log_odds_chosen": 0.963671863079071,
      "log_odds_ratio": -0.47148436307907104,
      "logits/chosen": -0.983203113079071,
      "logits/rejected": -0.900390625,
      "logps/chosen": -0.5596679449081421,
      "logps/rejected": -1.114648461341858,
      "loss": 0.6987,
      "nll_loss": 0.7320312261581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02797241136431694,
      "rewards/margins": 0.02770080603659153,
      "rewards/rejected": -0.05572509765625,
      "step": 12330
    },
    {
      "epoch": 0.8989910028047936,
      "grad_norm": 2.3190302693923286,
      "learning_rate": 4.5010353572119344e-07,
      "log_odds_chosen": 1.1062500476837158,
      "log_odds_ratio": -0.4500976502895355,
      "logits/chosen": -1.010351538658142,
      "logits/rejected": -0.908984363079071,
      "logps/chosen": -0.552050769329071,
      "logps/rejected": -1.2439453601837158,
      "loss": 0.6876,
      "nll_loss": 0.685351550579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02761230431497097,
      "rewards/margins": 0.03458862379193306,
      "rewards/rejected": -0.06223144382238388,
      "step": 12340
    },
    {
      "epoch": 0.8997195206352675,
      "grad_norm": 2.3774608866692444,
      "learning_rate": 4.4992127066584757e-07,
      "log_odds_chosen": 1.195947289466858,
      "log_odds_ratio": -0.4127441346645355,
      "logits/chosen": -0.999218761920929,
      "logits/rejected": -0.890820324420929,
      "logps/chosen": -0.52783203125,
      "logps/rejected": -1.254492163658142,
      "loss": 0.6971,
      "nll_loss": 0.672070324420929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.02639160118997097,
      "rewards/margins": 0.03634338453412056,
      "rewards/rejected": -0.06267090141773224,
      "step": 12350
    },
    {
      "epoch": 0.9004480384657414,
      "grad_norm": 2.39833175319709,
      "learning_rate": 4.497392268507216e-07,
      "log_odds_chosen": 1.137109398841858,
      "log_odds_ratio": -0.43769532442092896,
      "logits/chosen": -0.9658203125,
      "logits/rejected": -0.8626953363418579,
      "logps/chosen": -0.5040038824081421,
      "logps/rejected": -1.1232421398162842,
      "loss": 0.69,
      "nll_loss": 0.6719726324081421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02521972730755806,
      "rewards/margins": 0.03099517896771431,
      "rewards/rejected": -0.05618896335363388,
      "step": 12360
    },
    {
      "epoch": 0.9011765562962154,
      "grad_norm": 2.4897104979935576,
      "learning_rate": 4.4955740382859443e-07,
      "log_odds_chosen": 1.037512183189392,
      "log_odds_ratio": -0.4769043028354645,
      "logits/chosen": -1.0281250476837158,
      "logits/rejected": -0.876953125,
      "logps/chosen": -0.5477539300918579,
      "logps/rejected": -1.1414062976837158,
      "loss": 0.7042,
      "nll_loss": 0.6884765625,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02736816368997097,
      "rewards/margins": 0.029708098620176315,
      "rewards/rejected": -0.05710449069738388,
      "step": 12370
    },
    {
      "epoch": 0.9019050741266893,
      "grad_norm": 2.0739838026425406,
      "learning_rate": 4.493758011535097e-07,
      "log_odds_chosen": 1.016992211341858,
      "log_odds_ratio": -0.4555419981479645,
      "logits/chosen": -1.001562476158142,
      "logits/rejected": -0.8798828125,
      "logps/chosen": -0.567675769329071,
      "logps/rejected": -1.192773461341858,
      "loss": 0.6947,
      "nll_loss": 0.681640625,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02838745154440403,
      "rewards/margins": 0.03128356859087944,
      "rewards/rejected": -0.05965576320886612,
      "step": 12380
    },
    {
      "epoch": 0.9026335919571632,
      "grad_norm": 2.277733871727454,
      "learning_rate": 4.4919441838077096e-07,
      "log_odds_chosen": 1.386328101158142,
      "log_odds_ratio": -0.38823240995407104,
      "logits/chosen": -1.013671875,
      "logits/rejected": -0.9345703125,
      "logps/chosen": -0.548046886920929,
      "logps/rejected": -1.4113280773162842,
      "loss": 0.6983,
      "nll_loss": 0.6533203125,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.02739868126809597,
      "rewards/margins": 0.04320983961224556,
      "rewards/rejected": -0.07062987983226776,
      "step": 12390
    },
    {
      "epoch": 0.9033621097876371,
      "grad_norm": 2.5736133790521847,
      "learning_rate": 4.4901325506693733e-07,
      "log_odds_chosen": 1.0879395008087158,
      "log_odds_ratio": -0.44169920682907104,
      "logits/chosen": -0.953320324420929,
      "logits/rejected": -0.8539062738418579,
      "logps/chosen": -0.533398449420929,
      "logps/rejected": -1.168554663658142,
      "loss": 0.6741,
      "nll_loss": 0.648242175579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02666625939309597,
      "rewards/margins": 0.03179626539349556,
      "rewards/rejected": -0.05844726413488388,
      "step": 12400
    },
    {
      "epoch": 0.904090627618111,
      "grad_norm": 2.7893674240101185,
      "learning_rate": 4.4883231076981863e-07,
      "log_odds_chosen": 1.24609375,
      "log_odds_ratio": -0.41865235567092896,
      "logits/chosen": -0.9951171875,
      "logits/rejected": -0.8994140625,
      "logps/chosen": -0.53369140625,
      "logps/rejected": -1.273046851158142,
      "loss": 0.7171,
      "nll_loss": 0.6817382574081421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02667236328125,
      "rewards/margins": 0.03693847730755806,
      "rewards/rejected": -0.06362304836511612,
      "step": 12410
    },
    {
      "epoch": 0.9048191454485849,
      "grad_norm": 2.310809536502376,
      "learning_rate": 4.4865158504847126e-07,
      "log_odds_chosen": 1.165136694908142,
      "log_odds_ratio": -0.4134277403354645,
      "logits/chosen": -0.9947265386581421,
      "logits/rejected": -0.9095703363418579,
      "logps/chosen": -0.4951171875,
      "logps/rejected": -1.166406273841858,
      "loss": 0.6882,
      "nll_loss": 0.647265613079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.024749755859375,
      "rewards/margins": 0.03358154371380806,
      "rewards/rejected": -0.05827636644244194,
      "step": 12420
    },
    {
      "epoch": 0.9055476632790588,
      "grad_norm": 2.290739160727941,
      "learning_rate": 4.484710774631934e-07,
      "log_odds_chosen": 1.418554663658142,
      "log_odds_ratio": -0.3698486387729645,
      "logits/chosen": -0.966796875,
      "logits/rejected": -0.8568359613418579,
      "logps/chosen": -0.5361328125,
      "logps/rejected": -1.377343773841858,
      "loss": 0.7099,
      "nll_loss": 0.690136730670929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.02681884728372097,
      "rewards/margins": 0.042082976549863815,
      "rewards/rejected": -0.06887207180261612,
      "step": 12430
    },
    {
      "epoch": 0.9062761811095327,
      "grad_norm": 2.372598964955632,
      "learning_rate": 4.482907875755204e-07,
      "log_odds_chosen": 1.196874976158142,
      "log_odds_ratio": -0.41206055879592896,
      "logits/chosen": -0.9466797113418579,
      "logits/rejected": -0.862109363079071,
      "logps/chosen": -0.5098632574081421,
      "logps/rejected": -1.1941406726837158,
      "loss": 0.7125,
      "nll_loss": 0.658984363079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.025482177734375,
      "rewards/margins": 0.03422241285443306,
      "rewards/rejected": -0.05971679836511612,
      "step": 12440
    },
    {
      "epoch": 0.9070046989400066,
      "grad_norm": 2.6263398674127707,
      "learning_rate": 4.481107149482209e-07,
      "log_odds_chosen": 1.13671875,
      "log_odds_ratio": -0.4242187440395355,
      "logits/chosen": -0.9384765625,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.5684570074081421,
      "logps/rejected": -1.2374999523162842,
      "loss": 0.7067,
      "nll_loss": 0.7012695074081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02843627892434597,
      "rewards/margins": 0.03345642238855362,
      "rewards/rejected": -0.06187744066119194,
      "step": 12450
    },
    {
      "epoch": 0.9077332167704805,
      "grad_norm": 2.323678011658644,
      "learning_rate": 4.479308591452914e-07,
      "log_odds_chosen": 1.186669945716858,
      "log_odds_ratio": -0.4434570372104645,
      "logits/chosen": -0.986132800579071,
      "logits/rejected": -0.8685547113418579,
      "logps/chosen": -0.510449230670929,
      "logps/rejected": -1.224218726158142,
      "loss": 0.6779,
      "nll_loss": 0.664257824420929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.02554016187787056,
      "rewards/margins": 0.03571166843175888,
      "rewards/rejected": -0.06121826171875,
      "step": 12460
    },
    {
      "epoch": 0.9084617346009544,
      "grad_norm": 2.1127935863180785,
      "learning_rate": 4.477512197319529e-07,
      "log_odds_chosen": 1.126123070716858,
      "log_odds_ratio": -0.43798828125,
      "logits/chosen": -1.004492163658142,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.5582031011581421,
      "logps/rejected": -1.2287108898162842,
      "loss": 0.6901,
      "nll_loss": 0.666015625,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02791748009622097,
      "rewards/margins": 0.0334930419921875,
      "rewards/rejected": -0.06142578274011612,
      "step": 12470
    },
    {
      "epoch": 0.9091902524314283,
      "grad_norm": 2.6564659206575794,
      "learning_rate": 4.475717962746456e-07,
      "log_odds_chosen": 1.1530272960662842,
      "log_odds_ratio": -0.44501954317092896,
      "logits/chosen": -0.988085925579071,
      "logits/rejected": -0.856640636920929,
      "logps/chosen": -0.5249999761581421,
      "logps/rejected": -1.1826171875,
      "loss": 0.7026,
      "nll_loss": 0.695117175579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02626953087747097,
      "rewards/margins": 0.03284301608800888,
      "rewards/rejected": -0.05913085862994194,
      "step": 12480
    },
    {
      "epoch": 0.9099187702619022,
      "grad_norm": 2.6963011636318526,
      "learning_rate": 4.4739258834102515e-07,
      "log_odds_chosen": 1.116455078125,
      "log_odds_ratio": -0.45751953125,
      "logits/chosen": -0.9927734136581421,
      "logits/rejected": -0.883984386920929,
      "logps/chosen": -0.5240234136581421,
      "logps/rejected": -1.203515648841858,
      "loss": 0.71,
      "nll_loss": 0.6690429449081421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02622680738568306,
      "rewards/margins": 0.03397522121667862,
      "rewards/rejected": -0.06015624850988388,
      "step": 12490
    },
    {
      "epoch": 0.910647288092376,
      "grad_norm": 2.427514840215125,
      "learning_rate": 4.47213595499958e-07,
      "log_odds_chosen": 1.0350463390350342,
      "log_odds_ratio": -0.46406251192092896,
      "logits/chosen": -1.0109374523162842,
      "logits/rejected": -0.9095703363418579,
      "logps/chosen": -0.5328124761581421,
      "logps/rejected": -1.1572265625,
      "loss": 0.7059,
      "nll_loss": 0.673535168170929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02662353590130806,
      "rewards/margins": 0.03126678615808487,
      "rewards/rejected": -0.057861328125,
      "step": 12500
    },
    {
      "epoch": 0.91137580592285,
      "grad_norm": 2.522091766355448,
      "learning_rate": 4.4703481732151677e-07,
      "log_odds_chosen": 1.3126952648162842,
      "log_odds_ratio": -0.413818359375,
      "logits/chosen": -0.9847656488418579,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.550000011920929,
      "logps/rejected": -1.3525390625,
      "loss": 0.6713,
      "nll_loss": 0.717578113079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.027496337890625,
      "rewards/margins": 0.04012451320886612,
      "rewards/rejected": -0.06761474907398224,
      "step": 12510
    },
    {
      "epoch": 0.9121043237533238,
      "grad_norm": 6.3913292214101505,
      "learning_rate": 4.468562533769765e-07,
      "log_odds_chosen": 1.322851538658142,
      "log_odds_ratio": -0.43828123807907104,
      "logits/chosen": -0.9873046875,
      "logits/rejected": -0.848828136920929,
      "logps/chosen": -0.5370117425918579,
      "logps/rejected": -1.343359351158142,
      "loss": 0.7038,
      "nll_loss": 0.670117199420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02682190015912056,
      "rewards/margins": 0.04036559909582138,
      "rewards/rejected": -0.06718750298023224,
      "step": 12520
    },
    {
      "epoch": 0.9128328415837977,
      "grad_norm": 2.4629238641482076,
      "learning_rate": 4.466779032388099e-07,
      "log_odds_chosen": 1.135278344154358,
      "log_odds_ratio": -0.456298828125,
      "logits/chosen": -0.9810546636581421,
      "logits/rejected": -0.873046875,
      "logps/chosen": -0.5615234375,
      "logps/rejected": -1.2716796398162842,
      "loss": 0.7073,
      "nll_loss": 0.6558593511581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02808837965130806,
      "rewards/margins": 0.035446930676698685,
      "rewards/rejected": -0.06356201320886612,
      "step": 12530
    },
    {
      "epoch": 0.9135613594142716,
      "grad_norm": 2.5225240246058065,
      "learning_rate": 4.464997664806832e-07,
      "log_odds_chosen": 1.1828124523162842,
      "log_odds_ratio": -0.4423828125,
      "logits/chosen": -0.990234375,
      "logits/rejected": -0.8837890625,
      "logps/chosen": -0.55517578125,
      "logps/rejected": -1.265039086341858,
      "loss": 0.6974,
      "nll_loss": 0.652636706829071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02776489220559597,
      "rewards/margins": 0.03545989841222763,
      "rewards/rejected": -0.06325683742761612,
      "step": 12540
    },
    {
      "epoch": 0.9142898772447455,
      "grad_norm": 2.6087167368090003,
      "learning_rate": 4.463218426774518e-07,
      "log_odds_chosen": 1.1829102039337158,
      "log_odds_ratio": -0.46064454317092896,
      "logits/chosen": -0.9820312261581421,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.5225585699081421,
      "logps/rejected": -1.2550780773162842,
      "loss": 0.6902,
      "nll_loss": 0.7037109136581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02613525465130806,
      "rewards/margins": 0.03663025051355362,
      "rewards/rejected": -0.06273193657398224,
      "step": 12550
    },
    {
      "epoch": 0.9150183950752194,
      "grad_norm": 2.2523447494640063,
      "learning_rate": 4.461441314051561e-07,
      "log_odds_chosen": 1.093359351158142,
      "log_odds_ratio": -0.46782225370407104,
      "logits/chosen": -1.026953101158142,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.5462890863418579,
      "logps/rejected": -1.2236328125,
      "loss": 0.6993,
      "nll_loss": 0.6690429449081421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02730102464556694,
      "rewards/margins": 0.03386840969324112,
      "rewards/rejected": -0.06119384616613388,
      "step": 12560
    },
    {
      "epoch": 0.9157469129056933,
      "grad_norm": 2.220800013537966,
      "learning_rate": 4.4596663224101716e-07,
      "log_odds_chosen": 1.17578125,
      "log_odds_ratio": -0.41845703125,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.862500011920929,
      "logps/chosen": -0.5589843988418579,
      "logps/rejected": -1.2443358898162842,
      "loss": 0.6982,
      "nll_loss": 0.7393554449081421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.0279541015625,
      "rewards/margins": 0.03428344801068306,
      "rewards/rejected": -0.06220703199505806,
      "step": 12570
    },
    {
      "epoch": 0.9164754307361672,
      "grad_norm": 3.1243722500059206,
      "learning_rate": 4.457893447634325e-07,
      "log_odds_chosen": 1.2507812976837158,
      "log_odds_ratio": -0.41796875,
      "logits/chosen": -0.9771484136581421,
      "logits/rejected": -0.8837890625,
      "logps/chosen": -0.506054699420929,
      "logps/rejected": -1.1857421398162842,
      "loss": 0.7044,
      "nll_loss": 0.673046886920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02529602125287056,
      "rewards/margins": 0.033935546875,
      "rewards/rejected": -0.05921630933880806,
      "step": 12580
    },
    {
      "epoch": 0.9172039485666411,
      "grad_norm": 2.5763298501031624,
      "learning_rate": 4.456122685519722e-07,
      "log_odds_chosen": 1.126855492591858,
      "log_odds_ratio": -0.4286132752895355,
      "logits/chosen": -1.005859375,
      "logits/rejected": -0.901562511920929,
      "logps/chosen": -0.570605456829071,
      "logps/rejected": -1.2412109375,
      "loss": 0.7032,
      "nll_loss": 0.7081054449081421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02851562574505806,
      "rewards/margins": 0.03352966159582138,
      "rewards/rejected": -0.06204833835363388,
      "step": 12590
    },
    {
      "epoch": 0.9179324663971151,
      "grad_norm": 2.5259666377306003,
      "learning_rate": 4.45435403187374e-07,
      "log_odds_chosen": 1.3146483898162842,
      "log_odds_ratio": -0.39677733182907104,
      "logits/chosen": -1.014257788658142,
      "logits/rejected": -0.8716796636581421,
      "logps/chosen": -0.5248047113418579,
      "logps/rejected": -1.289648413658142,
      "loss": 0.6821,
      "nll_loss": 0.650390625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.02625122107565403,
      "rewards/margins": 0.0382232666015625,
      "rewards/rejected": -0.06446532905101776,
      "step": 12600
    },
    {
      "epoch": 0.918660984227589,
      "grad_norm": 2.312080637642262,
      "learning_rate": 4.452587482515399e-07,
      "log_odds_chosen": 1.143164038658142,
      "log_odds_ratio": -0.42719727754592896,
      "logits/chosen": -1.046484351158142,
      "logits/rejected": -0.9208984375,
      "logps/chosen": -0.53173828125,
      "logps/rejected": -1.212304711341858,
      "loss": 0.6932,
      "nll_loss": 0.653515636920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02656860277056694,
      "rewards/margins": 0.03403015062212944,
      "rewards/rejected": -0.06059570237994194,
      "step": 12610
    },
    {
      "epoch": 0.9193895020580629,
      "grad_norm": 2.6418399103363113,
      "learning_rate": 4.450823033275315e-07,
      "log_odds_chosen": 1.1222655773162842,
      "log_odds_ratio": -0.4393554627895355,
      "logits/chosen": -1.02734375,
      "logits/rejected": -0.8896484375,
      "logps/chosen": -0.54248046875,
      "logps/rejected": -1.2306640148162842,
      "loss": 0.7129,
      "nll_loss": 0.621777355670929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02712402306497097,
      "rewards/margins": 0.03437500074505806,
      "rewards/rejected": -0.06149902194738388,
      "step": 12620
    },
    {
      "epoch": 0.9201180198885368,
      "grad_norm": 2.79005034388128,
      "learning_rate": 4.449060679995661e-07,
      "log_odds_chosen": 1.256445288658142,
      "log_odds_ratio": -0.413818359375,
      "logits/chosen": -0.979296863079071,
      "logits/rejected": -0.8968750238418579,
      "logps/chosen": -0.5440429449081421,
      "logps/rejected": -1.272070288658142,
      "loss": 0.6953,
      "nll_loss": 0.6670898199081421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.027191162109375,
      "rewards/margins": 0.03648681566119194,
      "rewards/rejected": -0.06361083686351776,
      "step": 12630
    },
    {
      "epoch": 0.9208465377190107,
      "grad_norm": 2.3632256520225456,
      "learning_rate": 4.4473004185301255e-07,
      "log_odds_chosen": 1.264550805091858,
      "log_odds_ratio": -0.42011719942092896,
      "logits/chosen": -0.9574218988418579,
      "logits/rejected": -0.845703125,
      "logps/chosen": -0.534863293170929,
      "logps/rejected": -1.281835913658142,
      "loss": 0.6922,
      "nll_loss": 0.724316418170929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02672119066119194,
      "rewards/margins": 0.037375640124082565,
      "rewards/rejected": -0.06406249850988388,
      "step": 12640
    },
    {
      "epoch": 0.9215750555494846,
      "grad_norm": 2.2903462247514472,
      "learning_rate": 4.4455422447438707e-07,
      "log_odds_chosen": 1.255859375,
      "log_odds_ratio": -0.42265623807907104,
      "logits/chosen": -1.00390625,
      "logits/rejected": -0.9154297113418579,
      "logps/chosen": -0.5707031488418579,
      "logps/rejected": -1.3525390625,
      "loss": 0.707,
      "nll_loss": 0.730664074420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02855834923684597,
      "rewards/margins": 0.03912658616900444,
      "rewards/rejected": -0.06767578423023224,
      "step": 12650
    },
    {
      "epoch": 0.9223035733799585,
      "grad_norm": 2.700498376546593,
      "learning_rate": 4.443786154513493e-07,
      "log_odds_chosen": 1.140722632408142,
      "log_odds_ratio": -0.44501954317092896,
      "logits/chosen": -1.0207030773162842,
      "logits/rejected": -0.8988281488418579,
      "logps/chosen": -0.549609363079071,
      "logps/rejected": -1.2517578601837158,
      "loss": 0.7073,
      "nll_loss": 0.6998046636581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02749023400247097,
      "rewards/margins": 0.035025786608457565,
      "rewards/rejected": -0.06253661960363388,
      "step": 12660
    },
    {
      "epoch": 0.9230320912104324,
      "grad_norm": 2.197637771599495,
      "learning_rate": 4.4420321437269797e-07,
      "log_odds_chosen": 1.098535180091858,
      "log_odds_ratio": -0.4661621153354645,
      "logits/chosen": -0.970898449420929,
      "logits/rejected": -0.8880859613418579,
      "logps/chosen": -0.53515625,
      "logps/rejected": -1.1619141101837158,
      "loss": 0.6875,
      "nll_loss": 0.69287109375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02676696702837944,
      "rewards/margins": 0.03132019191980362,
      "rewards/rejected": -0.05809326097369194,
      "step": 12670
    },
    {
      "epoch": 0.9237606090409063,
      "grad_norm": 2.6194997266893183,
      "learning_rate": 4.4402802082836747e-07,
      "log_odds_chosen": 1.1499512195587158,
      "log_odds_ratio": -0.4803710877895355,
      "logits/chosen": -0.9994140863418579,
      "logits/rejected": -0.849414050579071,
      "logps/chosen": -0.5599609613418579,
      "logps/rejected": -1.265625,
      "loss": 0.6901,
      "nll_loss": 0.646289050579071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02800292894244194,
      "rewards/margins": 0.03526916354894638,
      "rewards/rejected": -0.06326904147863388,
      "step": 12680
    },
    {
      "epoch": 0.9244891268713802,
      "grad_norm": 2.788471052554246,
      "learning_rate": 4.4385303440942295e-07,
      "log_odds_chosen": 1.2794921398162842,
      "log_odds_ratio": -0.42963868379592896,
      "logits/chosen": -0.9720703363418579,
      "logits/rejected": -0.907031238079071,
      "logps/chosen": -0.5189453363418579,
      "logps/rejected": -1.2443358898162842,
      "loss": 0.6816,
      "nll_loss": 0.6519531011581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02596435509622097,
      "rewards/margins": 0.0362548828125,
      "rewards/rejected": -0.06224365159869194,
      "step": 12690
    },
    {
      "epoch": 0.9252176447018541,
      "grad_norm": 4.028108753727416,
      "learning_rate": 4.4367825470805695e-07,
      "log_odds_chosen": 1.347265601158142,
      "log_odds_ratio": -0.40888673067092896,
      "logits/chosen": -0.992382824420929,
      "logits/rejected": -0.8525390625,
      "logps/chosen": -0.53515625,
      "logps/rejected": -1.3259766101837158,
      "loss": 0.6856,
      "nll_loss": 0.6673828363418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02678832970559597,
      "rewards/margins": 0.03952331468462944,
      "rewards/rejected": -0.06629638373851776,
      "step": 12700
    },
    {
      "epoch": 0.925946162532328,
      "grad_norm": 2.2690438628970715,
      "learning_rate": 4.4350368131758535e-07,
      "log_odds_chosen": 1.242773413658142,
      "log_odds_ratio": -0.4127441346645355,
      "logits/chosen": -0.965624988079071,
      "logits/rejected": -0.854296863079071,
      "logps/chosen": -0.5458008050918579,
      "logps/rejected": -1.294921875,
      "loss": 0.7144,
      "nll_loss": 0.638378918170929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.02727661095559597,
      "rewards/margins": 0.03747100755572319,
      "rewards/rejected": -0.06473388522863388,
      "step": 12710
    },
    {
      "epoch": 0.9266746803628019,
      "grad_norm": 3.784391149550131,
      "learning_rate": 4.4332931383244295e-07,
      "log_odds_chosen": 1.2412109375,
      "log_odds_ratio": -0.41826170682907104,
      "logits/chosen": -1.0041015148162842,
      "logits/rejected": -0.836132824420929,
      "logps/chosen": -0.5234375,
      "logps/rejected": -1.296289086341858,
      "loss": 0.6951,
      "nll_loss": 0.6640625,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02617187425494194,
      "rewards/margins": 0.03857726976275444,
      "rewards/rejected": -0.06474609673023224,
      "step": 12720
    },
    {
      "epoch": 0.9274031981932758,
      "grad_norm": 2.3671531704892383,
      "learning_rate": 4.4315515184818025e-07,
      "log_odds_chosen": 1.2799804210662842,
      "log_odds_ratio": -0.4366699159145355,
      "logits/chosen": -1.009765625,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.5438476800918579,
      "logps/rejected": -1.308984398841858,
      "loss": 0.686,
      "nll_loss": 0.6749023199081421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.027191162109375,
      "rewards/margins": 0.03829803317785263,
      "rewards/rejected": -0.06546630710363388,
      "step": 12730
    },
    {
      "epoch": 0.9281317160237497,
      "grad_norm": 3.4206511137563376,
      "learning_rate": 4.4298119496145874e-07,
      "log_odds_chosen": 1.2517821788787842,
      "log_odds_ratio": -0.4242187440395355,
      "logits/chosen": -0.995312511920929,
      "logits/rejected": -0.8863281011581421,
      "logps/chosen": -0.51025390625,
      "logps/rejected": -1.2576172351837158,
      "loss": 0.7026,
      "nll_loss": 0.6441406011581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.0255126953125,
      "rewards/margins": 0.03737487643957138,
      "rewards/rejected": -0.06289062649011612,
      "step": 12740
    },
    {
      "epoch": 0.9288602338542236,
      "grad_norm": 4.113786414107207,
      "learning_rate": 4.428074427700477e-07,
      "log_odds_chosen": 1.1786377429962158,
      "log_odds_ratio": -0.4501953125,
      "logits/chosen": -0.973828136920929,
      "logits/rejected": -0.8765624761581421,
      "logps/chosen": -0.49775391817092896,
      "logps/rejected": -1.205078125,
      "loss": 0.6882,
      "nll_loss": 0.6395508050918579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02489013597369194,
      "rewards/margins": 0.035349272191524506,
      "rewards/rejected": -0.06028442457318306,
      "step": 12750
    },
    {
      "epoch": 0.9295887516846975,
      "grad_norm": 3.1579891246778025,
      "learning_rate": 4.4263389487281947e-07,
      "log_odds_chosen": 1.1960937976837158,
      "log_odds_ratio": -0.431640625,
      "logits/chosen": -0.9791015386581421,
      "logits/rejected": -0.859570324420929,
      "logps/chosen": -0.5057617425918579,
      "logps/rejected": -1.212499976158142,
      "loss": 0.6946,
      "nll_loss": 0.6016601324081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.025299072265625,
      "rewards/margins": 0.03535766527056694,
      "rewards/rejected": -0.06068115308880806,
      "step": 12760
    },
    {
      "epoch": 0.9303172695151714,
      "grad_norm": 3.1510223442246903,
      "learning_rate": 4.4246055086974623e-07,
      "log_odds_chosen": 1.09197998046875,
      "log_odds_ratio": -0.4942871034145355,
      "logits/chosen": -0.9609375,
      "logits/rejected": -0.8759765625,
      "logps/chosen": -0.5099121332168579,
      "logps/rejected": -1.2062499523162842,
      "loss": 0.6833,
      "nll_loss": 0.6494140625,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02551879920065403,
      "rewards/margins": 0.03479919582605362,
      "rewards/rejected": -0.060302734375,
      "step": 12770
    },
    {
      "epoch": 0.9310457873456452,
      "grad_norm": 2.3194024275458123,
      "learning_rate": 4.4228741036189615e-07,
      "log_odds_chosen": 1.213476538658142,
      "log_odds_ratio": -0.4188476502895355,
      "logits/chosen": -0.9798828363418579,
      "logits/rejected": -0.857617199420929,
      "logps/chosen": -0.53955078125,
      "logps/rejected": -1.2822265625,
      "loss": 0.674,
      "nll_loss": 0.6709960699081421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02698974683880806,
      "rewards/margins": 0.03707580640912056,
      "rewards/rejected": -0.06411132961511612,
      "step": 12780
    },
    {
      "epoch": 0.9317743051761191,
      "grad_norm": 2.5882357682722676,
      "learning_rate": 4.421144729514288e-07,
      "log_odds_chosen": 1.308984398841858,
      "log_odds_ratio": -0.4195800721645355,
      "logits/chosen": -1.0076172351837158,
      "logits/rejected": -0.896679699420929,
      "logps/chosen": -0.51416015625,
      "logps/rejected": -1.317968726158142,
      "loss": 0.6734,
      "nll_loss": 0.615039050579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02572021447122097,
      "rewards/margins": 0.04015503078699112,
      "rewards/rejected": -0.06585693359375,
      "step": 12790
    },
    {
      "epoch": 0.932502823006593,
      "grad_norm": 9.224187032568574,
      "learning_rate": 4.419417382415922e-07,
      "log_odds_chosen": 1.2734375,
      "log_odds_ratio": -0.390380859375,
      "logits/chosen": -0.9541015625,
      "logits/rejected": -0.8501952886581421,
      "logps/chosen": -0.52734375,
      "logps/rejected": -1.2517578601837158,
      "loss": 0.7055,
      "nll_loss": 0.6751953363418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.026397705078125,
      "rewards/margins": 0.03616943210363388,
      "rewards/rejected": -0.06256103515625,
      "step": 12800
    },
    {
      "epoch": 0.9332313408370669,
      "grad_norm": 2.5140408785616137,
      "learning_rate": 4.4176920583671857e-07,
      "log_odds_chosen": 1.2414062023162842,
      "log_odds_ratio": -0.4268554747104645,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.5257812738418579,
      "logps/rejected": -1.248437523841858,
      "loss": 0.6955,
      "nll_loss": 0.678027331829071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02626342698931694,
      "rewards/margins": 0.03611450269818306,
      "rewards/rejected": -0.06239013746380806,
      "step": 12810
    },
    {
      "epoch": 0.9339598586675409,
      "grad_norm": 3.337050589159645,
      "learning_rate": 4.415968753422204e-07,
      "log_odds_chosen": 1.33203125,
      "log_odds_ratio": -0.3829589784145355,
      "logits/chosen": -0.9664062261581421,
      "logits/rejected": -0.8773437738418579,
      "logps/chosen": -0.5077148675918579,
      "logps/rejected": -1.2878906726837158,
      "loss": 0.6781,
      "nll_loss": 0.654296875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.025390625,
      "rewards/margins": 0.03894958645105362,
      "rewards/rejected": -0.06431885063648224,
      "step": 12820
    },
    {
      "epoch": 0.9346883764980148,
      "grad_norm": 2.241618735749779,
      "learning_rate": 4.414247463645868e-07,
      "log_odds_chosen": 1.241113305091858,
      "log_odds_ratio": -0.42529296875,
      "logits/chosen": -0.9906250238418579,
      "logits/rejected": -0.8792968988418579,
      "logps/chosen": -0.5208984613418579,
      "logps/rejected": -1.248046875,
      "loss": 0.697,
      "nll_loss": 0.631542980670929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.026031494140625,
      "rewards/margins": 0.036346435546875,
      "rewards/rejected": -0.06242675706744194,
      "step": 12830
    },
    {
      "epoch": 0.9354168943284887,
      "grad_norm": 3.3831591510150254,
      "learning_rate": 4.4125281851137985e-07,
      "log_odds_chosen": 1.120751976966858,
      "log_odds_ratio": -0.43876951932907104,
      "logits/chosen": -1.018945336341858,
      "logits/rejected": -0.877148449420929,
      "logps/chosen": -0.5313476324081421,
      "logps/rejected": -1.194726586341858,
      "loss": 0.7174,
      "nll_loss": 0.6900390386581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02656250074505806,
      "rewards/margins": 0.03318176418542862,
      "rewards/rejected": -0.05976562574505806,
      "step": 12840
    },
    {
      "epoch": 0.9361454121589626,
      "grad_norm": 2.424136386523423,
      "learning_rate": 4.4108109139123093e-07,
      "log_odds_chosen": 1.164160132408142,
      "log_odds_ratio": -0.44062501192092896,
      "logits/chosen": -1.033203125,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.521289050579071,
      "logps/rejected": -1.219335913658142,
      "loss": 0.6774,
      "nll_loss": 0.661328136920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02607421949505806,
      "rewards/margins": 0.03488311916589737,
      "rewards/rejected": -0.06097412109375,
      "step": 12850
    },
    {
      "epoch": 0.9368739299894365,
      "grad_norm": 2.4347180446093057,
      "learning_rate": 4.409095646138363e-07,
      "log_odds_chosen": 0.9904419183731079,
      "log_odds_ratio": -0.48066407442092896,
      "logits/chosen": -0.9580078125,
      "logits/rejected": -0.849804699420929,
      "logps/chosen": -0.55126953125,
      "logps/rejected": -1.12109375,
      "loss": 0.697,
      "nll_loss": 0.696582019329071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02756042405962944,
      "rewards/margins": 0.02847595140337944,
      "rewards/rejected": -0.05599365383386612,
      "step": 12860
    },
    {
      "epoch": 0.9376024478199104,
      "grad_norm": 2.3549369764621666,
      "learning_rate": 4.4073823778995433e-07,
      "log_odds_chosen": 1.2464721202850342,
      "log_odds_ratio": -0.417236328125,
      "logits/chosen": -0.973828136920929,
      "logits/rejected": -0.9009765386581421,
      "logps/chosen": -0.5218750238418579,
      "logps/rejected": -1.288671851158142,
      "loss": 0.6841,
      "nll_loss": 0.722460925579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.026092529296875,
      "rewards/margins": 0.03841857984662056,
      "rewards/rejected": -0.064453125,
      "step": 12870
    },
    {
      "epoch": 0.9383309656503843,
      "grad_norm": 2.199468525521083,
      "learning_rate": 4.405671105314009e-07,
      "log_odds_chosen": 1.167517066001892,
      "log_odds_ratio": -0.447509765625,
      "logits/chosen": -0.9994140863418579,
      "logits/rejected": -0.871289074420929,
      "logps/chosen": -0.518847644329071,
      "logps/rejected": -1.192285180091858,
      "loss": 0.6943,
      "nll_loss": 0.621386706829071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02592163160443306,
      "rewards/margins": 0.03370513767004013,
      "rewards/rejected": -0.05961913987994194,
      "step": 12880
    },
    {
      "epoch": 0.9390594834808582,
      "grad_norm": 2.4807239647089645,
      "learning_rate": 4.4039618245104644e-07,
      "log_odds_chosen": 1.19580078125,
      "log_odds_ratio": -0.4345703125,
      "logits/chosen": -0.990234375,
      "logits/rejected": -0.911328136920929,
      "logps/chosen": -0.5072265863418579,
      "logps/rejected": -1.165624976158142,
      "loss": 0.6591,
      "nll_loss": 0.6473633050918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02535705640912056,
      "rewards/margins": 0.03291625902056694,
      "rewards/rejected": -0.05823974683880806,
      "step": 12890
    },
    {
      "epoch": 0.9397880013113321,
      "grad_norm": 2.252589652023913,
      "learning_rate": 4.4022545316281195e-07,
      "log_odds_chosen": 0.997314453125,
      "log_odds_ratio": -0.4998535215854645,
      "logits/chosen": -0.9740234613418579,
      "logits/rejected": -0.8685547113418579,
      "logps/chosen": -0.552929699420929,
      "logps/rejected": -1.1652343273162842,
      "loss": 0.7041,
      "nll_loss": 0.716015636920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02765502966940403,
      "rewards/margins": 0.03064422681927681,
      "rewards/rejected": -0.05825195461511612,
      "step": 12900
    },
    {
      "epoch": 0.940516519141806,
      "grad_norm": 2.1450997609842815,
      "learning_rate": 4.40054922281665e-07,
      "log_odds_chosen": 1.0248901844024658,
      "log_odds_ratio": -0.47705078125,
      "logits/chosen": -1.038476586341858,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.551953136920929,
      "logps/rejected": -1.16455078125,
      "loss": 0.6743,
      "nll_loss": 0.678515613079071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02759399451315403,
      "rewards/margins": 0.030594635754823685,
      "rewards/rejected": -0.05823364108800888,
      "step": 12910
    },
    {
      "epoch": 0.9412450369722799,
      "grad_norm": 2.0122965475239134,
      "learning_rate": 4.398845894236168e-07,
      "log_odds_chosen": 1.1748535633087158,
      "log_odds_ratio": -0.4248046875,
      "logits/chosen": -0.9951171875,
      "logits/rejected": -0.9183593988418579,
      "logps/chosen": -0.5482422113418579,
      "logps/rejected": -1.231054663658142,
      "loss": 0.6945,
      "nll_loss": 0.6732422113418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02741088904440403,
      "rewards/margins": 0.03418121486902237,
      "rewards/rejected": -0.06156005710363388,
      "step": 12920
    },
    {
      "epoch": 0.9419735548027538,
      "grad_norm": 3.0526128403572836,
      "learning_rate": 4.397144542057178e-07,
      "log_odds_chosen": 1.02783203125,
      "log_odds_ratio": -0.46894532442092896,
      "logits/chosen": -0.974414050579071,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -0.5140625238418579,
      "logps/rejected": -1.0919921398162842,
      "loss": 0.6971,
      "nll_loss": 0.669238269329071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.02570190466940403,
      "rewards/margins": 0.02886505052447319,
      "rewards/rejected": -0.05455322191119194,
      "step": 12930
    },
    {
      "epoch": 0.9427020726332277,
      "grad_norm": 2.256011783680989,
      "learning_rate": 4.3954451624605487e-07,
      "log_odds_chosen": 1.135400414466858,
      "log_odds_ratio": -0.42548829317092896,
      "logits/chosen": -0.977343738079071,
      "logits/rejected": -0.878125011920929,
      "logps/chosen": -0.5328124761581421,
      "logps/rejected": -1.2185547351837158,
      "loss": 0.6906,
      "nll_loss": 0.6641601324081421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02664794959127903,
      "rewards/margins": 0.03425903245806694,
      "rewards/rejected": -0.0609130859375,
      "step": 12940
    },
    {
      "epoch": 0.9434305904637016,
      "grad_norm": 2.0197317532797676,
      "learning_rate": 4.3937477516374683e-07,
      "log_odds_chosen": 0.994433581829071,
      "log_odds_ratio": -0.48857420682907104,
      "logits/chosen": -0.9703124761581421,
      "logits/rejected": -0.8775390386581421,
      "logps/chosen": -0.47587889432907104,
      "logps/rejected": -1.0412108898162842,
      "loss": 0.66,
      "nll_loss": 0.603808581829071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02379150316119194,
      "rewards/margins": 0.028279876336455345,
      "rewards/rejected": -0.05206909030675888,
      "step": 12950
    },
    {
      "epoch": 0.9441591082941755,
      "grad_norm": 2.509972407688853,
      "learning_rate": 4.392052305789416e-07,
      "log_odds_chosen": 1.3439452648162842,
      "log_odds_ratio": -0.3980468809604645,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.872851550579071,
      "logps/chosen": -0.50244140625,
      "logps/rejected": -1.3298828601837158,
      "loss": 0.682,
      "nll_loss": 0.6631835699081421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02514038048684597,
      "rewards/margins": 0.04135742038488388,
      "rewards/rejected": -0.06654052436351776,
      "step": 12960
    },
    {
      "epoch": 0.9448876261246494,
      "grad_norm": 3.028652326521763,
      "learning_rate": 4.3903588211281233e-07,
      "log_odds_chosen": 1.230371117591858,
      "log_odds_ratio": -0.44287109375,
      "logits/chosen": -0.985546886920929,
      "logits/rejected": -0.888476550579071,
      "logps/chosen": -0.511914074420929,
      "logps/rejected": -1.237890601158142,
      "loss": 0.6694,
      "nll_loss": 0.619433581829071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02559204027056694,
      "rewards/margins": 0.03632964938879013,
      "rewards/rejected": -0.06187744066119194,
      "step": 12970
    },
    {
      "epoch": 0.9456161439551233,
      "grad_norm": 3.1852244602449997,
      "learning_rate": 4.3886672938755363e-07,
      "log_odds_chosen": 1.097070336341858,
      "log_odds_ratio": -0.4625000059604645,
      "logits/chosen": -0.9365234375,
      "logits/rejected": -0.850390613079071,
      "logps/chosen": -0.547167956829071,
      "logps/rejected": -1.2306640148162842,
      "loss": 0.6791,
      "nll_loss": 0.694531261920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02734985388815403,
      "rewards/margins": 0.03419799730181694,
      "rewards/rejected": -0.06151122972369194,
      "step": 12980
    },
    {
      "epoch": 0.9463446617855972,
      "grad_norm": 2.1100160059031223,
      "learning_rate": 4.386977720263786e-07,
      "log_odds_chosen": 1.0419433116912842,
      "log_odds_ratio": -0.47685545682907104,
      "logits/chosen": -0.986132800579071,
      "logits/rejected": -0.895703136920929,
      "logps/chosen": -0.5147460699081421,
      "logps/rejected": -1.1218750476837158,
      "loss": 0.672,
      "nll_loss": 0.66552734375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02574462816119194,
      "rewards/margins": 0.03037872351706028,
      "rewards/rejected": -0.05605468899011612,
      "step": 12990
    },
    {
      "epoch": 0.9470731796160711,
      "grad_norm": 2.1947650204363396,
      "learning_rate": 4.3852900965351467e-07,
      "log_odds_chosen": 1.3015625476837158,
      "log_odds_ratio": -0.4027343690395355,
      "logits/chosen": -0.977734386920929,
      "logits/rejected": -0.846484363079071,
      "logps/chosen": -0.5537109375,
      "logps/rejected": -1.3359375,
      "loss": 0.7105,
      "nll_loss": 0.713671863079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02766113355755806,
      "rewards/margins": 0.03912353515625,
      "rewards/rejected": -0.06674804538488388,
      "step": 13000
    },
    {
      "epoch": 0.947801697446545,
      "grad_norm": 2.200517885969583,
      "learning_rate": 4.383604418942005e-07,
      "log_odds_chosen": 1.125390648841858,
      "log_odds_ratio": -0.463623046875,
      "logits/chosen": -0.969531238079071,
      "logits/rejected": -0.8857421875,
      "logps/chosen": -0.5615234375,
      "logps/rejected": -1.2199218273162842,
      "loss": 0.6953,
      "nll_loss": 0.6576172113418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02805175818502903,
      "rewards/margins": 0.03288879245519638,
      "rewards/rejected": -0.06097412109375,
      "step": 13010
    },
    {
      "epoch": 0.9485302152770189,
      "grad_norm": 4.005727979636282,
      "learning_rate": 4.3819206837468245e-07,
      "log_odds_chosen": 1.3826172351837158,
      "log_odds_ratio": -0.3785156309604645,
      "logits/chosen": -0.968554675579071,
      "logits/rejected": -0.863085925579071,
      "logps/chosen": -0.4571289122104645,
      "logps/rejected": -1.2316405773162842,
      "loss": 0.6778,
      "nll_loss": 0.618847668170929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.02286376990377903,
      "rewards/margins": 0.03875885158777237,
      "rewards/rejected": -0.06158447265625,
      "step": 13020
    },
    {
      "epoch": 0.9492587331074928,
      "grad_norm": 4.162110701179134,
      "learning_rate": 4.380238887222108e-07,
      "log_odds_chosen": 1.2000000476837158,
      "log_odds_ratio": -0.4156738221645355,
      "logits/chosen": -0.9810546636581421,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.528124988079071,
      "logps/rejected": -1.230078101158142,
      "loss": 0.6861,
      "nll_loss": 0.633007824420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.026397705078125,
      "rewards/margins": 0.03507690504193306,
      "rewards/rejected": -0.06148681789636612,
      "step": 13030
    },
    {
      "epoch": 0.9499872509379667,
      "grad_norm": 2.4686626419445754,
      "learning_rate": 4.3785590256503677e-07,
      "log_odds_chosen": 1.130761742591858,
      "log_odds_ratio": -0.45185548067092896,
      "logits/chosen": -1.0031249523162842,
      "logits/rejected": -0.8880859613418579,
      "logps/chosen": -0.53515625,
      "logps/rejected": -1.185546875,
      "loss": 0.686,
      "nll_loss": 0.69677734375,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.02673950232565403,
      "rewards/margins": 0.032562255859375,
      "rewards/rejected": -0.05927734449505806,
      "step": 13040
    },
    {
      "epoch": 0.9507157687684407,
      "grad_norm": 2.063196410957422,
      "learning_rate": 4.3768810953240856e-07,
      "log_odds_chosen": 1.040917992591858,
      "log_odds_ratio": -0.4619140625,
      "logits/chosen": -0.9800781011581421,
      "logits/rejected": -0.855664074420929,
      "logps/chosen": -0.5135742425918579,
      "logps/rejected": -1.113671898841858,
      "loss": 0.687,
      "nll_loss": 0.67529296875,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02567749097943306,
      "rewards/margins": 0.029975127428770065,
      "rewards/rejected": -0.05565185472369194,
      "step": 13050
    },
    {
      "epoch": 0.9514442865989146,
      "grad_norm": 2.5333542884733817,
      "learning_rate": 4.3752050925456824e-07,
      "log_odds_chosen": 1.3019530773162842,
      "log_odds_ratio": -0.39750975370407104,
      "logits/chosen": -0.967968761920929,
      "logits/rejected": -0.842968761920929,
      "logps/chosen": -0.49296873807907104,
      "logps/rejected": -1.257421851158142,
      "loss": 0.6653,
      "nll_loss": 0.6581054925918579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.024627685546875,
      "rewards/margins": 0.03827514499425888,
      "rewards/rejected": -0.06289062649011612,
      "step": 13060
    },
    {
      "epoch": 0.9521728044293885,
      "grad_norm": 2.109354575288355,
      "learning_rate": 4.373531013627483e-07,
      "log_odds_chosen": 1.031640648841858,
      "log_odds_ratio": -0.508105456829071,
      "logits/chosen": -0.964062511920929,
      "logits/rejected": -0.879101574420929,
      "logps/chosen": -0.5489257574081421,
      "logps/rejected": -1.1560547351837158,
      "loss": 0.6974,
      "nll_loss": 0.7095702886581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02744751051068306,
      "rewards/margins": 0.0303955078125,
      "rewards/rejected": -0.05781250074505806,
      "step": 13070
    },
    {
      "epoch": 0.9529013222598623,
      "grad_norm": 2.429729070885097,
      "learning_rate": 4.3718588548916816e-07,
      "log_odds_chosen": 0.9952636957168579,
      "log_odds_ratio": -0.4945312440395355,
      "logits/chosen": -0.9906250238418579,
      "logits/rejected": -0.8902343511581421,
      "logps/chosen": -0.590136706829071,
      "logps/rejected": -1.2158203125,
      "loss": 0.6893,
      "nll_loss": 0.699902355670929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02948608435690403,
      "rewards/margins": 0.03131408616900444,
      "rewards/rejected": -0.06077880784869194,
      "step": 13080
    },
    {
      "epoch": 0.9536298400903362,
      "grad_norm": 2.6198902351763946,
      "learning_rate": 4.3701886126703066e-07,
      "log_odds_chosen": 1.0719726085662842,
      "log_odds_ratio": -0.457763671875,
      "logits/chosen": -1.008398413658142,
      "logits/rejected": -0.873828113079071,
      "logps/chosen": -0.530078113079071,
      "logps/rejected": -1.1955077648162842,
      "loss": 0.7069,
      "nll_loss": 0.6449218988418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02650756761431694,
      "rewards/margins": 0.03328246995806694,
      "rewards/rejected": -0.05983886867761612,
      "step": 13090
    },
    {
      "epoch": 0.9543583579208101,
      "grad_norm": 3.386659387108535,
      "learning_rate": 4.3685202833051903e-07,
      "log_odds_chosen": 1.1071288585662842,
      "log_odds_ratio": -0.44091796875,
      "logits/chosen": -1.0029296875,
      "logits/rejected": -0.872851550579071,
      "logps/chosen": -0.51416015625,
      "logps/rejected": -1.1544921398162842,
      "loss": 0.6819,
      "nll_loss": 0.6488281488418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02572021447122097,
      "rewards/margins": 0.03197631984949112,
      "rewards/rejected": -0.05772705003619194,
      "step": 13100
    },
    {
      "epoch": 0.955086875751284,
      "grad_norm": 2.862180460307986,
      "learning_rate": 4.3668538631479314e-07,
      "log_odds_chosen": 0.9659668207168579,
      "log_odds_ratio": -0.49560546875,
      "logits/chosen": -0.970507800579071,
      "logits/rejected": -0.893750011920929,
      "logps/chosen": -0.56201171875,
      "logps/rejected": -1.1384766101837158,
      "loss": 0.6947,
      "nll_loss": 0.697070300579071,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.02811889722943306,
      "rewards/margins": 0.028784941881895065,
      "rewards/rejected": -0.05692138522863388,
      "step": 13110
    },
    {
      "epoch": 0.9558153935817579,
      "grad_norm": 3.754794236039552,
      "learning_rate": 4.3651893485598645e-07,
      "log_odds_chosen": 1.329687476158142,
      "log_odds_ratio": -0.41157227754592896,
      "logits/chosen": -0.9917968511581421,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.535449206829071,
      "logps/rejected": -1.3310546875,
      "loss": 0.6837,
      "nll_loss": 0.6728515625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02677612379193306,
      "rewards/margins": 0.03975830227136612,
      "rewards/rejected": -0.0665283203125,
      "step": 13120
    },
    {
      "epoch": 0.9565439114122318,
      "grad_norm": 2.4471967251950826,
      "learning_rate": 4.3635267359120244e-07,
      "log_odds_chosen": 1.048974633216858,
      "log_odds_ratio": -0.43632811307907104,
      "logits/chosen": -1.022070288658142,
      "logits/rejected": -0.896289050579071,
      "logps/chosen": -0.525195300579071,
      "logps/rejected": -1.1300780773162842,
      "loss": 0.6838,
      "nll_loss": 0.650683581829071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02625732496380806,
      "rewards/margins": 0.03026275709271431,
      "rewards/rejected": -0.05648193508386612,
      "step": 13130
    },
    {
      "epoch": 0.9572724292427057,
      "grad_norm": 2.980784495111857,
      "learning_rate": 4.361866021585114e-07,
      "log_odds_chosen": 1.1033203601837158,
      "log_odds_ratio": -0.433349609375,
      "logits/chosen": -0.9791015386581421,
      "logits/rejected": -0.891406238079071,
      "logps/chosen": -0.5199218988418579,
      "logps/rejected": -1.1824219226837158,
      "loss": 0.6812,
      "nll_loss": 0.640917956829071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02597656287252903,
      "rewards/margins": 0.03311920166015625,
      "rewards/rejected": -0.05910644680261612,
      "step": 13140
    },
    {
      "epoch": 0.9580009470731796,
      "grad_norm": 2.5107788469375047,
      "learning_rate": 4.360207201969474e-07,
      "log_odds_chosen": 1.306787133216858,
      "log_odds_ratio": -0.3614257872104645,
      "logits/chosen": -0.995898425579071,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.5166991949081421,
      "logps/rejected": -1.2843749523162842,
      "loss": 0.6871,
      "nll_loss": 0.593554675579071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.02583007887005806,
      "rewards/margins": 0.03837432712316513,
      "rewards/rejected": -0.06418456882238388,
      "step": 13150
    },
    {
      "epoch": 0.9587294649036535,
      "grad_norm": 2.3640195288268884,
      "learning_rate": 4.358550273465042e-07,
      "log_odds_chosen": 1.1676757335662842,
      "log_odds_ratio": -0.4364257752895355,
      "logits/chosen": -1.009374976158142,
      "logits/rejected": -0.896679699420929,
      "logps/chosen": -0.5279296636581421,
      "logps/rejected": -1.2267577648162842,
      "loss": 0.691,
      "nll_loss": 0.66650390625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02637939527630806,
      "rewards/margins": 0.03495178371667862,
      "rewards/rejected": -0.06130371242761612,
      "step": 13160
    },
    {
      "epoch": 0.9594579827341274,
      "grad_norm": 2.6789558728670126,
      "learning_rate": 4.356895232481329e-07,
      "log_odds_chosen": 1.170678734779358,
      "log_odds_ratio": -0.442138671875,
      "logits/chosen": -0.993945300579071,
      "logits/rejected": -0.862500011920929,
      "logps/chosen": -0.5328124761581421,
      "logps/rejected": -1.2214844226837158,
      "loss": 0.6669,
      "nll_loss": 0.6767578125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02664489671587944,
      "rewards/margins": 0.03435669094324112,
      "rewards/rejected": -0.06099853664636612,
      "step": 13170
    },
    {
      "epoch": 0.9601865005646013,
      "grad_norm": 2.2608798180644247,
      "learning_rate": 4.3552420754373796e-07,
      "log_odds_chosen": 1.060546875,
      "log_odds_ratio": -0.48784178495407104,
      "logits/chosen": -0.9349609613418579,
      "logits/rejected": -0.806445300579071,
      "logps/chosen": -0.542285144329071,
      "logps/rejected": -1.194726586341858,
      "loss": 0.7005,
      "nll_loss": 0.71728515625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02710571326315403,
      "rewards/margins": 0.03257446363568306,
      "rewards/rejected": -0.05967407301068306,
      "step": 13180
    },
    {
      "epoch": 0.9609150183950752,
      "grad_norm": 2.2900835769396424,
      "learning_rate": 4.353590798761745e-07,
      "log_odds_chosen": 1.237890601158142,
      "log_odds_ratio": -0.42353516817092896,
      "logits/chosen": -0.970507800579071,
      "logits/rejected": -0.8603515625,
      "logps/chosen": -0.5448242425918579,
      "logps/rejected": -1.2712891101837158,
      "loss": 0.6885,
      "nll_loss": 0.704296886920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02726440504193306,
      "rewards/margins": 0.03635253757238388,
      "rewards/rejected": -0.0635986328125,
      "step": 13190
    },
    {
      "epoch": 0.9616435362255491,
      "grad_norm": 2.6779538809398296,
      "learning_rate": 4.351941398892447e-07,
      "log_odds_chosen": 1.1018555164337158,
      "log_odds_ratio": -0.4779296815395355,
      "logits/chosen": -0.9537109136581421,
      "logits/rejected": -0.861523449420929,
      "logps/chosen": -0.510449230670929,
      "logps/rejected": -1.180078148841858,
      "loss": 0.6903,
      "nll_loss": 0.6327148675918579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02552490308880806,
      "rewards/margins": 0.03346862643957138,
      "rewards/rejected": -0.05897216871380806,
      "step": 13200
    },
    {
      "epoch": 0.962372054056023,
      "grad_norm": 2.5269637242312855,
      "learning_rate": 4.350293872276944e-07,
      "log_odds_chosen": 1.150390625,
      "log_odds_ratio": -0.4405273497104645,
      "logits/chosen": -0.942578136920929,
      "logits/rejected": -0.8394531011581421,
      "logps/chosen": -0.511914074420929,
      "logps/rejected": -1.2136719226837158,
      "loss": 0.7075,
      "nll_loss": 0.661914050579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02559814415872097,
      "rewards/margins": 0.03509368747472763,
      "rewards/rejected": -0.06071777269244194,
      "step": 13210
    },
    {
      "epoch": 0.9631005718864969,
      "grad_norm": 3.1588572108741713,
      "learning_rate": 4.348648215372106e-07,
      "log_odds_chosen": 1.0898926258087158,
      "log_odds_ratio": -0.4827636778354645,
      "logits/chosen": -0.982226550579071,
      "logits/rejected": -0.8763672113418579,
      "logps/chosen": -0.5648437738418579,
      "logps/rejected": -1.221777319908142,
      "loss": 0.6866,
      "nll_loss": 0.765820324420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02824096754193306,
      "rewards/margins": 0.032811738550662994,
      "rewards/rejected": -0.06108398362994194,
      "step": 13220
    },
    {
      "epoch": 0.9638290897169708,
      "grad_norm": 2.5095608339270923,
      "learning_rate": 4.347004424644176e-07,
      "log_odds_chosen": 1.297460913658142,
      "log_odds_ratio": -0.3907714784145355,
      "logits/chosen": -0.970507800579071,
      "logits/rejected": -0.8716796636581421,
      "logps/chosen": -0.5419921875,
      "logps/rejected": -1.322851538658142,
      "loss": 0.6889,
      "nll_loss": 0.703320324420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02707519568502903,
      "rewards/margins": 0.03905029222369194,
      "rewards/rejected": -0.06617431342601776,
      "step": 13230
    },
    {
      "epoch": 0.9645576075474447,
      "grad_norm": 2.3499029278090955,
      "learning_rate": 4.345362496568739e-07,
      "log_odds_chosen": 1.285742163658142,
      "log_odds_ratio": -0.4088378846645355,
      "logits/chosen": -0.9585937261581421,
      "logits/rejected": -0.8384765386581421,
      "logps/chosen": -0.5316406488418579,
      "logps/rejected": -1.3484375476837158,
      "loss": 0.6845,
      "nll_loss": 0.670703113079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02657470665872097,
      "rewards/margins": 0.040863037109375,
      "rewards/rejected": -0.06743164360523224,
      "step": 13240
    },
    {
      "epoch": 0.9652861253779186,
      "grad_norm": 2.6965572661761796,
      "learning_rate": 4.343722427630694e-07,
      "log_odds_chosen": 1.384912133216858,
      "log_odds_ratio": -0.4015136659145355,
      "logits/chosen": -1.0089843273162842,
      "logits/rejected": -0.8851562738418579,
      "logps/chosen": -0.559765636920929,
      "logps/rejected": -1.4306640625,
      "loss": 0.6768,
      "nll_loss": 0.65966796875,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02799682691693306,
      "rewards/margins": 0.04349059984087944,
      "rewards/rejected": -0.07152099907398224,
      "step": 13250
    },
    {
      "epoch": 0.9660146432083925,
      "grad_norm": 2.202093606405359,
      "learning_rate": 4.3420842143242187e-07,
      "log_odds_chosen": 1.0264160633087158,
      "log_odds_ratio": -0.48662108182907104,
      "logits/chosen": -0.971875011920929,
      "logits/rejected": -0.917187511920929,
      "logps/chosen": -0.5350586175918579,
      "logps/rejected": -1.151757836341858,
      "loss": 0.6928,
      "nll_loss": 0.6416991949081421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02675781212747097,
      "rewards/margins": 0.030857086181640625,
      "rewards/rejected": -0.05761108547449112,
      "step": 13260
    },
    {
      "epoch": 0.9667431610388664,
      "grad_norm": 2.374719909464103,
      "learning_rate": 4.340447853152738e-07,
      "log_odds_chosen": 1.1180419921875,
      "log_odds_ratio": -0.43549805879592896,
      "logits/chosen": -0.9947265386581421,
      "logits/rejected": -0.886914074420929,
      "logps/chosen": -0.548828125,
      "logps/rejected": -1.239843726158142,
      "loss": 0.684,
      "nll_loss": 0.631054699420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02742919884622097,
      "rewards/margins": 0.03456573560833931,
      "rewards/rejected": -0.06198730319738388,
      "step": 13270
    },
    {
      "epoch": 0.9674716788693404,
      "grad_norm": 2.784473553003913,
      "learning_rate": 4.338813340628896e-07,
      "log_odds_chosen": 1.22509765625,
      "log_odds_ratio": -0.46037596464157104,
      "logits/chosen": -1.0029296875,
      "logits/rejected": -0.8974609375,
      "logps/chosen": -0.48779296875,
      "logps/rejected": -1.24609375,
      "loss": 0.6642,
      "nll_loss": 0.60986328125,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.024383544921875,
      "rewards/margins": 0.03798217698931694,
      "rewards/rejected": -0.06234131008386612,
      "step": 13280
    },
    {
      "epoch": 0.9682001966998143,
      "grad_norm": 2.1898356699810004,
      "learning_rate": 4.337180673274522e-07,
      "log_odds_chosen": 1.0068359375,
      "log_odds_ratio": -0.48193359375,
      "logits/chosen": -1.001367211341858,
      "logits/rejected": -0.872851550579071,
      "logps/chosen": -0.513378918170929,
      "logps/rejected": -1.111328125,
      "loss": 0.7052,
      "nll_loss": 0.6380859613418579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02567138709127903,
      "rewards/margins": 0.02992553636431694,
      "rewards/rejected": -0.05557861179113388,
      "step": 13290
    },
    {
      "epoch": 0.9689287145302882,
      "grad_norm": 3.450678056945945,
      "learning_rate": 4.3355498476206e-07,
      "log_odds_chosen": 1.091894507408142,
      "log_odds_ratio": -0.444091796875,
      "logits/chosen": -0.9740234613418579,
      "logits/rejected": -0.8373047113418579,
      "logps/chosen": -0.5370117425918579,
      "logps/rejected": -1.225195288658142,
      "loss": 0.6807,
      "nll_loss": 0.6292968988418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02687377855181694,
      "rewards/margins": 0.03443298488855362,
      "rewards/rejected": -0.06131591647863388,
      "step": 13300
    },
    {
      "epoch": 0.9696572323607621,
      "grad_norm": 2.3223446270898793,
      "learning_rate": 4.3339208602072376e-07,
      "log_odds_chosen": 0.9809814691543579,
      "log_odds_ratio": -0.49150389432907104,
      "logits/chosen": -0.983593761920929,
      "logits/rejected": -0.866992175579071,
      "logps/chosen": -0.54248046875,
      "logps/rejected": -1.114648461341858,
      "loss": 0.6988,
      "nll_loss": 0.725781261920929,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.02711181715130806,
      "rewards/margins": 0.028578568249940872,
      "rewards/rejected": -0.05571288987994194,
      "step": 13310
    },
    {
      "epoch": 0.970385750191236,
      "grad_norm": 4.05678354225028,
      "learning_rate": 4.332293707583637e-07,
      "log_odds_chosen": 1.0974853038787842,
      "log_odds_ratio": -0.4874511659145355,
      "logits/chosen": -0.984179675579071,
      "logits/rejected": -0.9066406488418579,
      "logps/chosen": -0.592480480670929,
      "logps/rejected": -1.244140625,
      "loss": 0.6865,
      "nll_loss": 0.6734374761581421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02964172326028347,
      "rewards/margins": 0.03261108323931694,
      "rewards/rejected": -0.06228027492761612,
      "step": 13320
    },
    {
      "epoch": 0.9711142680217099,
      "grad_norm": 2.465641014757163,
      "learning_rate": 4.3306683863080583e-07,
      "log_odds_chosen": 1.237158179283142,
      "log_odds_ratio": -0.434326171875,
      "logits/chosen": -0.9794921875,
      "logits/rejected": -0.8880859613418579,
      "logps/chosen": -0.5711914300918579,
      "logps/rejected": -1.3449218273162842,
      "loss": 0.6826,
      "nll_loss": 0.702441394329071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02858276292681694,
      "rewards/margins": 0.0386810302734375,
      "rewards/rejected": -0.06722412258386612,
      "step": 13330
    },
    {
      "epoch": 0.9718427858521838,
      "grad_norm": 2.742226843687582,
      "learning_rate": 4.3290448929477996e-07,
      "log_odds_chosen": 1.099511742591858,
      "log_odds_ratio": -0.4180664122104645,
      "logits/chosen": -0.9449218511581421,
      "logits/rejected": -0.8714843988418579,
      "logps/chosen": -0.514355480670929,
      "logps/rejected": -1.122460961341858,
      "loss": 0.6866,
      "nll_loss": 0.6739257574081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.025726318359375,
      "rewards/margins": 0.03042449988424778,
      "rewards/rejected": -0.05615234375,
      "step": 13340
    },
    {
      "epoch": 0.9725713036826577,
      "grad_norm": 2.2539874565179208,
      "learning_rate": 4.3274232240791545e-07,
      "log_odds_chosen": 1.266210913658142,
      "log_odds_ratio": -0.41484373807907104,
      "logits/chosen": -0.9935547113418579,
      "logits/rejected": -0.86328125,
      "logps/chosen": -0.551074206829071,
      "logps/rejected": -1.3191406726837158,
      "loss": 0.6759,
      "nll_loss": 0.6773437261581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02755126915872097,
      "rewards/margins": 0.03843078762292862,
      "rewards/rejected": -0.06593017280101776,
      "step": 13350
    },
    {
      "epoch": 0.9732998215131315,
      "grad_norm": 2.971967341982762,
      "learning_rate": 4.325803376287393e-07,
      "log_odds_chosen": 1.5558593273162842,
      "log_odds_ratio": -0.3682617247104645,
      "logits/chosen": -0.9488281011581421,
      "logits/rejected": -0.8441406488418579,
      "logps/chosen": -0.5010741949081421,
      "logps/rejected": -1.4490234851837158,
      "loss": 0.6866,
      "nll_loss": 0.644238293170929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02506408654153347,
      "rewards/margins": 0.0473480224609375,
      "rewards/rejected": -0.07243652641773224,
      "step": 13360
    },
    {
      "epoch": 0.9740283393436054,
      "grad_norm": 2.8818749404690207,
      "learning_rate": 4.32418534616672e-07,
      "log_odds_chosen": 1.0822265148162842,
      "log_odds_ratio": -0.4607177674770355,
      "logits/chosen": -0.9429687261581421,
      "logits/rejected": -0.849609375,
      "logps/chosen": -0.573925793170929,
      "logps/rejected": -1.2478516101837158,
      "loss": 0.6869,
      "nll_loss": 0.758593738079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02866210974752903,
      "rewards/margins": 0.033698271960020065,
      "rewards/rejected": -0.06236572191119194,
      "step": 13370
    },
    {
      "epoch": 0.9747568571740793,
      "grad_norm": 2.8128927651488493,
      "learning_rate": 4.3225691303202553e-07,
      "log_odds_chosen": 1.1565430164337158,
      "log_odds_ratio": -0.46796876192092896,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.8365234136581421,
      "logps/chosen": -0.5467773675918579,
      "logps/rejected": -1.2546875476837158,
      "loss": 0.6917,
      "nll_loss": 0.666210949420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02732544019818306,
      "rewards/margins": 0.03543243557214737,
      "rewards/rejected": -0.06270752102136612,
      "step": 13380
    },
    {
      "epoch": 0.9754853750045532,
      "grad_norm": 2.6197609452472,
      "learning_rate": 4.320954725359999e-07,
      "log_odds_chosen": 1.4083983898162842,
      "log_odds_ratio": -0.3865722715854645,
      "logits/chosen": -0.974414050579071,
      "logits/rejected": -0.883593738079071,
      "logps/chosen": -0.49873048067092896,
      "logps/rejected": -1.3380858898162842,
      "loss": 0.6895,
      "nll_loss": 0.6998046636581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02493896521627903,
      "rewards/margins": 0.04196777194738388,
      "rewards/rejected": -0.06693115085363388,
      "step": 13390
    },
    {
      "epoch": 0.9762138928350271,
      "grad_norm": 2.4071208638031507,
      "learning_rate": 4.319342127906801e-07,
      "log_odds_chosen": 1.27880859375,
      "log_odds_ratio": -0.40839844942092896,
      "logits/chosen": -0.975781261920929,
      "logits/rejected": -0.8814452886581421,
      "logps/chosen": -0.5176757574081421,
      "logps/rejected": -1.2511718273162842,
      "loss": 0.6655,
      "nll_loss": 0.652148425579071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02588501013815403,
      "rewards/margins": 0.03668823093175888,
      "rewards/rejected": -0.06257323920726776,
      "step": 13400
    },
    {
      "epoch": 0.976942410665501,
      "grad_norm": 2.451162515336652,
      "learning_rate": 4.317731334590332e-07,
      "log_odds_chosen": 1.291406273841858,
      "log_odds_ratio": -0.4140625,
      "logits/chosen": -0.9927734136581421,
      "logits/rejected": -0.8824218511581421,
      "logps/chosen": -0.5225585699081421,
      "logps/rejected": -1.278906226158142,
      "loss": 0.6946,
      "nll_loss": 0.7139648199081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02613525465130806,
      "rewards/margins": 0.03780517727136612,
      "rewards/rejected": -0.06392822414636612,
      "step": 13410
    },
    {
      "epoch": 0.9776709284959749,
      "grad_norm": 2.641429521196596,
      "learning_rate": 4.316122342049055e-07,
      "log_odds_chosen": 1.3909180164337158,
      "log_odds_ratio": -0.40498048067092896,
      "logits/chosen": -0.9750000238418579,
      "logits/rejected": -0.8853515386581421,
      "logps/chosen": -0.504589855670929,
      "logps/rejected": -1.3253905773162842,
      "loss": 0.6782,
      "nll_loss": 0.623046875,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0252227783203125,
      "rewards/margins": 0.04097595065832138,
      "rewards/rejected": -0.06622314453125,
      "step": 13420
    },
    {
      "epoch": 0.9783994463264488,
      "grad_norm": 2.529526586837493,
      "learning_rate": 4.314515146930197e-07,
      "log_odds_chosen": 1.0286743640899658,
      "log_odds_ratio": -0.46357423067092896,
      "logits/chosen": -1.0388672351837158,
      "logits/rejected": -0.915820300579071,
      "logps/chosen": -0.530957043170929,
      "logps/rejected": -1.133203148841858,
      "loss": 0.6776,
      "nll_loss": 0.6318359375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02653198316693306,
      "rewards/margins": 0.030136872082948685,
      "rewards/rejected": -0.056640625,
      "step": 13430
    },
    {
      "epoch": 0.9791279641569227,
      "grad_norm": 2.3291567612555606,
      "learning_rate": 4.3129097458897137e-07,
      "log_odds_chosen": 1.3416016101837158,
      "log_odds_ratio": -0.41328126192092896,
      "logits/chosen": -0.976367175579071,
      "logits/rejected": -0.8550781011581421,
      "logps/chosen": -0.5362304449081421,
      "logps/rejected": -1.3224608898162842,
      "loss": 0.6689,
      "nll_loss": 0.642285168170929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.02678222581744194,
      "rewards/margins": 0.03932647779583931,
      "rewards/rejected": -0.06619872897863388,
      "step": 13440
    },
    {
      "epoch": 0.9798564819873966,
      "grad_norm": 2.323423103861552,
      "learning_rate": 4.311306135592269e-07,
      "log_odds_chosen": 1.129296898841858,
      "log_odds_ratio": -0.4645019471645355,
      "logits/chosen": -0.993359386920929,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -0.5506836175918579,
      "logps/rejected": -1.2302734851837158,
      "loss": 0.6762,
      "nll_loss": 0.701953113079071,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02753295935690403,
      "rewards/margins": 0.03399658203125,
      "rewards/rejected": -0.06154785305261612,
      "step": 13450
    },
    {
      "epoch": 0.9805849998178705,
      "grad_norm": 3.1037180796038717,
      "learning_rate": 4.309704312711197e-07,
      "log_odds_chosen": 1.1731445789337158,
      "log_odds_ratio": -0.41455078125,
      "logits/chosen": -0.9599609375,
      "logits/rejected": -0.865429699420929,
      "logps/chosen": -0.4839843809604645,
      "logps/rejected": -1.186132788658142,
      "loss": 0.6607,
      "nll_loss": 0.6050781011581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02421264722943306,
      "rewards/margins": 0.03510131686925888,
      "rewards/rejected": -0.05933837965130806,
      "step": 13460
    },
    {
      "epoch": 0.9813135176483444,
      "grad_norm": 2.4548504011406798,
      "learning_rate": 4.3081042739284794e-07,
      "log_odds_chosen": 1.3064453601837158,
      "log_odds_ratio": -0.40104979276657104,
      "logits/chosen": -0.9654296636581421,
      "logits/rejected": -0.8246093988418579,
      "logps/chosen": -0.5414062738418579,
      "logps/rejected": -1.3478515148162842,
      "loss": 0.6856,
      "nll_loss": 0.708984375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02708129957318306,
      "rewards/margins": 0.04032440111041069,
      "rewards/rejected": -0.06741943210363388,
      "step": 13470
    },
    {
      "epoch": 0.9820420354788183,
      "grad_norm": 2.7954040953219867,
      "learning_rate": 4.306506015934716e-07,
      "log_odds_chosen": 1.207128882408142,
      "log_odds_ratio": -0.4310546815395355,
      "logits/chosen": -0.989062488079071,
      "logits/rejected": -0.8828125,
      "logps/chosen": -0.5316406488418579,
      "logps/rejected": -1.2421875,
      "loss": 0.6678,
      "nll_loss": 0.652148425579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02658691443502903,
      "rewards/margins": 0.035532377660274506,
      "rewards/rejected": -0.06212768703699112,
      "step": 13480
    },
    {
      "epoch": 0.9827705533092922,
      "grad_norm": 2.7784259334939083,
      "learning_rate": 4.3049095354290916e-07,
      "log_odds_chosen": 1.039306640625,
      "log_odds_ratio": -0.46333009004592896,
      "logits/chosen": -0.9583984613418579,
      "logits/rejected": -0.861328125,
      "logps/chosen": -0.5121093988418579,
      "logps/rejected": -1.0783202648162842,
      "loss": 0.6827,
      "nll_loss": 0.65673828125,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.02560119703412056,
      "rewards/margins": 0.02831420861184597,
      "rewards/rejected": -0.05393066257238388,
      "step": 13490
    },
    {
      "epoch": 0.9834990711397662,
      "grad_norm": 2.3954368197624984,
      "learning_rate": 4.3033148291193525e-07,
      "log_odds_chosen": 1.223486304283142,
      "log_odds_ratio": -0.4365234375,
      "logits/chosen": -0.9775390625,
      "logits/rejected": -0.9007812738418579,
      "logps/chosen": -0.4764648377895355,
      "logps/rejected": -1.148828148841858,
      "loss": 0.6745,
      "nll_loss": 0.62451171875,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.02384033240377903,
      "rewards/margins": 0.03360138088464737,
      "rewards/rejected": -0.05740966647863388,
      "step": 13500
    },
    {
      "epoch": 0.9842275889702401,
      "grad_norm": 2.6433386102777408,
      "learning_rate": 4.3017218937217743e-07,
      "log_odds_chosen": 1.132714867591858,
      "log_odds_ratio": -0.43994140625,
      "logits/chosen": -0.994335949420929,
      "logits/rejected": -0.8857421875,
      "logps/chosen": -0.5430663824081421,
      "logps/rejected": -1.1716797351837158,
      "loss": 0.6804,
      "nll_loss": 0.65673828125,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02718505822122097,
      "rewards/margins": 0.03146057203412056,
      "rewards/rejected": -0.05859375,
      "step": 13510
    },
    {
      "epoch": 0.984956106800714,
      "grad_norm": 2.2618305700216297,
      "learning_rate": 4.3001307259611346e-07,
      "log_odds_chosen": 1.0744140148162842,
      "log_odds_ratio": -0.43266600370407104,
      "logits/chosen": -0.974609375,
      "logits/rejected": -0.9017578363418579,
      "logps/chosen": -0.49560546875,
      "logps/rejected": -1.0681641101837158,
      "loss": 0.6899,
      "nll_loss": 0.713671863079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0247802734375,
      "rewards/margins": 0.02857055701315403,
      "rewards/rejected": -0.05333251878619194,
      "step": 13520
    },
    {
      "epoch": 0.9856846246311879,
      "grad_norm": 2.403225377822068,
      "learning_rate": 4.298541322570687e-07,
      "log_odds_chosen": 1.337792992591858,
      "log_odds_ratio": -0.38520509004592896,
      "logits/chosen": -0.992382824420929,
      "logits/rejected": -0.8802734613418579,
      "logps/chosen": -0.49199217557907104,
      "logps/rejected": -1.2824218273162842,
      "loss": 0.666,
      "nll_loss": 0.6636718511581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.02460937574505806,
      "rewards/margins": 0.03946533054113388,
      "rewards/rejected": -0.06407471001148224,
      "step": 13530
    },
    {
      "epoch": 0.9864131424616618,
      "grad_norm": 2.5383709741922127,
      "learning_rate": 4.296953680292129e-07,
      "log_odds_chosen": 1.3123047351837158,
      "log_odds_ratio": -0.43115234375,
      "logits/chosen": -0.9525390863418579,
      "logits/rejected": -0.8695312738418579,
      "logps/chosen": -0.561328113079071,
      "logps/rejected": -1.408789038658142,
      "loss": 0.6887,
      "nll_loss": 0.6861327886581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02807006798684597,
      "rewards/margins": 0.04232482984662056,
      "rewards/rejected": -0.07044677436351776,
      "step": 13540
    },
    {
      "epoch": 0.9871416602921357,
      "grad_norm": 2.315778216490527,
      "learning_rate": 4.2953677958755773e-07,
      "log_odds_chosen": 1.387304663658142,
      "log_odds_ratio": -0.4046386778354645,
      "logits/chosen": -0.9458984136581421,
      "logits/rejected": -0.8257812261581421,
      "logps/chosen": -0.5333007574081421,
      "logps/rejected": -1.3849608898162842,
      "loss": 0.6826,
      "nll_loss": 0.7103515863418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02666625939309597,
      "rewards/margins": 0.0425872802734375,
      "rewards/rejected": -0.06923828274011612,
      "step": 13550
    },
    {
      "epoch": 0.9878701781226096,
      "grad_norm": 2.579981999911914,
      "learning_rate": 4.2937836660795393e-07,
      "log_odds_chosen": 1.0349609851837158,
      "log_odds_ratio": -0.469970703125,
      "logits/chosen": -0.986132800579071,
      "logits/rejected": -0.890820324420929,
      "logps/chosen": -0.5702148675918579,
      "logps/rejected": -1.169335961341858,
      "loss": 0.6887,
      "nll_loss": 0.690234363079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02850341796875,
      "rewards/margins": 0.02999420091509819,
      "rewards/rejected": -0.05854492262005806,
      "step": 13560
    },
    {
      "epoch": 0.9885986959530835,
      "grad_norm": 2.0244490220940983,
      "learning_rate": 4.292201287670881e-07,
      "log_odds_chosen": 1.108789086341858,
      "log_odds_ratio": -0.4583984315395355,
      "logits/chosen": -0.9525390863418579,
      "logits/rejected": -0.855664074420929,
      "logps/chosen": -0.55322265625,
      "logps/rejected": -1.2042968273162842,
      "loss": 0.6739,
      "nll_loss": 0.650585949420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02766113355755806,
      "rewards/margins": 0.03254394605755806,
      "rewards/rejected": -0.06015624850988388,
      "step": 13570
    },
    {
      "epoch": 0.9893272137835574,
      "grad_norm": 2.8204393102874765,
      "learning_rate": 4.290620657424806e-07,
      "log_odds_chosen": 1.243261694908142,
      "log_odds_ratio": -0.41044920682907104,
      "logits/chosen": -0.9263671636581421,
      "logits/rejected": -0.8101562261581421,
      "logps/chosen": -0.529980480670929,
      "logps/rejected": -1.271875023841858,
      "loss": 0.6798,
      "nll_loss": 0.625683605670929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.02650756761431694,
      "rewards/margins": 0.03711242601275444,
      "rewards/rejected": -0.0635986328125,
      "step": 13580
    },
    {
      "epoch": 0.9900557316140313,
      "grad_norm": 2.2307655043441925,
      "learning_rate": 4.2890417721248234e-07,
      "log_odds_chosen": 1.271875023841858,
      "log_odds_ratio": -0.447509765625,
      "logits/chosen": -0.9439452886581421,
      "logits/rejected": -0.849609375,
      "logps/chosen": -0.507617175579071,
      "logps/rejected": -1.2234375476837158,
      "loss": 0.6672,
      "nll_loss": 0.6317383050918579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.02539672888815403,
      "rewards/margins": 0.03580932691693306,
      "rewards/rejected": -0.06118164211511612,
      "step": 13590
    },
    {
      "epoch": 0.9907842494445052,
      "grad_norm": 2.476402804919889,
      "learning_rate": 4.2874646285627207e-07,
      "log_odds_chosen": 1.352929711341858,
      "log_odds_ratio": -0.39594727754592896,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.8675781488418579,
      "logps/chosen": -0.56494140625,
      "logps/rejected": -1.388085961341858,
      "loss": 0.6952,
      "nll_loss": 0.6939452886581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.02825317345559597,
      "rewards/margins": 0.04114379733800888,
      "rewards/rejected": -0.06939697265625,
      "step": 13600
    },
    {
      "epoch": 0.9915127672749791,
      "grad_norm": 4.862708147044757,
      "learning_rate": 4.2858892235385406e-07,
      "log_odds_chosen": 1.3054687976837158,
      "log_odds_ratio": -0.38359373807907104,
      "logits/chosen": -1.060937523841858,
      "logits/rejected": -0.8958984613418579,
      "logps/chosen": -0.4913085997104645,
      "logps/rejected": -1.2214844226837158,
      "loss": 0.6766,
      "nll_loss": 0.626660168170929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.02454834058880806,
      "rewards/margins": 0.03650818020105362,
      "rewards/rejected": -0.06109619140625,
      "step": 13610
    },
    {
      "epoch": 0.992241285105453,
      "grad_norm": 2.137576512492482,
      "learning_rate": 4.2843155538605465e-07,
      "log_odds_chosen": 1.2392578125,
      "log_odds_ratio": -0.42915040254592896,
      "logits/chosen": -0.9857422113418579,
      "logits/rejected": -0.8896484375,
      "logps/chosen": -0.597949206829071,
      "logps/rejected": -1.340234398841858,
      "loss": 0.6795,
      "nll_loss": 0.682812511920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.02987060509622097,
      "rewards/margins": 0.037139892578125,
      "rewards/rejected": -0.06702880561351776,
      "step": 13620
    },
    {
      "epoch": 0.9929698029359268,
      "grad_norm": 2.333165650530158,
      "learning_rate": 4.2827436163452e-07,
      "log_odds_chosen": 1.1496093273162842,
      "log_odds_ratio": -0.4278320372104645,
      "logits/chosen": -1.0119140148162842,
      "logits/rejected": -0.8828125,
      "logps/chosen": -0.563769519329071,
      "logps/rejected": -1.271093726158142,
      "loss": 0.6565,
      "nll_loss": 0.6806640625,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.028167724609375,
      "rewards/margins": 0.03540649265050888,
      "rewards/rejected": -0.06352539360523224,
      "step": 13630
    },
    {
      "epoch": 0.9936983207664007,
      "grad_norm": 2.38481558654931,
      "learning_rate": 4.281173407817136e-07,
      "log_odds_chosen": 1.180273413658142,
      "log_odds_ratio": -0.42792969942092896,
      "logits/chosen": -1.008398413658142,
      "logits/rejected": -0.90234375,
      "logps/chosen": -0.5218750238418579,
      "logps/rejected": -1.247460961341858,
      "loss": 0.6986,
      "nll_loss": 0.665234386920929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02610473707318306,
      "rewards/margins": 0.03624267503619194,
      "rewards/rejected": -0.062408447265625,
      "step": 13640
    },
    {
      "epoch": 0.9944268385968746,
      "grad_norm": 2.6250158135956956,
      "learning_rate": 4.279604925109129e-07,
      "log_odds_chosen": 1.2197265625,
      "log_odds_ratio": -0.412353515625,
      "logits/chosen": -0.982226550579071,
      "logits/rejected": -0.8525390625,
      "logps/chosen": -0.5018554925918579,
      "logps/rejected": -1.2158203125,
      "loss": 0.6752,
      "nll_loss": 0.623339831829071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.02507934533059597,
      "rewards/margins": 0.0357666015625,
      "rewards/rejected": -0.06087646633386612,
      "step": 13650
    },
    {
      "epoch": 0.9951553564273485,
      "grad_norm": 2.348268635593028,
      "learning_rate": 4.278038165062074e-07,
      "log_odds_chosen": 1.1162598133087158,
      "log_odds_ratio": -0.45917969942092896,
      "logits/chosen": -0.9478515386581421,
      "logits/rejected": -0.833203136920929,
      "logps/chosen": -0.537792980670929,
      "logps/rejected": -1.2156250476837158,
      "loss": 0.6777,
      "nll_loss": 0.637011706829071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.02692260779440403,
      "rewards/margins": 0.03383789211511612,
      "rewards/rejected": -0.06075439602136612,
      "step": 13660
    },
    {
      "epoch": 0.9958838742578224,
      "grad_norm": 2.587341000729853,
      "learning_rate": 4.276473124524952e-07,
      "log_odds_chosen": 0.989990234375,
      "log_odds_ratio": -0.47871094942092896,
      "logits/chosen": -0.9638671875,
      "logits/rejected": -0.8828125,
      "logps/chosen": -0.517578125,
      "logps/rejected": -1.0859375,
      "loss": 0.6782,
      "nll_loss": 0.669921875,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.02586669847369194,
      "rewards/margins": 0.02840118482708931,
      "rewards/rejected": -0.05426635593175888,
      "step": 13670
    },
    {
      "epoch": 0.9966123920882963,
      "grad_norm": 2.3286733596753875,
      "learning_rate": 4.2749098003548097e-07,
      "log_odds_chosen": 1.2705078125,
      "log_odds_ratio": -0.428466796875,
      "logits/chosen": -0.9742187261581421,
      "logits/rejected": -0.8682616949081421,
      "logps/chosen": -0.5244140625,
      "logps/rejected": -1.273046851158142,
      "loss": 0.6711,
      "nll_loss": 0.643847644329071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02619628980755806,
      "rewards/margins": 0.0374603271484375,
      "rewards/rejected": -0.06367187201976776,
      "step": 13680
    },
    {
      "epoch": 0.9973409099187702,
      "grad_norm": 2.323077118537739,
      "learning_rate": 4.2733481894167295e-07,
      "log_odds_chosen": 1.4304687976837158,
      "log_odds_ratio": -0.348876953125,
      "logits/chosen": -0.991992175579071,
      "logits/rejected": -0.873242199420929,
      "logps/chosen": -0.5186523199081421,
      "logps/rejected": -1.3898437023162842,
      "loss": 0.6767,
      "nll_loss": 0.64453125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.02593383751809597,
      "rewards/margins": 0.04353637620806694,
      "rewards/rejected": -0.06947021186351776,
      "step": 13690
    },
    {
      "epoch": 0.9980694277492441,
      "grad_norm": 2.401727543525845,
      "learning_rate": 4.271788288583805e-07,
      "log_odds_chosen": 1.223730444908142,
      "log_odds_ratio": -0.43989259004592896,
      "logits/chosen": -0.956250011920929,
      "logits/rejected": -0.8681640625,
      "logps/chosen": -0.537304699420929,
      "logps/rejected": -1.3054687976837158,
      "loss": 0.6852,
      "nll_loss": 0.6465820074081421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.02686767652630806,
      "rewards/margins": 0.03838195651769638,
      "rewards/rejected": -0.06525878608226776,
      "step": 13700
    },
    {
      "epoch": 0.998797945579718,
      "grad_norm": 2.513072378029441,
      "learning_rate": 4.270230094737114e-07,
      "log_odds_chosen": 1.216894507408142,
      "log_odds_ratio": -0.4292968809604645,
      "logits/chosen": -0.9712890386581421,
      "logits/rejected": -0.867968738079071,
      "logps/chosen": -0.55419921875,
      "logps/rejected": -1.3253905773162842,
      "loss": 0.6892,
      "nll_loss": 0.7408202886581421,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0277099609375,
      "rewards/margins": 0.03860778734087944,
      "rewards/rejected": -0.06630859524011612,
      "step": 13710
    },
    {
      "epoch": 0.9995264634101919,
      "grad_norm": 3.1256982392034733,
      "learning_rate": 4.268673604765692e-07,
      "log_odds_chosen": 1.2734375,
      "log_odds_ratio": -0.41669923067092896,
      "logits/chosen": -0.995898425579071,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.5146484375,
      "logps/rejected": -1.3005859851837158,
      "loss": 0.666,
      "nll_loss": 0.7025390863418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.025726318359375,
      "rewards/margins": 0.03925628587603569,
      "rewards/rejected": -0.06500244140625,
      "step": 13720
    },
    {
      "epoch": 0.9999635741084762,
      "step": 13726,
      "total_flos": 0.0,
      "train_loss": 0.826562102466635,
      "train_runtime": 95982.8093,
      "train_samples_per_second": 18.305,
      "train_steps_per_second": 0.143
    }
  ],
  "logging_steps": 10,
  "max_steps": 13726,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 10000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}