{
  "best_metric": 1.2853127717971802,
  "best_model_checkpoint": "models/qwen2.5-3b-orpo-mini/checkpoint-5000",
  "epoch": 0.6458696635019053,
  "eval_steps": 5000,
  "global_step": 5000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0012917393270038106,
      "grad_norm": 13.57978335426149,
      "learning_rate": 8e-07,
      "log_odds_chosen": 0.7457031011581421,
      "log_odds_ratio": -0.582324206829071,
      "logits/chosen": -0.8775390386581421,
      "logits/rejected": -0.690625011920929,
      "logps/chosen": -1.5751953125,
      "logps/rejected": -2.2093749046325684,
      "loss": 2.1309,
      "nll_loss": 2.068359375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.15766601264476776,
      "rewards/margins": 0.06313476711511612,
      "rewards/rejected": -0.22084960341453552,
      "step": 10
    },
    {
      "epoch": 0.0025834786540076212,
      "grad_norm": 9.015259176400363,
      "learning_rate": 1.6e-06,
      "log_odds_chosen": 0.697924792766571,
      "log_odds_ratio": -0.607666015625,
      "logits/chosen": -0.927734375,
      "logits/rejected": -0.7220703363418579,
      "logps/chosen": -1.47265625,
      "logps/rejected": -2.0589842796325684,
      "loss": 2.0202,
      "nll_loss": 1.851953148841858,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.14726562798023224,
      "rewards/margins": 0.05870361253619194,
      "rewards/rejected": -0.20590820908546448,
      "step": 20
    },
    {
      "epoch": 0.003875217981011432,
      "grad_norm": 6.508079183700648,
      "learning_rate": 2.4e-06,
      "log_odds_chosen": 0.41191405057907104,
      "log_odds_ratio": -0.6729491949081421,
      "logits/chosen": -0.830078125,
      "logits/rejected": -0.7201172113418579,
      "logps/chosen": -1.361328125,
      "logps/rejected": -1.716796875,
      "loss": 1.8363,
      "nll_loss": 1.7890625,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13608399033546448,
      "rewards/margins": 0.03561706468462944,
      "rewards/rejected": -0.1717529296875,
      "step": 30
    },
    {
      "epoch": 0.0051669573080152425,
      "grad_norm": 4.1418217427341135,
      "learning_rate": 3.2e-06,
      "log_odds_chosen": 0.43736571073532104,
      "log_odds_ratio": -0.65234375,
      "logits/chosen": -0.961718738079071,
      "logits/rejected": -0.786914050579071,
      "logps/chosen": -1.343359351158142,
      "logps/rejected": -1.712499976158142,
      "loss": 1.6739,
      "nll_loss": 1.623046875,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.1343994140625,
      "rewards/margins": 0.03701172024011612,
      "rewards/rejected": -0.17136231064796448,
      "step": 40
    },
    {
      "epoch": 0.006458696635019053,
      "grad_norm": 3.891215988627372,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.39990234375,
      "log_odds_ratio": -0.6719726324081421,
      "logits/chosen": -1.0011718273162842,
      "logits/rejected": -0.806445300579071,
      "logps/chosen": -1.2794921398162842,
      "logps/rejected": -1.5851562023162842,
      "loss": 1.5865,
      "nll_loss": 1.547265648841858,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.12790527939796448,
      "rewards/margins": 0.03055114671587944,
      "rewards/rejected": -0.15847167372703552,
      "step": 50
    },
    {
      "epoch": 0.007750435962022864,
      "grad_norm": 4.039127205522744,
      "learning_rate": 4.8e-06,
      "log_odds_chosen": 0.506274402141571,
      "log_odds_ratio": -0.626757800579071,
      "logits/chosen": -0.980664074420929,
      "logits/rejected": -0.8042968511581421,
      "logps/chosen": -1.211328148841858,
      "logps/rejected": -1.630273461341858,
      "loss": 1.5511,
      "nll_loss": 1.474609375,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.12102051079273224,
      "rewards/margins": 0.04187621921300888,
      "rewards/rejected": -0.16311034560203552,
      "step": 60
    },
    {
      "epoch": 0.009042175289026674,
      "grad_norm": 3.8212934458361683,
      "learning_rate": 5.6e-06,
      "log_odds_chosen": 0.5448242425918579,
      "log_odds_ratio": -0.5633789300918579,
      "logits/chosen": -1.006445288658142,
      "logits/rejected": -0.8216797113418579,
      "logps/chosen": -1.113867163658142,
      "logps/rejected": -1.533593773841858,
      "loss": 1.5156,
      "nll_loss": 1.4289062023162842,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.111328125,
      "rewards/margins": 0.04198760911822319,
      "rewards/rejected": -0.15329590439796448,
      "step": 70
    },
    {
      "epoch": 0.010333914616030485,
      "grad_norm": 4.139999531300743,
      "learning_rate": 6.4e-06,
      "log_odds_chosen": 0.42210692167282104,
      "log_odds_ratio": -0.647656261920929,
      "logits/chosen": -0.931445300579071,
      "logits/rejected": -0.792187511920929,
      "logps/chosen": -1.133203148841858,
      "logps/rejected": -1.473046898841858,
      "loss": 1.4818,
      "nll_loss": 1.534765601158142,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11335448920726776,
      "rewards/margins": 0.03402252122759819,
      "rewards/rejected": -0.14724120497703552,
      "step": 80
    },
    {
      "epoch": 0.011625653943034296,
      "grad_norm": 4.095742838448429,
      "learning_rate": 7.2e-06,
      "log_odds_chosen": 0.630126953125,
      "log_odds_ratio": -0.5445312261581421,
      "logits/chosen": -1.0246093273162842,
      "logits/rejected": -0.8226562738418579,
      "logps/chosen": -1.065820336341858,
      "logps/rejected": -1.55859375,
      "loss": 1.445,
      "nll_loss": 1.379296898841858,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.10654296725988388,
      "rewards/margins": 0.04931640625,
      "rewards/rejected": -0.15573731064796448,
      "step": 90
    },
    {
      "epoch": 0.012917393270038106,
      "grad_norm": 3.6159578716273097,
      "learning_rate": 8e-06,
      "log_odds_chosen": 0.5294555425643921,
      "log_odds_ratio": -0.5914062261581421,
      "logits/chosen": -0.971484363079071,
      "logits/rejected": -0.7845703363418579,
      "logps/chosen": -0.984375,
      "logps/rejected": -1.394921898841858,
      "loss": 1.4029,
      "nll_loss": 1.2683594226837158,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.09841308742761612,
      "rewards/margins": 0.04103546217083931,
      "rewards/rejected": -0.13945312798023224,
      "step": 100
    },
    {
      "epoch": 0.014209132597041917,
      "grad_norm": 2.9958642261714714,
      "learning_rate": 7.627700713964738e-06,
      "log_odds_chosen": 0.719775378704071,
      "log_odds_ratio": -0.512988269329071,
      "logits/chosen": -0.973828136920929,
      "logits/rejected": -0.774218738079071,
      "logps/chosen": -0.896289050579071,
      "logps/rejected": -1.4296875,
      "loss": 1.3539,
      "nll_loss": 1.283203125,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.08962402492761612,
      "rewards/margins": 0.0533599853515625,
      "rewards/rejected": -0.14309081435203552,
      "step": 110
    },
    {
      "epoch": 0.015500871924045728,
      "grad_norm": 2.763883836653562,
      "learning_rate": 7.3029674334022146e-06,
      "log_odds_chosen": 0.669055163860321,
      "log_odds_ratio": -0.5625976324081421,
      "logits/chosen": -0.920703113079071,
      "logits/rejected": -0.7630859613418579,
      "logps/chosen": -0.8623046875,
      "logps/rejected": -1.3429687023162842,
      "loss": 1.3486,
      "nll_loss": 1.367578148841858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.08620605617761612,
      "rewards/margins": 0.04801177978515625,
      "rewards/rejected": -0.13427734375,
      "step": 120
    },
    {
      "epoch": 0.016792611251049538,
      "grad_norm": 1.5794408995353049,
      "learning_rate": 7.016464154456233e-06,
      "log_odds_chosen": 0.602734386920929,
      "log_odds_ratio": -0.55615234375,
      "logits/chosen": -0.9853515625,
      "logits/rejected": -0.7574218511581421,
      "logps/chosen": -0.9164062738418579,
      "logps/rejected": -1.35546875,
      "loss": 1.2912,
      "nll_loss": 1.273046851158142,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.0916748046875,
      "rewards/margins": 0.0439605712890625,
      "rewards/rejected": -0.13557128608226776,
      "step": 130
    },
    {
      "epoch": 0.018084350578053347,
      "grad_norm": 1.5829096246432857,
      "learning_rate": 6.7612340378281325e-06,
      "log_odds_chosen": 0.6971679925918579,
      "log_odds_ratio": -0.5078125,
      "logits/chosen": -0.9798828363418579,
      "logits/rejected": -0.8091796636581421,
      "logps/chosen": -0.841992199420929,
      "logps/rejected": -1.327539086341858,
      "loss": 1.2957,
      "nll_loss": 1.210546851158142,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.08420410007238388,
      "rewards/margins": 0.04866638034582138,
      "rewards/rejected": -0.13288573920726776,
      "step": 140
    },
    {
      "epoch": 0.01937608990505716,
      "grad_norm": 1.432454201798768,
      "learning_rate": 6.531972647421809e-06,
      "log_odds_chosen": 0.56121826171875,
      "log_odds_ratio": -0.5838867425918579,
      "logits/chosen": -0.988085925579071,
      "logits/rejected": -0.792773425579071,
      "logps/chosen": -0.867382824420929,
      "logps/rejected": -1.2921874523162842,
      "loss": 1.282,
      "nll_loss": 1.230859398841858,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08674316108226776,
      "rewards/margins": 0.04250030592083931,
      "rewards/rejected": -0.12910155951976776,
      "step": 150
    },
    {
      "epoch": 0.02066782923206097,
      "grad_norm": 1.4396105037255706,
      "learning_rate": 6.3245553203367575e-06,
      "log_odds_chosen": 0.723095715045929,
      "log_odds_ratio": -0.5807129144668579,
      "logits/chosen": -0.976757824420929,
      "logits/rejected": -0.7816406488418579,
      "logps/chosen": -0.910937488079071,
      "logps/rejected": -1.438867211341858,
      "loss": 1.2837,
      "nll_loss": 1.1925780773162842,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.09111328423023224,
      "rewards/margins": 0.05277099460363388,
      "rewards/rejected": -0.14387206733226776,
      "step": 160
    },
    {
      "epoch": 0.02195956855906478,
      "grad_norm": 1.5718553661375603,
      "learning_rate": 6.135719910778963e-06,
      "log_odds_chosen": 0.43505859375,
      "log_odds_ratio": -0.6630859375,
      "logits/chosen": -0.9906250238418579,
      "logits/rejected": -0.8466796875,
      "logps/chosen": -0.8970702886581421,
      "logps/rejected": -1.2218749523162842,
      "loss": 1.3177,
      "nll_loss": 1.223046898841858,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08969726413488388,
      "rewards/margins": 0.032463837414979935,
      "rewards/rejected": -0.12224121391773224,
      "step": 170
    },
    {
      "epoch": 0.023251307886068592,
      "grad_norm": 1.3767240784172705,
      "learning_rate": 5.962847939999439e-06,
      "log_odds_chosen": 0.37218016386032104,
      "log_odds_ratio": -0.67138671875,
      "logits/chosen": -1.019140601158142,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.876953125,
      "logps/rejected": -1.1736328601837158,
      "loss": 1.2854,
      "nll_loss": 1.2292969226837158,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.08779297024011612,
      "rewards/margins": 0.02956542931497097,
      "rewards/rejected": -0.11733398586511612,
      "step": 180
    },
    {
      "epoch": 0.024543047213072402,
      "grad_norm": 1.3134241077743467,
      "learning_rate": 5.803810000880094e-06,
      "log_odds_chosen": 0.744091808795929,
      "log_odds_ratio": -0.54345703125,
      "logits/chosen": -1.037500023841858,
      "logits/rejected": -0.8558593988418579,
      "logps/chosen": -0.856249988079071,
      "logps/rejected": -1.39453125,
      "loss": 1.2528,
      "nll_loss": 1.2082030773162842,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08563232421875,
      "rewards/margins": 0.05387268215417862,
      "rewards/rejected": -0.1395263671875,
      "step": 190
    },
    {
      "epoch": 0.02583478654007621,
      "grad_norm": 1.6298294179019939,
      "learning_rate": 5.65685424949238e-06,
      "log_odds_chosen": 0.714306652545929,
      "log_odds_ratio": -0.529980480670929,
      "logits/chosen": -1.020117163658142,
      "logits/rejected": -0.7992187738418579,
      "logps/chosen": -0.8050781488418579,
      "logps/rejected": -1.302734375,
      "loss": 1.2724,
      "nll_loss": 1.170507788658142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.08059082180261612,
      "rewards/margins": 0.04983215406537056,
      "rewards/rejected": -0.13029785454273224,
      "step": 200
    },
    {
      "epoch": 0.027126525867080024,
      "grad_norm": 1.507921032061368,
      "learning_rate": 5.5205244747388325e-06,
      "log_odds_chosen": 0.7364257574081421,
      "log_odds_ratio": -0.5204101800918579,
      "logits/chosen": -1.0037109851837158,
      "logits/rejected": -0.818164050579071,
      "logps/chosen": -0.802539050579071,
      "logps/rejected": -1.3097655773162842,
      "loss": 1.2354,
      "nll_loss": 1.163476586341858,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.08029785007238388,
      "rewards/margins": 0.05073852464556694,
      "rewards/rejected": -0.13103027641773224,
      "step": 210
    },
    {
      "epoch": 0.028418265194083834,
      "grad_norm": 1.4978528887230322,
      "learning_rate": 5.393598899705936e-06,
      "log_odds_chosen": 0.698291003704071,
      "log_odds_ratio": -0.5777343511581421,
      "logits/chosen": -1.0333983898162842,
      "logits/rejected": -0.9166015386581421,
      "logps/chosen": -0.8734375238418579,
      "logps/rejected": -1.3830077648162842,
      "loss": 1.2485,
      "nll_loss": 1.175195336341858,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08736572414636612,
      "rewards/margins": 0.05084228515625,
      "rewards/rejected": -0.13820800185203552,
      "step": 220
    },
    {
      "epoch": 0.029710004521087643,
      "grad_norm": 1.3157923309930322,
      "learning_rate": 5.275043787166296e-06,
      "log_odds_chosen": 0.797985851764679,
      "log_odds_ratio": -0.5220702886581421,
      "logits/chosen": -0.968945324420929,
      "logits/rejected": -0.796875,
      "logps/chosen": -0.8003906011581421,
      "logps/rejected": -1.361328125,
      "loss": 1.2564,
      "nll_loss": 1.2208983898162842,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08001708984375,
      "rewards/margins": 0.05607910081744194,
      "rewards/rejected": -0.13613280653953552,
      "step": 230
    },
    {
      "epoch": 0.031001743848091456,
      "grad_norm": 1.3029095867800957,
      "learning_rate": 5.163977794943223e-06,
      "log_odds_chosen": 0.666735827922821,
      "log_odds_ratio": -0.5936523675918579,
      "logits/chosen": -0.9878906011581421,
      "logits/rejected": -0.7890625,
      "logps/chosen": -0.8501952886581421,
      "logps/rejected": -1.367578148841858,
      "loss": 1.243,
      "nll_loss": 1.213281273841858,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.0849609375,
      "rewards/margins": 0.05176391452550888,
      "rewards/rejected": -0.13676758110523224,
      "step": 240
    },
    {
      "epoch": 0.03229348317509526,
      "grad_norm": 1.6577406958794012,
      "learning_rate": 5.059644256269407e-06,
      "log_odds_chosen": 0.8603515625,
      "log_odds_ratio": -0.501757800579071,
      "logits/chosen": -1.0107421875,
      "logits/rejected": -0.782421886920929,
      "logps/chosen": -0.839062511920929,
      "logps/rejected": -1.458593726158142,
      "loss": 1.256,
      "nll_loss": 1.2488281726837158,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08391113579273224,
      "rewards/margins": 0.06190337985754013,
      "rewards/rejected": -0.14589843153953552,
      "step": 250
    },
    {
      "epoch": 0.033585222502099076,
      "grad_norm": 1.519776375229694,
      "learning_rate": 4.961389383568338e-06,
      "log_odds_chosen": 0.8218749761581421,
      "log_odds_ratio": -0.5096679925918579,
      "logits/chosen": -1.012304663658142,
      "logits/rejected": -0.8544921875,
      "logps/chosen": -0.8226562738418579,
      "logps/rejected": -1.393164038658142,
      "loss": 1.2413,
      "nll_loss": 1.154687523841858,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.08232422173023224,
      "rewards/margins": 0.056915283203125,
      "rewards/rejected": -0.13920898735523224,
      "step": 260
    },
    {
      "epoch": 0.03487696182910289,
      "grad_norm": 1.2548399290524286,
      "learning_rate": 4.8686449556014755e-06,
      "log_odds_chosen": 1.0139648914337158,
      "log_odds_ratio": -0.460205078125,
      "logits/chosen": -1.058984398841858,
      "logits/rejected": -0.84765625,
      "logps/chosen": -0.826367199420929,
      "logps/rejected": -1.546875,
      "loss": 1.2262,
      "nll_loss": 1.21484375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.08266601711511612,
      "rewards/margins": 0.07209014892578125,
      "rewards/rejected": -0.15458984673023224,
      "step": 270
    },
    {
      "epoch": 0.036168701156106695,
      "grad_norm": 1.5984353445925055,
      "learning_rate": 4.780914437337574e-06,
      "log_odds_chosen": 0.66357421875,
      "log_odds_ratio": -0.5630859136581421,
      "logits/chosen": -1.040429711341858,
      "logits/rejected": -0.8501952886581421,
      "logps/chosen": -0.8548828363418579,
      "logps/rejected": -1.3201172351837158,
      "loss": 1.2324,
      "nll_loss": 1.2384765148162842,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.08551025390625,
      "rewards/margins": 0.04647216945886612,
      "rewards/rejected": -0.1319580078125,
      "step": 280
    },
    {
      "epoch": 0.03746044048311051,
      "grad_norm": 1.6589910923977143,
      "learning_rate": 4.697761756117627e-06,
      "log_odds_chosen": 0.772900402545929,
      "log_odds_ratio": -0.536572277545929,
      "logits/chosen": -1.0710937976837158,
      "logits/rejected": -0.9033203125,
      "logps/chosen": -0.833203136920929,
      "logps/rejected": -1.399999976158142,
      "loss": 1.2117,
      "nll_loss": 1.136132836341858,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.08328857272863388,
      "rewards/margins": 0.05676879733800888,
      "rewards/rejected": -0.14006347954273224,
      "step": 290
    },
    {
      "epoch": 0.03875217981011432,
      "grad_norm": 1.52233985546576,
      "learning_rate": 4.618802153517006e-06,
      "log_odds_chosen": 0.70599365234375,
      "log_odds_ratio": -0.531933605670929,
      "logits/chosen": -1.0207030773162842,
      "logits/rejected": -0.893750011920929,
      "logps/chosen": -0.819140613079071,
      "logps/rejected": -1.3171875476837158,
      "loss": 1.2061,
      "nll_loss": 1.2380859851837158,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.08195801079273224,
      "rewards/margins": 0.04980163648724556,
      "rewards/rejected": -0.13173827528953552,
      "step": 300
    },
    {
      "epoch": 0.04004391913711813,
      "grad_norm": 1.357780293868318,
      "learning_rate": 4.543694673976518e-06,
      "log_odds_chosen": 0.9074951410293579,
      "log_odds_ratio": -0.517578125,
      "logits/chosen": -1.0673828125,
      "logits/rejected": -0.87109375,
      "logps/chosen": -0.8226562738418579,
      "logps/rejected": -1.493554711341858,
      "loss": 1.2304,
      "nll_loss": 1.100000023841858,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.08225097507238388,
      "rewards/margins": 0.06709747016429901,
      "rewards/rejected": -0.14919432997703552,
      "step": 310
    },
    {
      "epoch": 0.04133565846412194,
      "grad_norm": 1.3116044732536714,
      "learning_rate": 4.472135954999579e-06,
      "log_odds_chosen": 0.7847900390625,
      "log_odds_ratio": -0.5294433832168579,
      "logits/chosen": -0.9779297113418579,
      "logits/rejected": -0.7958984375,
      "logps/chosen": -0.8515625,
      "logps/rejected": -1.4207031726837158,
      "loss": 1.2428,
      "nll_loss": 1.2488281726837158,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.08510742336511612,
      "rewards/margins": 0.056915283203125,
      "rewards/rejected": -0.14206543564796448,
      "step": 320
    },
    {
      "epoch": 0.04262739779112575,
      "grad_norm": 1.3217309885822108,
      "learning_rate": 4.403855060505443e-06,
      "log_odds_chosen": 0.754589855670929,
      "log_odds_ratio": -0.5194336175918579,
      "logits/chosen": -0.9449218511581421,
      "logits/rejected": -0.8314453363418579,
      "logps/chosen": -0.7962890863418579,
      "logps/rejected": -1.296484351158142,
      "loss": 1.2492,
      "nll_loss": 1.1677734851837158,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07962646335363388,
      "rewards/margins": 0.05009765550494194,
      "rewards/rejected": -0.12978515028953552,
      "step": 330
    },
    {
      "epoch": 0.04391913711812956,
      "grad_norm": 1.3450790759999323,
      "learning_rate": 4.338609156373123e-06,
      "log_odds_chosen": 0.742382824420929,
      "log_odds_ratio": -0.55126953125,
      "logits/chosen": -1.0363280773162842,
      "logits/rejected": -0.853320300579071,
      "logps/chosen": -0.8544921875,
      "logps/rejected": -1.3798828125,
      "loss": 1.1961,
      "nll_loss": 1.2121093273162842,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.08552245795726776,
      "rewards/margins": 0.052581787109375,
      "rewards/rejected": -0.1380615234375,
      "step": 340
    },
    {
      "epoch": 0.04521087644513337,
      "grad_norm": 1.138834104724891,
      "learning_rate": 4.27617987059879e-06,
      "log_odds_chosen": 0.912792980670929,
      "log_odds_ratio": -0.501269519329071,
      "logits/chosen": -1.0107421875,
      "logits/rejected": -0.852734386920929,
      "logps/chosen": -0.8119140863418579,
      "logps/rejected": -1.474023461341858,
      "loss": 1.2553,
      "nll_loss": 1.1662108898162842,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.0811767578125,
      "rewards/margins": 0.06629333645105362,
      "rewards/rejected": -0.14743652939796448,
      "step": 350
    },
    {
      "epoch": 0.046502615772137185,
      "grad_norm": 1.6315524180890024,
      "learning_rate": 4.216370213557839e-06,
      "log_odds_chosen": 0.9598633050918579,
      "log_odds_ratio": -0.49492186307907104,
      "logits/chosen": -1.030664086341858,
      "logits/rejected": -0.826367199420929,
      "logps/chosen": -0.781054675579071,
      "logps/rejected": -1.508398413658142,
      "loss": 1.2208,
      "nll_loss": 1.104101538658142,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07805176079273224,
      "rewards/margins": 0.07277832180261612,
      "rewards/rejected": -0.15080566704273224,
      "step": 360
    },
    {
      "epoch": 0.04779435509914099,
      "grad_norm": 1.1630076751733809,
      "learning_rate": 4.15900195928029e-06,
      "log_odds_chosen": 0.9056152105331421,
      "log_odds_ratio": -0.558300793170929,
      "logits/chosen": -0.976367175579071,
      "logits/rejected": -0.818359375,
      "logps/chosen": -0.773632824420929,
      "logps/rejected": -1.421289086341858,
      "loss": 1.2368,
      "nll_loss": 1.1970703601837158,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07730712741613388,
      "rewards/margins": 0.06474151462316513,
      "rewards/rejected": -0.14208984375,
      "step": 370
    },
    {
      "epoch": 0.049086094426144804,
      "grad_norm": 1.198531705797501,
      "learning_rate": 4.103913408340617e-06,
      "log_odds_chosen": 1.035791039466858,
      "log_odds_ratio": -0.4349609315395355,
      "logits/chosen": -1.05078125,
      "logits/rejected": -0.8255859613418579,
      "logps/chosen": -0.788281261920929,
      "logps/rejected": -1.501953125,
      "loss": 1.2,
      "nll_loss": 1.165624976158142,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07879638671875,
      "rewards/margins": 0.07134399563074112,
      "rewards/rejected": -0.15012207627296448,
      "step": 380
    },
    {
      "epoch": 0.05037783375314862,
      "grad_norm": 1.2866984720866645,
      "learning_rate": 4.050957468334666e-06,
      "log_odds_chosen": 0.688720703125,
      "log_odds_ratio": -0.5982421636581421,
      "logits/chosen": -1.0841796398162842,
      "logits/rejected": -0.886914074420929,
      "logps/chosen": -0.839062511920929,
      "logps/rejected": -1.362890601158142,
      "loss": 1.2168,
      "nll_loss": 1.108984351158142,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.0838623046875,
      "rewards/margins": 0.05236206203699112,
      "rewards/rejected": -0.13623046875,
      "step": 390
    },
    {
      "epoch": 0.05166957308015242,
      "grad_norm": 1.3060742996838322,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.8260253667831421,
      "log_odds_ratio": -0.510058581829071,
      "logits/chosen": -1.0810546875,
      "logits/rejected": -0.869335949420929,
      "logps/chosen": -0.76953125,
      "logps/rejected": -1.3564453125,
      "loss": 1.2281,
      "nll_loss": 1.0818359851837158,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07695312798023224,
      "rewards/margins": 0.05866088718175888,
      "rewards/rejected": -0.13557128608226776,
      "step": 400
    },
    {
      "epoch": 0.052961312407156236,
      "grad_norm": 1.3232085389194796,
      "learning_rate": 3.950918386598359e-06,
      "log_odds_chosen": 0.726245105266571,
      "log_odds_ratio": -0.5389648675918579,
      "logits/chosen": -1.0363280773162842,
      "logits/rejected": -0.908203125,
      "logps/chosen": -0.8062499761581421,
      "logps/rejected": -1.2902343273162842,
      "loss": 1.2057,
      "nll_loss": 1.1681640148162842,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08051757514476776,
      "rewards/margins": 0.04848175123333931,
      "rewards/rejected": -0.12900391221046448,
      "step": 410
    },
    {
      "epoch": 0.05425305173416005,
      "grad_norm": 1.3310533920182912,
      "learning_rate": 3.903600291794132e-06,
      "log_odds_chosen": 0.7430664300918579,
      "log_odds_ratio": -0.568652331829071,
      "logits/chosen": -0.994140625,
      "logits/rejected": -0.8783203363418579,
      "logps/chosen": -0.7994140386581421,
      "logps/rejected": -1.33984375,
      "loss": 1.179,
      "nll_loss": 1.1482422351837158,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07991943508386612,
      "rewards/margins": 0.05406494066119194,
      "rewards/rejected": -0.13383789360523224,
      "step": 420
    },
    {
      "epoch": 0.055544791061163855,
      "grad_norm": 1.2551346500804013,
      "learning_rate": 3.857942577363297e-06,
      "log_odds_chosen": 0.844189465045929,
      "log_odds_ratio": -0.5028320550918579,
      "logits/chosen": -1.0994141101837158,
      "logits/rejected": -0.923828125,
      "logps/chosen": -0.796679675579071,
      "logps/rejected": -1.405859351158142,
      "loss": 1.1981,
      "nll_loss": 1.175390601158142,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07963867485523224,
      "rewards/margins": 0.06090698391199112,
      "rewards/rejected": -0.14060059189796448,
      "step": 430
    },
    {
      "epoch": 0.05683653038816767,
      "grad_norm": 37.25593914170561,
      "learning_rate": 3.813850356982369e-06,
      "log_odds_chosen": 1.015869140625,
      "log_odds_ratio": -0.4696289002895355,
      "logits/chosen": -1.0460937023162842,
      "logits/rejected": -0.838085949420929,
      "logps/chosen": -0.7992187738418579,
      "logps/rejected": -1.5291016101837158,
      "loss": 1.1967,
      "nll_loss": 1.1847655773162842,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07993163913488388,
      "rewards/margins": 0.07289429008960724,
      "rewards/rejected": -0.15268555283546448,
      "step": 440
    },
    {
      "epoch": 0.05812826971517148,
      "grad_norm": 1.7214286466475308,
      "learning_rate": 3.7712361663282537e-06,
      "log_odds_chosen": 0.7060302495956421,
      "log_odds_ratio": -0.5516601800918579,
      "logits/chosen": -1.0437500476837158,
      "logits/rejected": -0.8228515386581421,
      "logps/chosen": -0.8017578125,
      "logps/rejected": -1.326171875,
      "loss": 1.1972,
      "nll_loss": 1.181054711341858,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.08017577975988388,
      "rewards/margins": 0.05238189548254013,
      "rewards/rejected": -0.132568359375,
      "step": 450
    },
    {
      "epoch": 0.05942000904217529,
      "grad_norm": 1.231591554381852,
      "learning_rate": 3.730019232961255e-06,
      "log_odds_chosen": 1.069921851158142,
      "log_odds_ratio": -0.4735351502895355,
      "logits/chosen": -1.0460937023162842,
      "logits/rejected": -0.8089843988418579,
      "logps/chosen": -0.8062499761581421,
      "logps/rejected": -1.574609398841858,
      "loss": 1.2112,
      "nll_loss": 1.124609351158142,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08060302585363388,
      "rewards/margins": 0.07672119140625,
      "rewards/rejected": -0.157470703125,
      "step": 460
    },
    {
      "epoch": 0.0607117483691791,
      "grad_norm": 1.4297656309951992,
      "learning_rate": 3.6901248321155403e-06,
      "log_odds_chosen": 0.815380871295929,
      "log_odds_ratio": -0.552734375,
      "logits/chosen": -1.043554663658142,
      "logits/rejected": -0.9156249761581421,
      "logps/chosen": -0.8023437261581421,
      "logps/rejected": -1.3820312023162842,
      "loss": 1.1824,
      "nll_loss": 1.086328148841858,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.08028564602136612,
      "rewards/margins": 0.05801696702837944,
      "rewards/rejected": -0.13825683295726776,
      "step": 470
    },
    {
      "epoch": 0.06200348769618291,
      "grad_norm": 1.2813646787848072,
      "learning_rate": 3.6514837167011073e-06,
      "log_odds_chosen": 0.914013683795929,
      "log_odds_ratio": -0.515429675579071,
      "logits/chosen": -1.0623047351837158,
      "logits/rejected": -0.872265636920929,
      "logps/chosen": -0.80078125,
      "logps/rejected": -1.4591796398162842,
      "loss": 1.1598,
      "nll_loss": 1.106054663658142,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.080078125,
      "rewards/margins": 0.06585693359375,
      "rewards/rejected": -0.14597168564796448,
      "step": 480
    },
    {
      "epoch": 0.06329522702318673,
      "grad_norm": 1.295033903721843,
      "learning_rate": 3.6140316116210052e-06,
      "log_odds_chosen": 0.8544677495956421,
      "log_odds_ratio": -0.51513671875,
      "logits/chosen": -1.107031226158142,
      "logits/rejected": -0.889843761920929,
      "logps/chosen": -0.7835937738418579,
      "logps/rejected": -1.3917968273162842,
      "loss": 1.1979,
      "nll_loss": 1.1541016101837158,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.078369140625,
      "rewards/margins": 0.060831449925899506,
      "rewards/rejected": -0.13908691704273224,
      "step": 490
    },
    {
      "epoch": 0.06458696635019053,
      "grad_norm": 1.4128400593890962,
      "learning_rate": 3.5777087639996634e-06,
      "log_odds_chosen": 0.8157714605331421,
      "log_odds_ratio": -0.4935058653354645,
      "logits/chosen": -1.1017577648162842,
      "logits/rejected": -0.8431640863418579,
      "logps/chosen": -0.7777343988418579,
      "logps/rejected": -1.3419921398162842,
      "loss": 1.2128,
      "nll_loss": 1.0935547351837158,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0777587890625,
      "rewards/margins": 0.05640869215130806,
      "rewards/rejected": -0.13420410454273224,
      "step": 500
    },
    {
      "epoch": 0.06587870567719434,
      "grad_norm": 1.2852041710404891,
      "learning_rate": 3.5424595421603814e-06,
      "log_odds_chosen": 1.0095703601837158,
      "log_odds_ratio": -0.4862304627895355,
      "logits/chosen": -0.9966796636581421,
      "logits/rejected": -0.8529297113418579,
      "logps/chosen": -0.7749999761581421,
      "logps/rejected": -1.4890625476837158,
      "loss": 1.1694,
      "nll_loss": 1.1570312976837158,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07744140923023224,
      "rewards/margins": 0.07141723483800888,
      "rewards/rejected": -0.14890137314796448,
      "step": 510
    },
    {
      "epoch": 0.06717044500419815,
      "grad_norm": 1.4925154616120668,
      "learning_rate": 3.5082320772281165e-06,
      "log_odds_chosen": 0.9048095941543579,
      "log_odds_ratio": -0.53125,
      "logits/chosen": -1.1417968273162842,
      "logits/rejected": -0.8558593988418579,
      "logps/chosen": -0.8736327886581421,
      "logps/rejected": -1.535742163658142,
      "loss": 1.2036,
      "nll_loss": 1.198632836341858,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08736572414636612,
      "rewards/margins": 0.066162109375,
      "rewards/rejected": -0.15361328423023224,
      "step": 520
    },
    {
      "epoch": 0.06846218433120196,
      "grad_norm": 1.423686346215787,
      "learning_rate": 3.474977942104555e-06,
      "log_odds_chosen": 0.978955090045929,
      "log_odds_ratio": -0.504638671875,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.845898449420929,
      "logps/chosen": -0.81640625,
      "logps/rejected": -1.526953101158142,
      "loss": 1.2069,
      "nll_loss": 1.1472656726837158,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.08162841945886612,
      "rewards/margins": 0.07106475532054901,
      "rewards/rejected": -0.15266112983226776,
      "step": 530
    },
    {
      "epoch": 0.06975392365820578,
      "grad_norm": 1.1828471125966862,
      "learning_rate": 3.442651863295481e-06,
      "log_odds_chosen": 0.994335949420929,
      "log_odds_ratio": -0.482421875,
      "logits/chosen": -1.140625,
      "logits/rejected": -0.908203125,
      "logps/chosen": -0.7484375238418579,
      "logps/rejected": -1.4425780773162842,
      "loss": 1.1964,
      "nll_loss": 1.115234375,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07487793266773224,
      "rewards/margins": 0.06933899223804474,
      "rewards/rejected": -0.1441650390625,
      "step": 540
    },
    {
      "epoch": 0.07104566298520959,
      "grad_norm": 1.6322240817234488,
      "learning_rate": 3.4112114616897665e-06,
      "log_odds_chosen": 1.223242163658142,
      "log_odds_ratio": -0.4161132872104645,
      "logits/chosen": -1.0632812976837158,
      "logits/rejected": -0.8373047113418579,
      "logps/chosen": -0.7730468511581421,
      "logps/rejected": -1.619140625,
      "loss": 1.1591,
      "nll_loss": 1.0417969226837158,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07729492336511612,
      "rewards/margins": 0.08452758938074112,
      "rewards/rejected": -0.16171875596046448,
      "step": 550
    },
    {
      "epoch": 0.07233740231221339,
      "grad_norm": 1.1804772247297062,
      "learning_rate": 3.3806170189140663e-06,
      "log_odds_chosen": 0.835522472858429,
      "log_odds_ratio": -0.502636730670929,
      "logits/chosen": -1.0812499523162842,
      "logits/rejected": -0.9173828363418579,
      "logps/chosen": -0.8060547113418579,
      "logps/rejected": -1.3992187976837158,
      "loss": 1.1847,
      "nll_loss": 1.0998046398162842,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08054199069738388,
      "rewards/margins": 0.05933227390050888,
      "rewards/rejected": -0.13984374701976776,
      "step": 560
    },
    {
      "epoch": 0.0736291416392172,
      "grad_norm": 1.4282885925870714,
      "learning_rate": 3.350831266333564e-06,
      "log_odds_chosen": 1.1495361328125,
      "log_odds_ratio": -0.4527831971645355,
      "logits/chosen": -1.0382812023162842,
      "logits/rejected": -0.8695312738418579,
      "logps/chosen": -0.778124988079071,
      "logps/rejected": -1.6056640148162842,
      "loss": 1.166,
      "nll_loss": 1.134179711341858,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07785644382238388,
      "rewards/margins": 0.082611083984375,
      "rewards/rejected": -0.160400390625,
      "step": 570
    },
    {
      "epoch": 0.07492088096622102,
      "grad_norm": 1.3784310853971684,
      "learning_rate": 3.3218191941495984e-06,
      "log_odds_chosen": 0.79150390625,
      "log_odds_ratio": -0.52734375,
      "logits/chosen": -1.07421875,
      "logits/rejected": -0.8960937261581421,
      "logps/chosen": -0.839648425579071,
      "logps/rejected": -1.3927733898162842,
      "loss": 1.1531,
      "nll_loss": 1.116601586341858,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.083984375,
      "rewards/margins": 0.0552825927734375,
      "rewards/rejected": -0.13923339545726776,
      "step": 580
    },
    {
      "epoch": 0.07621262029322483,
      "grad_norm": 1.2078009848726674,
      "learning_rate": 3.293547878370473e-06,
      "log_odds_chosen": 0.8414062261581421,
      "log_odds_ratio": -0.509472668170929,
      "logits/chosen": -0.9970703125,
      "logits/rejected": -0.8374999761581421,
      "logps/chosen": -0.815625011920929,
      "logps/rejected": -1.3898437023162842,
      "loss": 1.1933,
      "nll_loss": 1.180273413658142,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.08157958835363388,
      "rewards/margins": 0.05731201171875,
      "rewards/rejected": -0.13889160752296448,
      "step": 590
    },
    {
      "epoch": 0.07750435962022864,
      "grad_norm": 1.184462715911394,
      "learning_rate": 3.2659863237109044e-06,
      "log_odds_chosen": 0.911914050579071,
      "log_odds_ratio": -0.49580079317092896,
      "logits/chosen": -1.198632836341858,
      "logits/rejected": -0.9507812261581421,
      "logps/chosen": -0.791210949420929,
      "logps/rejected": -1.442968726158142,
      "loss": 1.173,
      "nll_loss": 1.0109374523162842,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07911376655101776,
      "rewards/margins": 0.06525878608226776,
      "rewards/rejected": -0.1444091796875,
      "step": 600
    },
    {
      "epoch": 0.07879609894723245,
      "grad_norm": 1.6021946518723025,
      "learning_rate": 3.239105320715664e-06,
      "log_odds_chosen": 0.998486340045929,
      "log_odds_ratio": -0.45708006620407104,
      "logits/chosen": -1.095703125,
      "logits/rejected": -0.868359386920929,
      "logps/chosen": -0.8216797113418579,
      "logps/rejected": -1.5265624523162842,
      "loss": 1.1769,
      "nll_loss": 1.117578148841858,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0821533203125,
      "rewards/margins": 0.070526123046875,
      "rewards/rejected": -0.15280762314796448,
      "step": 610
    },
    {
      "epoch": 0.08008783827423625,
      "grad_norm": 1.2909582175868575,
      "learning_rate": 3.2128773156099956e-06,
      "log_odds_chosen": 0.784375011920929,
      "log_odds_ratio": -0.502246081829071,
      "logits/chosen": -1.056249976158142,
      "logits/rejected": -0.888867199420929,
      "logps/chosen": -0.8443359136581421,
      "logps/rejected": -1.3855469226837158,
      "loss": 1.173,
      "nll_loss": 1.1560547351837158,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.08444824069738388,
      "rewards/margins": 0.05415039137005806,
      "rewards/rejected": -0.13862304389476776,
      "step": 620
    },
    {
      "epoch": 0.08137957760124007,
      "grad_norm": 1.3061074492525164,
      "learning_rate": 3.187276291558383e-06,
      "log_odds_chosen": 1.0955078601837158,
      "log_odds_ratio": -0.44916993379592896,
      "logits/chosen": -1.0593750476837158,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.7466796636581421,
      "logps/rejected": -1.5251953601837158,
      "loss": 1.1792,
      "nll_loss": 1.1199219226837158,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07469482719898224,
      "rewards/margins": 0.07789306342601776,
      "rewards/rejected": -0.152587890625,
      "step": 630
    },
    {
      "epoch": 0.08267131692824388,
      "grad_norm": 1.1987732550561676,
      "learning_rate": 3.1622776601683788e-06,
      "log_odds_chosen": 0.9761718511581421,
      "log_odds_ratio": -0.476806640625,
      "logits/chosen": -1.096093773841858,
      "logits/rejected": -0.8511718511581421,
      "logps/chosen": -0.843554675579071,
      "logps/rejected": -1.523828148841858,
      "loss": 1.169,
      "nll_loss": 1.141210913658142,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.08432617038488388,
      "rewards/margins": 0.06809081882238388,
      "rewards/rejected": -0.15227051079273224,
      "step": 640
    },
    {
      "epoch": 0.08396305625524769,
      "grad_norm": 1.231220076651413,
      "learning_rate": 3.1378581622109444e-06,
      "log_odds_chosen": 0.9027343988418579,
      "log_odds_ratio": -0.5249999761581421,
      "logits/chosen": -1.1154296398162842,
      "logits/rejected": -0.93359375,
      "logps/chosen": -0.7671874761581421,
      "logps/rejected": -1.417382836341858,
      "loss": 1.178,
      "nll_loss": 1.080078125,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07673339545726776,
      "rewards/margins": 0.06498870998620987,
      "rewards/rejected": -0.14165039360523224,
      "step": 650
    },
    {
      "epoch": 0.0852547955822515,
      "grad_norm": 1.177895385767675,
      "learning_rate": 3.113995776646092e-06,
      "log_odds_chosen": 0.99658203125,
      "log_odds_ratio": -0.513134777545929,
      "logits/chosen": -1.125585913658142,
      "logits/rejected": -0.884570300579071,
      "logps/chosen": -0.804882824420929,
      "logps/rejected": -1.560937523841858,
      "loss": 1.1983,
      "nll_loss": 1.124414086341858,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.0804443359375,
      "rewards/margins": 0.07564697414636612,
      "rewards/rejected": -0.15605469048023224,
      "step": 660
    },
    {
      "epoch": 0.08654653490925532,
      "grad_norm": 1.3589749210923037,
      "learning_rate": 3.090669637145023e-06,
      "log_odds_chosen": 1.0281250476837158,
      "log_odds_ratio": -0.4866699278354645,
      "logits/chosen": -1.07421875,
      "logits/rejected": -0.890625,
      "logps/chosen": -0.7826172113418579,
      "logps/rejected": -1.531640648841858,
      "loss": 1.1615,
      "nll_loss": 1.0947265625,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0782470703125,
      "rewards/margins": 0.07490234076976776,
      "rewards/rejected": -0.15314941108226776,
      "step": 670
    },
    {
      "epoch": 0.08783827423625912,
      "grad_norm": 1.3666977031934535,
      "learning_rate": 3.0678599553894814e-06,
      "log_odds_chosen": 1.055932641029358,
      "log_odds_ratio": -0.4605468809604645,
      "logits/chosen": -1.038671851158142,
      "logits/rejected": -0.826953113079071,
      "logps/chosen": -0.7533203363418579,
      "logps/rejected": -1.4949219226837158,
      "loss": 1.1771,
      "nll_loss": 1.1388671398162842,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07530517876148224,
      "rewards/margins": 0.07410278171300888,
      "rewards/rejected": -0.14948730170726776,
      "step": 680
    },
    {
      "epoch": 0.08913001356326293,
      "grad_norm": 1.268584413624482,
      "learning_rate": 3.0455479505075235e-06,
      "log_odds_chosen": 0.9961913824081421,
      "log_odds_ratio": -0.502734363079071,
      "logits/chosen": -1.0519530773162842,
      "logits/rejected": -0.849414050579071,
      "logps/chosen": -0.818164050579071,
      "logps/rejected": -1.5281250476837158,
      "loss": 1.1773,
      "nll_loss": 1.208984375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.08179931342601776,
      "rewards/margins": 0.07105102390050888,
      "rewards/rejected": -0.15278320014476776,
      "step": 690
    },
    {
      "epoch": 0.09042175289026674,
      "grad_norm": 1.2341733102160282,
      "learning_rate": 3.0237157840738173e-06,
      "log_odds_chosen": 0.9613037109375,
      "log_odds_ratio": -0.513867199420929,
      "logits/chosen": -1.0740234851837158,
      "logits/rejected": -0.9253906011581421,
      "logps/chosen": -0.8203125,
      "logps/rejected": -1.5166015625,
      "loss": 1.1401,
      "nll_loss": 1.076562523841858,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.08194579929113388,
      "rewards/margins": 0.06969757378101349,
      "rewards/rejected": -0.15163573622703552,
      "step": 700
    },
    {
      "epoch": 0.09171349221727056,
      "grad_norm": 1.5297617000438264,
      "learning_rate": 3.002346500163206e-06,
      "log_odds_chosen": 1.1159179210662842,
      "log_odds_ratio": -0.45068359375,
      "logits/chosen": -1.1667969226837158,
      "logits/rejected": -0.9107421636581421,
      "logps/chosen": -0.8003906011581421,
      "logps/rejected": -1.6300780773162842,
      "loss": 1.1659,
      "nll_loss": 1.177734375,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08001708984375,
      "rewards/margins": 0.08297576755285263,
      "rewards/rejected": -0.16303710639476776,
      "step": 710
    },
    {
      "epoch": 0.09300523154427437,
      "grad_norm": 1.5977789176609785,
      "learning_rate": 2.9814239699997195e-06,
      "log_odds_chosen": 0.7395995855331421,
      "log_odds_ratio": -0.53271484375,
      "logits/chosen": -1.1212890148162842,
      "logits/rejected": -0.9505859613418579,
      "logps/chosen": -0.7992187738418579,
      "logps/rejected": -1.313085913658142,
      "loss": 1.1897,
      "nll_loss": 1.1154296398162842,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07990722358226776,
      "rewards/margins": 0.05129547044634819,
      "rewards/rejected": -0.13120117783546448,
      "step": 720
    },
    {
      "epoch": 0.09429697087127818,
      "grad_norm": 1.1867570389615867,
      "learning_rate": 2.9609328407904207e-06,
      "log_odds_chosen": 1.176123023033142,
      "log_odds_ratio": -0.4395996034145355,
      "logits/chosen": -0.9892578125,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.770312488079071,
      "logps/rejected": -1.639062523841858,
      "loss": 1.1576,
      "nll_loss": 1.115820288658142,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07698974758386612,
      "rewards/margins": 0.0869293212890625,
      "rewards/rejected": -0.16386719048023224,
      "step": 730
    },
    {
      "epoch": 0.09558871019828198,
      "grad_norm": 1.2262028542430676,
      "learning_rate": 2.940858488375231e-06,
      "log_odds_chosen": 1.028906226158142,
      "log_odds_ratio": -0.47358399629592896,
      "logits/chosen": -1.176171898841858,
      "logits/rejected": -0.9681640863418579,
      "logps/chosen": -0.822070300579071,
      "logps/rejected": -1.5753905773162842,
      "loss": 1.195,
      "nll_loss": 1.1794922351837158,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08222655951976776,
      "rewards/margins": 0.075347900390625,
      "rewards/rejected": -0.15744629502296448,
      "step": 740
    },
    {
      "epoch": 0.0968804495252858,
      "grad_norm": 1.2567263809307754,
      "learning_rate": 2.9211869733608857e-06,
      "log_odds_chosen": 0.90283203125,
      "log_odds_ratio": -0.523242175579071,
      "logits/chosen": -1.0812499523162842,
      "logits/rejected": -0.9212890863418579,
      "logps/chosen": -0.8515625,
      "logps/rejected": -1.489843726158142,
      "loss": 1.1584,
      "nll_loss": 1.109960913658142,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.08521728217601776,
      "rewards/margins": 0.063720703125,
      "rewards/rejected": -0.14882811903953552,
      "step": 750
    },
    {
      "epoch": 0.09817218885228961,
      "grad_norm": 3.3244432009918663,
      "learning_rate": 2.901905000440047e-06,
      "log_odds_chosen": 1.102636694908142,
      "log_odds_ratio": -0.4515136778354645,
      "logits/chosen": -1.136328101158142,
      "logits/rejected": -0.9361327886581421,
      "logps/chosen": -0.7294921875,
      "logps/rejected": -1.5134766101837158,
      "loss": 1.1517,
      "nll_loss": 1.032617211341858,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07288818061351776,
      "rewards/margins": 0.07844848930835724,
      "rewards/rejected": -0.1514892578125,
      "step": 760
    },
    {
      "epoch": 0.09946392817929342,
      "grad_norm": 1.3972431013424784,
      "learning_rate": 2.8829998806257885e-06,
      "log_odds_chosen": 1.2353515625,
      "log_odds_ratio": -0.447021484375,
      "logits/chosen": -1.1953125,
      "logits/rejected": -0.8833984136581421,
      "logps/chosen": -0.8006836175918579,
      "logps/rejected": -1.703125,
      "loss": 1.1641,
      "nll_loss": 1.120703101158142,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.08011474460363388,
      "rewards/margins": 0.09016112983226776,
      "rewards/rejected": -0.17014160752296448,
      "step": 770
    },
    {
      "epoch": 0.10075566750629723,
      "grad_norm": 1.3282869052003041,
      "learning_rate": 2.8644594961577314e-06,
      "log_odds_chosen": 1.1076171398162842,
      "log_odds_ratio": -0.4720703065395355,
      "logits/chosen": -1.096289038658142,
      "logits/rejected": -0.905078113079071,
      "logps/chosen": -0.7958984375,
      "logps/rejected": -1.596093773841858,
      "loss": 1.1713,
      "nll_loss": 1.0832030773162842,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07962646335363388,
      "rewards/margins": 0.07999267429113388,
      "rewards/rejected": -0.15961913764476776,
      "step": 780
    },
    {
      "epoch": 0.10204740683330105,
      "grad_norm": 1.4352635500664344,
      "learning_rate": 2.84627226785928e-06,
      "log_odds_chosen": 0.95465087890625,
      "log_odds_ratio": -0.5184081792831421,
      "logits/chosen": -1.177148461341858,
      "logits/rejected": -0.9205077886581421,
      "logps/chosen": -0.8285156488418579,
      "logps/rejected": -1.559960961341858,
      "loss": 1.1289,
      "nll_loss": 1.1599609851837158,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08281250298023224,
      "rewards/margins": 0.07317809760570526,
      "rewards/rejected": -0.1561279296875,
      "step": 790
    },
    {
      "epoch": 0.10333914616030485,
      "grad_norm": 1.5912963583262987,
      "learning_rate": 2.82842712474619e-06,
      "log_odds_chosen": 0.859606921672821,
      "log_odds_ratio": -0.5381835699081421,
      "logits/chosen": -1.102929711341858,
      "logits/rejected": -0.9320312738418579,
      "logps/chosen": -0.807812511920929,
      "logps/rejected": -1.4509766101837158,
      "loss": 1.1634,
      "nll_loss": 1.1130859851837158,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.08082275092601776,
      "rewards/margins": 0.06420745700597763,
      "rewards/rejected": -0.14492186903953552,
      "step": 800
    },
    {
      "epoch": 0.10463088548730866,
      "grad_norm": 1.4630243987866478,
      "learning_rate": 2.810913475705226e-06,
      "log_odds_chosen": 1.005957007408142,
      "log_odds_ratio": -0.4715820252895355,
      "logits/chosen": -1.104882836341858,
      "logits/rejected": -0.878125011920929,
      "logps/chosen": -0.7671874761581421,
      "logps/rejected": -1.484765648841858,
      "loss": 1.1361,
      "nll_loss": 1.0478515625,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07667236030101776,
      "rewards/margins": 0.07175903022289276,
      "rewards/rejected": -0.14853516221046448,
      "step": 810
    },
    {
      "epoch": 0.10592262481431247,
      "grad_norm": 1.2872169145102348,
      "learning_rate": 2.7937211830783128e-06,
      "log_odds_chosen": 0.8750976324081421,
      "log_odds_ratio": -0.5582031011581421,
      "logits/chosen": -1.0945312976837158,
      "logits/rejected": -0.890820324420929,
      "logps/chosen": -0.8228515386581421,
      "logps/rejected": -1.457421898841858,
      "loss": 1.1461,
      "nll_loss": 1.187890648841858,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.08226318657398224,
      "rewards/margins": 0.06349487602710724,
      "rewards/rejected": -0.14582519233226776,
      "step": 820
    },
    {
      "epoch": 0.10721436414131628,
      "grad_norm": 1.3456631047705214,
      "learning_rate": 2.776840538002493e-06,
      "log_odds_chosen": 1.102148413658142,
      "log_odds_ratio": -0.451416015625,
      "logits/chosen": -1.145898461341858,
      "logits/rejected": -0.923828125,
      "logps/chosen": -0.757617175579071,
      "logps/rejected": -1.5544922351837158,
      "loss": 1.1471,
      "nll_loss": 1.06640625,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07579346001148224,
      "rewards/margins": 0.07951660454273224,
      "rewards/rejected": -0.15532226860523224,
      "step": 830
    },
    {
      "epoch": 0.1085061034683201,
      "grad_norm": 1.2631662398591137,
      "learning_rate": 2.7602622373694163e-06,
      "log_odds_chosen": 1.078125,
      "log_odds_ratio": -0.4767089784145355,
      "logits/chosen": -1.1144530773162842,
      "logits/rejected": -0.933789074420929,
      "logps/chosen": -0.7730468511581421,
      "logps/rejected": -1.5537109375,
      "loss": 1.1545,
      "nll_loss": 1.0974609851837158,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07736816257238388,
      "rewards/margins": 0.07791747897863388,
      "rewards/rejected": -0.15524902939796448,
      "step": 840
    },
    {
      "epoch": 0.10979784279532391,
      "grad_norm": 1.3780268169491492,
      "learning_rate": 2.743977362280141e-06,
      "log_odds_chosen": 0.964648425579071,
      "log_odds_ratio": -0.500048816204071,
      "logits/chosen": -1.037500023841858,
      "logits/rejected": -0.889843761920929,
      "logps/chosen": -0.817187488079071,
      "logps/rejected": -1.505468726158142,
      "loss": 1.1425,
      "nll_loss": 1.1220703125,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08172607421875,
      "rewards/margins": 0.06888427585363388,
      "rewards/rejected": -0.1505126953125,
      "step": 850
    },
    {
      "epoch": 0.11108958212232771,
      "grad_norm": 1.373161695970351,
      "learning_rate": 2.7279773578818937e-06,
      "log_odds_chosen": 1.1259765625,
      "log_odds_ratio": -0.45576173067092896,
      "logits/chosen": -1.0242187976837158,
      "logits/rejected": -0.855664074420929,
      "logps/chosen": -0.7642577886581421,
      "logps/rejected": -1.531640648841858,
      "loss": 1.1499,
      "nll_loss": 1.2355468273162842,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07637939602136612,
      "rewards/margins": 0.07664184272289276,
      "rewards/rejected": -0.15302734076976776,
      "step": 860
    },
    {
      "epoch": 0.11238132144933152,
      "grad_norm": 1.3194128142477222,
      "learning_rate": 2.7122540144832417e-06,
      "log_odds_chosen": 1.11572265625,
      "log_odds_ratio": -0.4810546934604645,
      "logits/chosen": -1.1394531726837158,
      "logits/rejected": -0.940234363079071,
      "logps/chosen": -0.781054675579071,
      "logps/rejected": -1.599218726158142,
      "loss": 1.1465,
      "nll_loss": 1.1085937023162842,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07819823920726776,
      "rewards/margins": 0.08179931342601776,
      "rewards/rejected": -0.15993651747703552,
      "step": 870
    },
    {
      "epoch": 0.11367306077633534,
      "grad_norm": 1.3699018634767623,
      "learning_rate": 2.696799449852968e-06,
      "log_odds_chosen": 0.997363269329071,
      "log_odds_ratio": -0.502978503704071,
      "logits/chosen": -1.181054711341858,
      "logits/rejected": -0.980664074420929,
      "logps/chosen": -0.7789062261581421,
      "logps/rejected": -1.512304663658142,
      "loss": 1.1613,
      "nll_loss": 1.1183593273162842,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07784423977136612,
      "rewards/margins": 0.07345886528491974,
      "rewards/rejected": -0.15134277939796448,
      "step": 880
    },
    {
      "epoch": 0.11496480010333915,
      "grad_norm": 1.327070907878745,
      "learning_rate": 2.6816060926159636e-06,
      "log_odds_chosen": 1.092675805091858,
      "log_odds_ratio": -0.4755859375,
      "logits/chosen": -1.0732421875,
      "logits/rejected": -0.883007824420929,
      "logps/chosen": -0.8623046875,
      "logps/rejected": -1.6749999523162842,
      "loss": 1.1307,
      "nll_loss": 1.1291015148162842,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.08623047173023224,
      "rewards/margins": 0.08126220852136612,
      "rewards/rejected": -0.16750487685203552,
      "step": 890
    },
    {
      "epoch": 0.11625653943034296,
      "grad_norm": 1.1915490143182448,
      "learning_rate": 2.6666666666666664e-06,
      "log_odds_chosen": 0.953857421875,
      "log_odds_ratio": -0.4917968809604645,
      "logits/chosen": -1.045507788658142,
      "logits/rejected": -0.915820300579071,
      "logps/chosen": -0.786328136920929,
      "logps/rejected": -1.41796875,
      "loss": 1.171,
      "nll_loss": 1.149023413658142,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07866211235523224,
      "rewards/margins": 0.06308440864086151,
      "rewards/rejected": -0.14189453423023224,
      "step": 900
    },
    {
      "epoch": 0.11754827875734676,
      "grad_norm": 1.4158282680234708,
      "learning_rate": 2.6519741765271837e-06,
      "log_odds_chosen": 1.0637328624725342,
      "log_odds_ratio": -0.478271484375,
      "logits/chosen": -1.1320312023162842,
      "logits/rejected": -0.941601574420929,
      "logps/chosen": -0.7564452886581421,
      "logps/rejected": -1.533593773841858,
      "loss": 1.1319,
      "nll_loss": 1.0759766101837158,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07567138969898224,
      "rewards/margins": 0.07762298732995987,
      "rewards/rejected": -0.15336914360523224,
      "step": 910
    },
    {
      "epoch": 0.11884001808435057,
      "grad_norm": 1.4979411861902598,
      "learning_rate": 2.637521893583148e-06,
      "log_odds_chosen": 1.232812523841858,
      "log_odds_ratio": -0.4005371034145355,
      "logits/chosen": -1.0720703601837158,
      "logits/rejected": -0.8804687261581421,
      "logps/chosen": -0.8119140863418579,
      "logps/rejected": -1.708593726158142,
      "loss": 1.1349,
      "nll_loss": 1.133203148841858,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.08122558891773224,
      "rewards/margins": 0.08980713039636612,
      "rewards/rejected": -0.1710205078125,
      "step": 920
    },
    {
      "epoch": 0.12013175741135439,
      "grad_norm": 1.4329687203340835,
      "learning_rate": 2.6233033431358115e-06,
      "log_odds_chosen": 1.117883324623108,
      "log_odds_ratio": -0.4404296875,
      "logits/chosen": -1.094335913658142,
      "logits/rejected": -0.8783203363418579,
      "logps/chosen": -0.737500011920929,
      "logps/rejected": -1.5349609851837158,
      "loss": 1.1592,
      "nll_loss": 1.089257836341858,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07379150390625,
      "rewards/margins": 0.07981643825769424,
      "rewards/rejected": -0.153564453125,
      "step": 930
    },
    {
      "epoch": 0.1214234967383582,
      "grad_norm": 1.3100609893803656,
      "learning_rate": 2.6093122922137685e-06,
      "log_odds_chosen": 1.1267821788787842,
      "log_odds_ratio": -0.500683605670929,
      "logits/chosen": -1.113671898841858,
      "logits/rejected": -0.943554699420929,
      "logps/chosen": -0.7984374761581421,
      "logps/rejected": -1.6189453601837158,
      "loss": 1.1455,
      "nll_loss": 1.0451171398162842,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07980956882238388,
      "rewards/margins": 0.08204193413257599,
      "rewards/rejected": -0.16184082627296448,
      "step": 940
    },
    {
      "epoch": 0.12271523606536201,
      "grad_norm": 1.2477819792797682,
      "learning_rate": 2.5955427380922006e-06,
      "log_odds_chosen": 1.266699194908142,
      "log_odds_ratio": -0.41845703125,
      "logits/chosen": -1.117578148841858,
      "logits/rejected": -0.8882812261581421,
      "logps/chosen": -0.7408202886581421,
      "logps/rejected": -1.663476586341858,
      "loss": 1.123,
      "nll_loss": 1.0281250476837158,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07415771484375,
      "rewards/margins": 0.092376708984375,
      "rewards/rejected": -0.16650390625,
      "step": 950
    },
    {
      "epoch": 0.12400697539236583,
      "grad_norm": 1.1896346898370782,
      "learning_rate": 2.5819888974716113e-06,
      "log_odds_chosen": 1.2431151866912842,
      "log_odds_ratio": -0.4400390684604645,
      "logits/chosen": -1.1259765625,
      "logits/rejected": -0.905468761920929,
      "logps/chosen": -0.7206054925918579,
      "logps/rejected": -1.581445336341858,
      "loss": 1.1184,
      "nll_loss": 1.0544922351837158,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07199706882238388,
      "rewards/margins": 0.08620300143957138,
      "rewards/rejected": -0.15817871689796448,
      "step": 960
    },
    {
      "epoch": 0.12529871471936962,
      "grad_norm": 1.372423060006083,
      "learning_rate": 2.5686451962717425e-06,
      "log_odds_chosen": 1.082910180091858,
      "log_odds_ratio": -0.503613293170929,
      "logits/chosen": -1.062890648841858,
      "logits/rejected": -0.8851562738418579,
      "logps/chosen": -0.8173828125,
      "logps/rejected": -1.6453125476837158,
      "loss": 1.1486,
      "nll_loss": 1.1298828125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08173827826976776,
      "rewards/margins": 0.08282165229320526,
      "rewards/rejected": -0.1646728515625,
      "step": 970
    },
    {
      "epoch": 0.12659045404637345,
      "grad_norm": 1.3070090355631485,
      "learning_rate": 2.5555062599997596e-06,
      "log_odds_chosen": 1.112280249595642,
      "log_odds_ratio": -0.4749511778354645,
      "logits/chosen": -1.079492211341858,
      "logits/rejected": -0.9228515625,
      "logps/chosen": -0.7783203125,
      "logps/rejected": -1.58203125,
      "loss": 1.1263,
      "nll_loss": 1.0712890625,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07783202826976776,
      "rewards/margins": 0.080352783203125,
      "rewards/rejected": -0.15822753310203552,
      "step": 980
    },
    {
      "epoch": 0.12788219337337725,
      "grad_norm": 1.3417780133407455,
      "learning_rate": 2.5425669046549126e-06,
      "log_odds_chosen": 1.10009765625,
      "log_odds_ratio": -0.4800781309604645,
      "logits/chosen": -1.109765648841858,
      "logits/rejected": -0.941601574420929,
      "logps/chosen": -0.780468761920929,
      "logps/rejected": -1.5595703125,
      "loss": 1.1309,
      "nll_loss": 1.0636718273162842,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07801513373851776,
      "rewards/margins": 0.077880859375,
      "rewards/rejected": -0.15578612685203552,
      "step": 990
    },
    {
      "epoch": 0.12917393270038105,
      "grad_norm": 1.3065765195099275,
      "learning_rate": 2.5298221281347034e-06,
      "log_odds_chosen": 1.2001953125,
      "log_odds_ratio": -0.46040040254592896,
      "logits/chosen": -1.0771484375,
      "logits/rejected": -0.8697265386581421,
      "logps/chosen": -0.7974609136581421,
      "logps/rejected": -1.659765601158142,
      "loss": 1.1659,
      "nll_loss": 1.142578125,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07972411811351776,
      "rewards/margins": 0.08632202446460724,
      "rewards/rejected": -0.16606445610523224,
      "step": 1000
    },
    {
      "epoch": 0.13046567202738488,
      "grad_norm": 1.3948602747639438,
      "learning_rate": 2.5172671021102103e-06,
      "log_odds_chosen": 0.920336902141571,
      "log_odds_ratio": -0.521435558795929,
      "logits/chosen": -1.177343726158142,
      "logits/rejected": -0.9976562261581421,
      "logps/chosen": -0.8033202886581421,
      "logps/rejected": -1.462890625,
      "loss": 1.1386,
      "nll_loss": 1.070898413658142,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.08033446967601776,
      "rewards/margins": 0.066192626953125,
      "rewards/rejected": -0.14645996689796448,
      "step": 1010
    },
    {
      "epoch": 0.13175741135438868,
      "grad_norm": 1.2271648513855602,
      "learning_rate": 2.504897164340598e-06,
      "log_odds_chosen": 0.995312511920929,
      "log_odds_ratio": -0.502197265625,
      "logits/chosen": -1.111718773841858,
      "logits/rejected": -0.9408203363418579,
      "logps/chosen": -0.8228515386581421,
      "logps/rejected": -1.5617187023162842,
      "loss": 1.1403,
      "nll_loss": 1.115234375,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08223877102136612,
      "rewards/margins": 0.07394103705883026,
      "rewards/rejected": -0.15615233778953552,
      "step": 1020
    },
    {
      "epoch": 0.1330491506813925,
      "grad_norm": 1.3919430776507808,
      "learning_rate": 2.492707811399023e-06,
      "log_odds_chosen": 1.2410156726837158,
      "log_odds_ratio": -0.4007812440395355,
      "logits/chosen": -1.1130859851837158,
      "logits/rejected": -0.8775390386581421,
      "logps/chosen": -0.759960949420929,
      "logps/rejected": -1.640039086341858,
      "loss": 1.131,
      "nll_loss": 1.0763671398162842,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0760498046875,
      "rewards/margins": 0.08796386420726776,
      "rewards/rejected": -0.16391602158546448,
      "step": 1030
    },
    {
      "epoch": 0.1343408900083963,
      "grad_norm": 1.3299014658160841,
      "learning_rate": 2.480694691784169e-06,
      "log_odds_chosen": 1.115991234779358,
      "log_odds_ratio": -0.476318359375,
      "logits/chosen": -1.0828125476837158,
      "logits/rejected": -0.8994140625,
      "logps/chosen": -0.802734375,
      "logps/rejected": -1.6337890625,
      "loss": 1.1392,
      "nll_loss": 1.199804663658142,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.08023681491613388,
      "rewards/margins": 0.08300018310546875,
      "rewards/rejected": -0.16328124701976776,
      "step": 1040
    },
    {
      "epoch": 0.13563262933540013,
      "grad_norm": 1.2762969025147817,
      "learning_rate": 2.4688535993934706e-06,
      "log_odds_chosen": 1.0980713367462158,
      "log_odds_ratio": -0.4600585997104645,
      "logits/chosen": -1.2019531726837158,
      "logits/rejected": -0.943554699420929,
      "logps/chosen": -0.759765625,
      "logps/rejected": -1.5626952648162842,
      "loss": 1.1387,
      "nll_loss": 1.0460937023162842,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07597656548023224,
      "rewards/margins": 0.08021392673254013,
      "rewards/rejected": -0.15622559189796448,
      "step": 1050
    },
    {
      "epoch": 0.13692436866240393,
      "grad_norm": 1.4729276168452141,
      "learning_rate": 2.457180467335805e-06,
      "log_odds_chosen": 1.1150391101837158,
      "log_odds_ratio": -0.4817871153354645,
      "logits/chosen": -1.0935547351837158,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.824414074420929,
      "logps/rejected": -1.670507788658142,
      "loss": 1.1409,
      "nll_loss": 1.1201171875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.08247070014476776,
      "rewards/margins": 0.08469848334789276,
      "rewards/rejected": -0.16704101860523224,
      "step": 1060
    },
    {
      "epoch": 0.13821610798940773,
      "grad_norm": 1.3201375381064102,
      "learning_rate": 2.4456713620629725e-06,
      "log_odds_chosen": 1.0822265148162842,
      "log_odds_ratio": -0.44038087129592896,
      "logits/chosen": -1.173828125,
      "logits/rejected": -0.930468738079071,
      "logps/chosen": -0.748046875,
      "logps/rejected": -1.5207030773162842,
      "loss": 1.1396,
      "nll_loss": 1.0343749523162842,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07478027045726776,
      "rewards/margins": 0.07741699367761612,
      "rewards/rejected": -0.1522216796875,
      "step": 1070
    },
    {
      "epoch": 0.13950784731641155,
      "grad_norm": 1.3273520687243323,
      "learning_rate": 2.4343224778007378e-06,
      "log_odds_chosen": 1.16357421875,
      "log_odds_ratio": -0.503173828125,
      "logits/chosen": -1.113867163658142,
      "logits/rejected": -0.9164062738418579,
      "logps/chosen": -0.807812511920929,
      "logps/rejected": -1.6755859851837158,
      "loss": 1.1062,
      "nll_loss": 1.0246093273162842,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.080810546875,
      "rewards/margins": 0.08669433742761612,
      "rewards/rejected": -0.16748046875,
      "step": 1080
    },
    {
      "epoch": 0.14079958664341535,
      "grad_norm": 1.3876866947546487,
      "learning_rate": 2.4231301312615306e-06,
      "log_odds_chosen": 0.9229491949081421,
      "log_odds_ratio": -0.517382800579071,
      "logits/chosen": -1.191992163658142,
      "logits/rejected": -0.9248046875,
      "logps/chosen": -0.774609386920929,
      "logps/rejected": -1.458984375,
      "loss": 1.1378,
      "nll_loss": 1.073632836341858,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07744140923023224,
      "rewards/margins": 0.06840820610523224,
      "rewards/rejected": -0.14589843153953552,
      "step": 1090
    },
    {
      "epoch": 0.14209132597041918,
      "grad_norm": 1.3715284318714909,
      "learning_rate": 2.412090756622109e-06,
      "log_odds_chosen": 1.021484375,
      "log_odds_ratio": -0.4747070372104645,
      "logits/chosen": -1.0998046398162842,
      "logits/rejected": -0.932421863079071,
      "logps/chosen": -0.7935546636581421,
      "logps/rejected": -1.525390625,
      "loss": 1.1362,
      "nll_loss": 1.1076171398162842,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.079345703125,
      "rewards/margins": 0.07316283881664276,
      "rewards/rejected": -0.15241698920726776,
      "step": 1100
    },
    {
      "epoch": 0.14338306529742298,
      "grad_norm": 1.5924740252054508,
      "learning_rate": 2.401200900750657e-06,
      "log_odds_chosen": 1.0475585460662842,
      "log_odds_ratio": -0.44902342557907104,
      "logits/chosen": -1.0802733898162842,
      "logits/rejected": -0.8617187738418579,
      "logps/chosen": -0.7339843511581421,
      "logps/rejected": -1.4353516101837158,
      "loss": 1.0923,
      "nll_loss": 1.035546898841858,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07338867336511612,
      "rewards/margins": 0.07010497897863388,
      "rewards/rejected": -0.14360351860523224,
      "step": 1110
    },
    {
      "epoch": 0.14467480462442678,
      "grad_norm": 1.2358991519735072,
      "learning_rate": 2.390457218668787e-06,
      "log_odds_chosen": 1.0911133289337158,
      "log_odds_ratio": -0.4476562440395355,
      "logits/chosen": -1.1443359851837158,
      "logits/rejected": -0.901171863079071,
      "logps/chosen": -0.771289050579071,
      "logps/rejected": -1.518945336341858,
      "loss": 1.0979,
      "nll_loss": 0.995312511920929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07708740234375,
      "rewards/margins": 0.07478637993335724,
      "rewards/rejected": -0.15187987685203552,
      "step": 1120
    },
    {
      "epoch": 0.1459665439514306,
      "grad_norm": 1.3488577378588829,
      "learning_rate": 2.379856469234918e-06,
      "log_odds_chosen": 1.2648437023162842,
      "log_odds_ratio": -0.44047850370407104,
      "logits/chosen": -1.0966796875,
      "logits/rejected": -0.9404296875,
      "logps/chosen": -0.806835949420929,
      "logps/rejected": -1.7421875,
      "loss": 1.1331,
      "nll_loss": 1.0724608898162842,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.08067627251148224,
      "rewards/margins": 0.09360351413488388,
      "rewards/rejected": -0.17421874403953552,
      "step": 1130
    },
    {
      "epoch": 0.1472582832784344,
      "grad_norm": 1.4388529655771254,
      "learning_rate": 2.369395511036369e-06,
      "log_odds_chosen": 1.0750000476837158,
      "log_odds_ratio": -0.48027342557907104,
      "logits/chosen": -1.126367211341858,
      "logits/rejected": -0.919921875,
      "logps/chosen": -0.8042968511581421,
      "logps/rejected": -1.564062476158142,
      "loss": 1.103,
      "nll_loss": 1.060937523841858,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08038330078125,
      "rewards/margins": 0.07592467963695526,
      "rewards/rejected": -0.15634766221046448,
      "step": 1140
    },
    {
      "epoch": 0.14855002260543823,
      "grad_norm": 1.3209082587268852,
      "learning_rate": 2.359071298478354e-06,
      "log_odds_chosen": 0.9400879144668579,
      "log_odds_ratio": -0.544628918170929,
      "logits/chosen": -1.113671898841858,
      "logits/rejected": -0.9183593988418579,
      "logps/chosen": -0.8265625238418579,
      "logps/rejected": -1.531640648841858,
      "loss": 1.1212,
      "nll_loss": 1.087499976158142,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.08259277045726776,
      "rewards/margins": 0.07053222507238388,
      "rewards/rejected": -0.15300293266773224,
      "step": 1150
    },
    {
      "epoch": 0.14984176193244203,
      "grad_norm": 1.2521610091460516,
      "learning_rate": 2.3488808780588137e-06,
      "log_odds_chosen": 1.1383788585662842,
      "log_odds_ratio": -0.4613281190395355,
      "logits/chosen": -1.117578148841858,
      "logits/rejected": -0.9017578363418579,
      "logps/chosen": -0.837109386920929,
      "logps/rejected": -1.6550781726837158,
      "loss": 1.116,
      "nll_loss": 1.1082031726837158,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.083740234375,
      "rewards/margins": 0.08187256008386612,
      "rewards/rejected": -0.16550293564796448,
      "step": 1160
    },
    {
      "epoch": 0.15113350125944586,
      "grad_norm": 1.2956595234243158,
      "learning_rate": 2.3388213848187446e-06,
      "log_odds_chosen": 1.450927734375,
      "log_odds_ratio": -0.37812501192092896,
      "logits/chosen": -1.171289086341858,
      "logits/rejected": -0.9261718988418579,
      "logps/chosen": -0.751953125,
      "logps/rejected": -1.7976562976837158,
      "loss": 1.1281,
      "nll_loss": 1.0732421875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07520751655101776,
      "rewards/margins": 0.10442505031824112,
      "rewards/rejected": -0.17966309189796448,
      "step": 1170
    },
    {
      "epoch": 0.15242524058644966,
      "grad_norm": 1.3800768816100841,
      "learning_rate": 2.328890038958328e-06,
      "log_odds_chosen": 1.398828148841858,
      "log_odds_ratio": -0.3961425721645355,
      "logits/chosen": -1.0763671398162842,
      "logits/rejected": -0.8607422113418579,
      "logps/chosen": -0.734082043170929,
      "logps/rejected": -1.7453124523162842,
      "loss": 1.1235,
      "nll_loss": 0.999218761920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07333984225988388,
      "rewards/margins": 0.10135497897863388,
      "rewards/rejected": -0.17473144829273224,
      "step": 1180
    },
    {
      "epoch": 0.15371697991345346,
      "grad_norm": 1.4944316671133717,
      "learning_rate": 2.3190841426097937e-06,
      "log_odds_chosen": 1.2546875476837158,
      "log_odds_ratio": -0.45146483182907104,
      "logits/chosen": -1.1154296398162842,
      "logits/rejected": -0.932421863079071,
      "logps/chosen": -0.7935546636581421,
      "logps/rejected": -1.7121093273162842,
      "loss": 1.1188,
      "nll_loss": 1.0310547351837158,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07933349907398224,
      "rewards/margins": 0.09170226752758026,
      "rewards/rejected": -0.17104491591453552,
      "step": 1190
    },
    {
      "epoch": 0.15500871924045728,
      "grad_norm": 1.2440668732866864,
      "learning_rate": 2.309401076758503e-06,
      "log_odds_chosen": 1.306249976158142,
      "log_odds_ratio": -0.4134277403354645,
      "logits/chosen": -1.1349608898162842,
      "logits/rejected": -0.9248046875,
      "logps/chosen": -0.761523425579071,
      "logps/rejected": -1.6945312023162842,
      "loss": 1.1292,
      "nll_loss": 1.0949218273162842,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07615967094898224,
      "rewards/margins": 0.09329833835363388,
      "rewards/rejected": -0.16953125596046448,
      "step": 1200
    },
    {
      "epoch": 0.15630045856746108,
      "grad_norm": 1.28882495876724,
      "learning_rate": 2.299838298304276e-06,
      "log_odds_chosen": 1.199804663658142,
      "log_odds_ratio": -0.42622071504592896,
      "logits/chosen": -1.138085961341858,
      "logits/rejected": -0.9658203125,
      "logps/chosen": -0.7623046636581421,
      "logps/rejected": -1.6062500476837158,
      "loss": 1.1146,
      "nll_loss": 1.103515625,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07625732570886612,
      "rewards/margins": 0.08441162109375,
      "rewards/rejected": -0.16069336235523224,
      "step": 1210
    },
    {
      "epoch": 0.1575921978944649,
      "grad_norm": 1.4835488646397357,
      "learning_rate": 2.2903933372554728e-06,
      "log_odds_chosen": 1.4255859851837158,
      "log_odds_ratio": -0.39091795682907104,
      "logits/chosen": -1.2109375,
      "logits/rejected": -0.9652343988418579,
      "logps/chosen": -0.761914074420929,
      "logps/rejected": -1.7804687023162842,
      "loss": 1.1125,
      "nll_loss": 0.994140625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07622070610523224,
      "rewards/margins": 0.10186157375574112,
      "rewards/rejected": -0.17824706435203552,
      "step": 1220
    },
    {
      "epoch": 0.1588839372214687,
      "grad_norm": 1.3699495727568183,
      "learning_rate": 2.281063794048804e-06,
      "log_odds_chosen": 1.237402319908142,
      "log_odds_ratio": -0.44575196504592896,
      "logits/chosen": -1.1083984375,
      "logits/rejected": -0.976757824420929,
      "logps/chosen": -0.7533203363418579,
      "logps/rejected": -1.6589844226837158,
      "loss": 1.106,
      "nll_loss": 1.059960961341858,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07528076320886612,
      "rewards/margins": 0.09058532863855362,
      "rewards/rejected": -0.16591796278953552,
      "step": 1230
    },
    {
      "epoch": 0.1601756765484725,
      "grad_norm": 1.44050030611481,
      "learning_rate": 2.271847336988259e-06,
      "log_odds_chosen": 1.083398461341858,
      "log_odds_ratio": -0.481689453125,
      "logits/chosen": -1.1033203601837158,
      "logits/rejected": -0.935351550579071,
      "logps/chosen": -0.809374988079071,
      "logps/rejected": -1.6017577648162842,
      "loss": 1.1177,
      "nll_loss": 1.057031273841858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.08094482123851776,
      "rewards/margins": 0.07926635444164276,
      "rewards/rejected": -0.16030272841453552,
      "step": 1240
    },
    {
      "epoch": 0.16146741587547633,
      "grad_norm": 1.3086122074478357,
      "learning_rate": 2.262741699796952e-06,
      "log_odds_chosen": 1.234277367591858,
      "log_odds_ratio": -0.41997069120407104,
      "logits/chosen": -1.2326171398162842,
      "logits/rejected": -0.958789050579071,
      "logps/chosen": -0.767773449420929,
      "logps/rejected": -1.665624976158142,
      "loss": 1.0948,
      "nll_loss": 1.0517578125,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07674560695886612,
      "rewards/margins": 0.08981017768383026,
      "rewards/rejected": -0.1666259765625,
      "step": 1250
    },
    {
      "epoch": 0.16275915520248013,
      "grad_norm": 1.2936055039074432,
      "learning_rate": 2.253744679276044e-06,
      "log_odds_chosen": 1.24365234375,
      "log_odds_ratio": -0.4651855528354645,
      "logits/chosen": -1.0837891101837158,
      "logits/rejected": -0.9564453363418579,
      "logps/chosen": -0.7847656011581421,
      "logps/rejected": -1.728515625,
      "loss": 1.1105,
      "nll_loss": 1.072656273841858,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07845459133386612,
      "rewards/margins": 0.09423675388097763,
      "rewards/rejected": -0.17270508408546448,
      "step": 1260
    },
    {
      "epoch": 0.16405089452948396,
      "grad_norm": 1.4266025957013813,
      "learning_rate": 2.244854133065255e-06,
      "log_odds_chosen": 1.263574242591858,
      "log_odds_ratio": -0.4093261659145355,
      "logits/chosen": -1.052343726158142,
      "logits/rejected": -0.895703136920929,
      "logps/chosen": -0.7308593988418579,
      "logps/rejected": -1.6142578125,
      "loss": 1.1341,
      "nll_loss": 1.201562523841858,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07308349758386612,
      "rewards/margins": 0.08831176906824112,
      "rewards/rejected": -0.1614990234375,
      "step": 1270
    },
    {
      "epoch": 0.16534263385648776,
      "grad_norm": 1.498154976991836,
      "learning_rate": 2.2360679774997895e-06,
      "log_odds_chosen": 1.291406273841858,
      "log_odds_ratio": -0.4771972596645355,
      "logits/chosen": -1.0886719226837158,
      "logits/rejected": -0.9515625238418579,
      "logps/chosen": -0.779296875,
      "logps/rejected": -1.754492163658142,
      "loss": 1.1113,
      "nll_loss": 1.068750023841858,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07791747897863388,
      "rewards/margins": 0.09754638373851776,
      "rewards/rejected": -0.17536620795726776,
      "step": 1280
    },
    {
      "epoch": 0.16663437318349156,
      "grad_norm": 1.4686616555046261,
      "learning_rate": 2.2273841855588183e-06,
      "log_odds_chosen": 1.124914526939392,
      "log_odds_ratio": -0.47343748807907104,
      "logits/chosen": -1.1814453601837158,
      "logits/rejected": -0.978320300579071,
      "logps/chosen": -0.8343750238418579,
      "logps/rejected": -1.6603515148162842,
      "loss": 1.1424,
      "nll_loss": 1.1671874523162842,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.08339843899011612,
      "rewards/margins": 0.08247070014476776,
      "rewards/rejected": -0.166015625,
      "step": 1290
    },
    {
      "epoch": 0.16792611251049538,
      "grad_norm": 1.3237206808679978,
      "learning_rate": 2.2188007849009167e-06,
      "log_odds_chosen": 1.2733154296875,
      "log_odds_ratio": -0.423828125,
      "logits/chosen": -1.2234375476837158,
      "logits/rejected": -0.94921875,
      "logps/chosen": -0.763671875,
      "logps/rejected": -1.6951172351837158,
      "loss": 1.1232,
      "nll_loss": 1.0324218273162842,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07636718451976776,
      "rewards/margins": 0.09322662651538849,
      "rewards/rejected": -0.16965332627296448,
      "step": 1300
    },
    {
      "epoch": 0.16921785183749918,
      "grad_norm": 1.4317095067575667,
      "learning_rate": 2.2103158559821502e-06,
      "log_odds_chosen": 1.0578124523162842,
      "log_odds_ratio": -0.47895509004592896,
      "logits/chosen": -1.160546898841858,
      "logits/rejected": -0.935351550579071,
      "logps/chosen": -0.822265625,
      "logps/rejected": -1.6296875476837158,
      "loss": 1.1068,
      "nll_loss": 1.1130859851837158,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.08221435546875,
      "rewards/margins": 0.0806732177734375,
      "rewards/rejected": -0.16289062798023224,
      "step": 1310
    },
    {
      "epoch": 0.170509591164503,
      "grad_norm": 1.3936612927296241,
      "learning_rate": 2.2019275302527213e-06,
      "log_odds_chosen": 1.143225073814392,
      "log_odds_ratio": -0.49067384004592896,
      "logits/chosen": -1.094335913658142,
      "logits/rejected": -0.954296886920929,
      "logps/chosen": -0.7623046636581421,
      "logps/rejected": -1.6062500476837158,
      "loss": 1.1001,
      "nll_loss": 1.01953125,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.076171875,
      "rewards/margins": 0.08434142917394638,
      "rewards/rejected": -0.16064453125,
      "step": 1320
    },
    {
      "epoch": 0.1718013304915068,
      "grad_norm": 1.3657127435779988,
      "learning_rate": 2.193633988428327e-06,
      "log_odds_chosen": 1.1318359375,
      "log_odds_ratio": -0.4637695252895355,
      "logits/chosen": -1.1541016101837158,
      "logits/rejected": -0.9644531011581421,
      "logps/chosen": -0.7568359375,
      "logps/rejected": -1.546484351158142,
      "loss": 1.1065,
      "nll_loss": 1.0623047351837158,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07569579780101776,
      "rewards/margins": 0.07891845703125,
      "rewards/rejected": -0.15461425483226776,
      "step": 1330
    },
    {
      "epoch": 0.17309306981851064,
      "grad_norm": 1.4254012715171065,
      "learning_rate": 2.185433458832612e-06,
      "log_odds_chosen": 1.1473267078399658,
      "log_odds_ratio": -0.4442382752895355,
      "logits/chosen": -1.151953101158142,
      "logits/rejected": -0.9765625,
      "logps/chosen": -0.798046886920929,
      "logps/rejected": -1.634765625,
      "loss": 1.1163,
      "nll_loss": 1.1144530773162842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07978515326976776,
      "rewards/margins": 0.08360596001148224,
      "rewards/rejected": -0.16335448622703552,
      "step": 1340
    },
    {
      "epoch": 0.17438480914551444,
      "grad_norm": 1.3545870936994966,
      "learning_rate": 2.177324215807269e-06,
      "log_odds_chosen": 1.1541016101837158,
      "log_odds_ratio": -0.45927733182907104,
      "logits/chosen": -1.1435546875,
      "logits/rejected": -0.9658203125,
      "logps/chosen": -0.817578136920929,
      "logps/rejected": -1.654296875,
      "loss": 1.1295,
      "nll_loss": 1.066015601158142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.08183594048023224,
      "rewards/margins": 0.08367309719324112,
      "rewards/rejected": -0.1654052734375,
      "step": 1350
    },
    {
      "epoch": 0.17567654847251823,
      "grad_norm": 1.2738765317640237,
      "learning_rate": 2.1693045781865616e-06,
      "log_odds_chosen": 1.1106445789337158,
      "log_odds_ratio": -0.465576171875,
      "logits/chosen": -1.149999976158142,
      "logits/rejected": -0.9623047113418579,
      "logps/chosen": -0.7508789300918579,
      "logps/rejected": -1.5535156726837158,
      "loss": 1.0934,
      "nll_loss": 1.075585961341858,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07503662258386612,
      "rewards/margins": 0.08033446967601776,
      "rewards/rejected": -0.15537109971046448,
      "step": 1360
    },
    {
      "epoch": 0.17696828779952206,
      "grad_norm": 1.9798266333560217,
      "learning_rate": 2.1613729078331965e-06,
      "log_odds_chosen": 1.122460961341858,
      "log_odds_ratio": -0.4314941465854645,
      "logits/chosen": -1.213476538658142,
      "logits/rejected": -0.9369140863418579,
      "logps/chosen": -0.74755859375,
      "logps/rejected": -1.5460937023162842,
      "loss": 1.1118,
      "nll_loss": 1.090429663658142,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07467041164636612,
      "rewards/margins": 0.07977294921875,
      "rewards/rejected": -0.15446777641773224,
      "step": 1370
    },
    {
      "epoch": 0.17826002712652586,
      "grad_norm": 1.4758894451151416,
      "learning_rate": 2.1535276082326617e-06,
      "log_odds_chosen": 1.2423827648162842,
      "log_odds_ratio": -0.4352050721645355,
      "logits/chosen": -1.091796875,
      "logits/rejected": -0.9244140386581421,
      "logps/chosen": -0.7679687738418579,
      "logps/rejected": -1.6765625476837158,
      "loss": 1.0908,
      "nll_loss": 0.9888671636581421,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07677002251148224,
      "rewards/margins": 0.09089355170726776,
      "rewards/rejected": -0.16774901747703552,
      "step": 1380
    },
    {
      "epoch": 0.1795517664535297,
      "grad_norm": 1.4533326056703724,
      "learning_rate": 2.14576712314328e-06,
      "log_odds_chosen": 1.2523925304412842,
      "log_odds_ratio": -0.43720704317092896,
      "logits/chosen": -1.143164038658142,
      "logits/rejected": -0.9419921636581421,
      "logps/chosen": -0.720898449420929,
      "logps/rejected": -1.6140625476837158,
      "loss": 1.0937,
      "nll_loss": 1.039453148841858,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07209472358226776,
      "rewards/margins": 0.089263916015625,
      "rewards/rejected": -0.16130371391773224,
      "step": 1390
    },
    {
      "epoch": 0.1808435057805335,
      "grad_norm": 1.3122823944760766,
      "learning_rate": 2.138089935299395e-06,
      "log_odds_chosen": 0.9754883050918579,
      "log_odds_ratio": -0.510058581829071,
      "logits/chosen": -1.158789038658142,
      "logits/rejected": -0.990429699420929,
      "logps/chosen": -0.7818359136581421,
      "logps/rejected": -1.484375,
      "loss": 1.0846,
      "nll_loss": 1.0330078601837158,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07827148586511612,
      "rewards/margins": 0.07023315131664276,
      "rewards/rejected": -0.14836426079273224,
      "step": 1400
    },
    {
      "epoch": 0.18213524510753729,
      "grad_norm": 1.3244432635399115,
      "learning_rate": 2.1304945651652297e-06,
      "log_odds_chosen": 1.2722656726837158,
      "log_odds_ratio": -0.4310058653354645,
      "logits/chosen": -1.149804711341858,
      "logits/rejected": -0.9359375238418579,
      "logps/chosen": -0.8238281011581421,
      "logps/rejected": -1.7550780773162842,
      "loss": 1.1142,
      "nll_loss": 1.1554687023162842,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.08232422173023224,
      "rewards/margins": 0.09328613430261612,
      "rewards/rejected": -0.17570801079273224,
      "step": 1410
    },
    {
      "epoch": 0.1834269844345411,
      "grad_norm": 1.2850671452196158,
      "learning_rate": 2.122979569737101e-06,
      "log_odds_chosen": 1.105371117591858,
      "log_odds_ratio": -0.43339842557907104,
      "logits/chosen": -1.1662108898162842,
      "logits/rejected": -0.9847656488418579,
      "logps/chosen": -0.7476562261581421,
      "logps/rejected": -1.529296875,
      "loss": 1.0991,
      "nll_loss": 1.0300781726837158,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07478027045726776,
      "rewards/margins": 0.07816161960363388,
      "rewards/rejected": -0.1529541015625,
      "step": 1420
    },
    {
      "epoch": 0.1847187237615449,
      "grad_norm": 2.4308567192893955,
      "learning_rate": 2.11554354139178e-06,
      "log_odds_chosen": 1.1113770008087158,
      "log_odds_ratio": -0.44941407442092896,
      "logits/chosen": -1.1435546875,
      "logits/rejected": -0.9453125,
      "logps/chosen": -0.8001953363418579,
      "logps/rejected": -1.596289038658142,
      "loss": 1.1047,
      "nll_loss": 1.0546875,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.08002929389476776,
      "rewards/margins": 0.07972107082605362,
      "rewards/rejected": -0.15974120795726776,
      "step": 1430
    },
    {
      "epoch": 0.18601046308854874,
      "grad_norm": 1.2772233655673262,
      "learning_rate": 2.1081851067789196e-06,
      "log_odds_chosen": 1.0105469226837158,
      "log_odds_ratio": -0.5079101324081421,
      "logits/chosen": -1.118554711341858,
      "logits/rejected": -0.8994140625,
      "logps/chosen": -0.81640625,
      "logps/rejected": -1.540624976158142,
      "loss": 1.1064,
      "nll_loss": 1.10546875,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0816650390625,
      "rewards/margins": 0.07232666015625,
      "rewards/rejected": -0.1539306640625,
      "step": 1440
    },
    {
      "epoch": 0.18730220241555254,
      "grad_norm": 1.5698697170646343,
      "learning_rate": 2.1009029257555606e-06,
      "log_odds_chosen": 1.3738281726837158,
      "log_odds_ratio": -0.40522462129592896,
      "logits/chosen": -1.2189452648162842,
      "logits/rejected": -0.9751952886581421,
      "logps/chosen": -0.782421886920929,
      "logps/rejected": -1.7888672351837158,
      "loss": 1.077,
      "nll_loss": 0.977343738079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07829590141773224,
      "rewards/margins": 0.10050048679113388,
      "rewards/rejected": -0.17868652939796448,
      "step": 1450
    },
    {
      "epoch": 0.18859394174255636,
      "grad_norm": 1.5025387469398077,
      "learning_rate": 2.0936956903608545e-06,
      "log_odds_chosen": 1.194799780845642,
      "log_odds_ratio": -0.4173339903354645,
      "logits/chosen": -1.068945288658142,
      "logits/rejected": -0.902539074420929,
      "logps/chosen": -0.772265613079071,
      "logps/rejected": -1.600000023841858,
      "loss": 1.1315,
      "nll_loss": 1.064062476158142,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0772705078125,
      "rewards/margins": 0.08281020820140839,
      "rewards/rejected": -0.16000977158546448,
      "step": 1460
    },
    {
      "epoch": 0.18988568106956016,
      "grad_norm": 1.226791941569367,
      "learning_rate": 2.0865621238292046e-06,
      "log_odds_chosen": 0.957653820514679,
      "log_odds_ratio": -0.49580079317092896,
      "logits/chosen": -1.1462891101837158,
      "logits/rejected": -0.952929675579071,
      "logps/chosen": -0.765429675579071,
      "logps/rejected": -1.431249976158142,
      "loss": 1.0962,
      "nll_loss": 1.024999976158142,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0765380859375,
      "rewards/margins": 0.066650390625,
      "rewards/rejected": -0.14291992783546448,
      "step": 1470
    },
    {
      "epoch": 0.19117742039656396,
      "grad_norm": 1.255816048846265,
      "learning_rate": 2.079500979640145e-06,
      "log_odds_chosen": 1.3544921875,
      "log_odds_ratio": -0.39433592557907104,
      "logits/chosen": -1.2179687023162842,
      "logits/rejected": -0.981640636920929,
      "logps/chosen": -0.744921863079071,
      "logps/rejected": -1.7136719226837158,
      "loss": 1.1083,
      "nll_loss": 1.0154297351837158,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07443847507238388,
      "rewards/margins": 0.09688110649585724,
      "rewards/rejected": -0.17136231064796448,
      "step": 1480
    },
    {
      "epoch": 0.1924691597235678,
      "grad_norm": 1.5093391663942994,
      "learning_rate": 2.072511040603359e-06,
      "log_odds_chosen": 1.44677734375,
      "log_odds_ratio": -0.4171386659145355,
      "logits/chosen": -1.064843773841858,
      "logits/rejected": -0.936328113079071,
      "logps/chosen": -0.732226550579071,
      "logps/rejected": -1.800390601158142,
      "loss": 1.093,
      "nll_loss": 1.03515625,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07322998344898224,
      "rewards/margins": 0.10672607272863388,
      "rewards/rejected": -0.179931640625,
      "step": 1490
    },
    {
      "epoch": 0.1937608990505716,
      "grad_norm": 1.4439729612192194,
      "learning_rate": 2.065591117977289e-06,
      "log_odds_chosen": 1.3293945789337158,
      "log_odds_ratio": -0.4292968809604645,
      "logits/chosen": -0.9779297113418579,
      "logits/rejected": -0.841015636920929,
      "logps/chosen": -0.741406261920929,
      "logps/rejected": -1.738867163658142,
      "loss": 1.0959,
      "nll_loss": 1.0681641101837158,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07419433444738388,
      "rewards/margins": 0.09969482570886612,
      "rewards/rejected": -0.173828125,
      "step": 1500
    },
    {
      "epoch": 0.19505263837757542,
      "grad_norm": 1.4160458169410626,
      "learning_rate": 2.058740050619915e-06,
      "log_odds_chosen": 1.3718750476837158,
      "log_odds_ratio": -0.4171386659145355,
      "logits/chosen": -1.183007836341858,
      "logits/rejected": -0.9681640863418579,
      "logps/chosen": -0.741406261920929,
      "logps/rejected": -1.753515601158142,
      "loss": 1.074,
      "nll_loss": 1.0636718273162842,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07413329929113388,
      "rewards/margins": 0.10120849311351776,
      "rewards/rejected": -0.17539063096046448,
      "step": 1510
    },
    {
      "epoch": 0.19634437770457921,
      "grad_norm": 1.547211341318252,
      "learning_rate": 2.0519567041703083e-06,
      "log_odds_chosen": 1.310937523841858,
      "log_odds_ratio": -0.4332275390625,
      "logits/chosen": -1.159570336341858,
      "logits/rejected": -0.9609375,
      "logps/chosen": -0.7626953125,
      "logps/rejected": -1.716406226158142,
      "loss": 1.0645,
      "nll_loss": 0.970898449420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07625732570886612,
      "rewards/margins": 0.095306396484375,
      "rewards/rejected": -0.171630859375,
      "step": 1520
    },
    {
      "epoch": 0.19763611703158301,
      "grad_norm": 1.6239989582832697,
      "learning_rate": 2.0452399702596544e-06,
      "log_odds_chosen": 1.317480444908142,
      "log_odds_ratio": -0.4146484434604645,
      "logits/chosen": -1.2160155773162842,
      "logits/rejected": -0.962109386920929,
      "logps/chosen": -0.799023449420929,
      "logps/rejected": -1.7570312023162842,
      "loss": 1.1072,
      "nll_loss": 1.0144531726837158,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07993163913488388,
      "rewards/margins": 0.09577026218175888,
      "rewards/rejected": -0.17578125,
      "step": 1530
    },
    {
      "epoch": 0.19892785635858684,
      "grad_norm": 1.260181584122507,
      "learning_rate": 2.0385887657505017e-06,
      "log_odds_chosen": 1.1710937023162842,
      "log_odds_ratio": -0.4537109434604645,
      "logits/chosen": -1.1357421875,
      "logits/rejected": -0.9947265386581421,
      "logps/chosen": -0.778124988079071,
      "logps/rejected": -1.6183593273162842,
      "loss": 1.0854,
      "nll_loss": 1.0359375476837158,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07781982421875,
      "rewards/margins": 0.08405761420726776,
      "rewards/rejected": -0.16191406548023224,
      "step": 1540
    },
    {
      "epoch": 0.20021959568559064,
      "grad_norm": 1.38588668926837,
      "learning_rate": 2.032002032003048e-06,
      "log_odds_chosen": 0.803417980670929,
      "log_odds_ratio": -0.5362793207168579,
      "logits/chosen": -1.227148413658142,
      "logits/rejected": -1.0226562023162842,
      "logps/chosen": -0.8232421875,
      "logps/rejected": -1.3849608898162842,
      "loss": 1.0894,
      "nll_loss": 1.074804663658142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.08229980617761612,
      "rewards/margins": 0.05617370456457138,
      "rewards/rejected": -0.13845214247703552,
      "step": 1550
    },
    {
      "epoch": 0.20151133501259447,
      "grad_norm": 1.17909080785775,
      "learning_rate": 2.025478734167333e-06,
      "log_odds_chosen": 1.1628906726837158,
      "log_odds_ratio": -0.4571289122104645,
      "logits/chosen": -1.067968726158142,
      "logits/rejected": -0.9234374761581421,
      "logps/chosen": -0.7769531011581421,
      "logps/rejected": -1.620703101158142,
      "loss": 1.1131,
      "nll_loss": 1.09765625,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07758788764476776,
      "rewards/margins": 0.08439941704273224,
      "rewards/rejected": -0.16208496689796448,
      "step": 1560
    },
    {
      "epoch": 0.20280307433959827,
      "grad_norm": 1.365830710002793,
      "learning_rate": 2.0190178605002747e-06,
      "log_odds_chosen": 1.1975586414337158,
      "log_odds_ratio": -0.44921875,
      "logits/chosen": -1.120703101158142,
      "logits/rejected": -0.9580078125,
      "logps/chosen": -0.7774413824081421,
      "logps/rejected": -1.6437499523162842,
      "loss": 1.0909,
      "nll_loss": 1.01953125,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0777587890625,
      "rewards/margins": 0.08661804348230362,
      "rewards/rejected": -0.16437987983226776,
      "step": 1570
    },
    {
      "epoch": 0.2040948136666021,
      "grad_norm": 1.2284103885166069,
      "learning_rate": 2.0126184217065104e-06,
      "log_odds_chosen": 1.25341796875,
      "log_odds_ratio": -0.47001951932907104,
      "logits/chosen": -1.0720703601837158,
      "logits/rejected": -0.8544921875,
      "logps/chosen": -0.793261706829071,
      "logps/rejected": -1.6689453125,
      "loss": 1.0909,
      "nll_loss": 1.048437476158142,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07937011867761612,
      "rewards/margins": 0.08757934719324112,
      "rewards/rejected": -0.16701659560203552,
      "step": 1580
    },
    {
      "epoch": 0.2053865529936059,
      "grad_norm": 1.3949109783939957,
      "learning_rate": 2.0062794503020765e-06,
      "log_odds_chosen": 1.168554663658142,
      "log_odds_ratio": -0.43212890625,
      "logits/chosen": -1.1531250476837158,
      "logits/rejected": -0.956250011920929,
      "logps/chosen": -0.792773425579071,
      "logps/rejected": -1.638671875,
      "loss": 1.1089,
      "nll_loss": 1.0662109851837158,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07928466796875,
      "rewards/margins": 0.08453979343175888,
      "rewards/rejected": -0.16379395127296448,
      "step": 1590
    },
    {
      "epoch": 0.2066782923206097,
      "grad_norm": 1.4752303000556217,
      "learning_rate": 2e-06,
      "log_odds_chosen": 1.273290991783142,
      "log_odds_ratio": -0.48344725370407104,
      "logits/chosen": -1.1076171398162842,
      "logits/rejected": -0.9541015625,
      "logps/chosen": -0.7925781011581421,
      "logps/rejected": -1.7683594226837158,
      "loss": 1.1097,
      "nll_loss": 1.155859351158142,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07927246391773224,
      "rewards/margins": 0.09764709323644638,
      "rewards/rejected": -0.17685547471046448,
      "step": 1600
    },
    {
      "epoch": 0.20797003164761352,
      "grad_norm": 1.2875741946501222,
      "learning_rate": 1.993779145116907e-06,
      "log_odds_chosen": 1.409765601158142,
      "log_odds_ratio": -0.41362303495407104,
      "logits/chosen": -1.0955078601837158,
      "logits/rejected": -0.8812500238418579,
      "logps/chosen": -0.748046875,
      "logps/rejected": -1.787500023841858,
      "loss": 1.095,
      "nll_loss": 1.0285155773162842,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07480468600988388,
      "rewards/margins": 0.10386963188648224,
      "rewards/rejected": -0.17880859971046448,
      "step": 1610
    },
    {
      "epoch": 0.20926177097461732,
      "grad_norm": 1.8412910054371145,
      "learning_rate": 1.987615979999813e-06,
      "log_odds_chosen": 1.2473633289337158,
      "log_odds_ratio": -0.48393553495407104,
      "logits/chosen": -1.2087891101837158,
      "logits/rejected": -0.9671875238418579,
      "logps/chosen": -0.8089843988418579,
      "logps/rejected": -1.7478516101837158,
      "loss": 1.0864,
      "nll_loss": 0.991992175579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.08092041313648224,
      "rewards/margins": 0.09385375678539276,
      "rewards/rejected": -0.17473144829273224,
      "step": 1620
    },
    {
      "epoch": 0.21055351030162114,
      "grad_norm": 1.3907484022272254,
      "learning_rate": 1.9815096184722797e-06,
      "log_odds_chosen": 1.1614258289337158,
      "log_odds_ratio": -0.4903808534145355,
      "logits/chosen": -1.165429711341858,
      "logits/rejected": -1.0031249523162842,
      "logps/chosen": -0.828125,
      "logps/rejected": -1.664453148841858,
      "loss": 1.0795,
      "nll_loss": 0.991406261920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08277587592601776,
      "rewards/margins": 0.08376464992761612,
      "rewards/rejected": -0.16650390625,
      "step": 1630
    },
    {
      "epoch": 0.21184524962862494,
      "grad_norm": 1.558134420817283,
      "learning_rate": 1.9754591932991793e-06,
      "log_odds_chosen": 1.151879906654358,
      "log_odds_ratio": -0.44794923067092896,
      "logits/chosen": -1.160546898841858,
      "logits/rejected": -0.9457031488418579,
      "logps/chosen": -0.7603515386581421,
      "logps/rejected": -1.6042969226837158,
      "loss": 1.0688,
      "nll_loss": 1.0226562023162842,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07601318508386612,
      "rewards/margins": 0.08431701362133026,
      "rewards/rejected": -0.16044922173023224,
      "step": 1640
    },
    {
      "epoch": 0.21313698895562874,
      "grad_norm": 1.2617805869856145,
      "learning_rate": 1.9694638556693235e-06,
      "log_odds_chosen": 1.240991234779358,
      "log_odds_ratio": -0.4515136778354645,
      "logits/chosen": -1.166015625,
      "logits/rejected": -0.986523449420929,
      "logps/chosen": -0.828320324420929,
      "logps/rejected": -1.7175781726837158,
      "loss": 1.0827,
      "nll_loss": 1.0929687023162842,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.08289794623851776,
      "rewards/margins": 0.08886261284351349,
      "rewards/rejected": -0.17167969048023224,
      "step": 1650
    },
    {
      "epoch": 0.21442872828263257,
      "grad_norm": 1.3083194807094314,
      "learning_rate": 1.963522774695264e-06,
      "log_odds_chosen": 1.0681641101837158,
      "log_odds_ratio": -0.4775390625,
      "logits/chosen": -1.137109398841858,
      "logits/rejected": -0.9468749761581421,
      "logps/chosen": -0.827343761920929,
      "logps/rejected": -1.6203124523162842,
      "loss": 1.0871,
      "nll_loss": 1.044921875,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.08265380561351776,
      "rewards/margins": 0.07925720512866974,
      "rewards/rejected": -0.16188964247703552,
      "step": 1660
    },
    {
      "epoch": 0.21572046760963637,
      "grad_norm": 1.2852632749576811,
      "learning_rate": 1.9576351369295853e-06,
      "log_odds_chosen": 1.300537109375,
      "log_odds_ratio": -0.4420410096645355,
      "logits/chosen": -1.2527344226837158,
      "logits/rejected": -0.995312511920929,
      "logps/chosen": -0.8115234375,
      "logps/rejected": -1.763281226158142,
      "loss": 1.0617,
      "nll_loss": 1.0554687976837158,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.08120117336511612,
      "rewards/margins": 0.09511718899011612,
      "rewards/rejected": -0.17641600966453552,
      "step": 1670
    },
    {
      "epoch": 0.2170122069366402,
      "grad_norm": 1.332740172809499,
      "learning_rate": 1.951800145897066e-06,
      "log_odds_chosen": 1.334570288658142,
      "log_odds_ratio": -0.43603515625,
      "logits/chosen": -1.1876952648162842,
      "logits/rejected": -0.970898449420929,
      "logps/chosen": -0.8382812738418579,
      "logps/rejected": -1.853906273841858,
      "loss": 1.079,
      "nll_loss": 1.0261719226837158,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.08389892429113388,
      "rewards/margins": 0.10160522162914276,
      "rewards/rejected": -0.18537597358226776,
      "step": 1680
    },
    {
      "epoch": 0.218303946263644,
      "grad_norm": 1.434736589458896,
      "learning_rate": 1.9460170216420797e-06,
      "log_odds_chosen": 1.3171875476837158,
      "log_odds_ratio": -0.45231932401657104,
      "logits/chosen": -1.2048828601837158,
      "logits/rejected": -0.971875011920929,
      "logps/chosen": -0.76953125,
      "logps/rejected": -1.763281226158142,
      "loss": 1.0672,
      "nll_loss": 1.031640648841858,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07694091647863388,
      "rewards/margins": 0.09941406548023224,
      "rewards/rejected": -0.17641600966453552,
      "step": 1690
    },
    {
      "epoch": 0.21959568559064782,
      "grad_norm": 1.3325454514926989,
      "learning_rate": 1.9402850002906637e-06,
      "log_odds_chosen": 1.4177734851837158,
      "log_odds_ratio": -0.38520509004592896,
      "logits/chosen": -1.2087891101837158,
      "logits/rejected": -0.9681640863418579,
      "logps/chosen": -0.7705078125,
      "logps/rejected": -1.7998046875,
      "loss": 1.0651,
      "nll_loss": 1.011328101158142,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07706298679113388,
      "rewards/margins": 0.10270996391773224,
      "rewards/rejected": -0.17983397841453552,
      "step": 1700
    },
    {
      "epoch": 0.22088742491765162,
      "grad_norm": 1.4075507837221597,
      "learning_rate": 1.9346033336266974e-06,
      "log_odds_chosen": 1.2162353992462158,
      "log_odds_ratio": -0.468017578125,
      "logits/chosen": -1.0966796875,
      "logits/rejected": -0.903124988079071,
      "logps/chosen": -0.783398449420929,
      "logps/rejected": -1.65625,
      "loss": 1.0673,
      "nll_loss": 1.023828148841858,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07828368991613388,
      "rewards/margins": 0.08742675930261612,
      "rewards/rejected": -0.16567382216453552,
      "step": 1710
    },
    {
      "epoch": 0.22217916424465542,
      "grad_norm": 1.5811870984542153,
      "learning_rate": 1.9289712886816486e-06,
      "log_odds_chosen": 1.0413086414337158,
      "log_odds_ratio": -0.47832030057907104,
      "logits/chosen": -1.13671875,
      "logits/rejected": -0.9564453363418579,
      "logps/chosen": -0.770703136920929,
      "logps/rejected": -1.549414038658142,
      "loss": 1.0528,
      "nll_loss": 0.9779297113418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07707519829273224,
      "rewards/margins": 0.07799072563648224,
      "rewards/rejected": -0.15488281846046448,
      "step": 1720
    },
    {
      "epoch": 0.22347090357165925,
      "grad_norm": 1.2767552843937628,
      "learning_rate": 1.92338814733738e-06,
      "log_odds_chosen": 1.3216552734375,
      "log_odds_ratio": -0.40937501192092896,
      "logits/chosen": -1.2951171398162842,
      "logits/rejected": -1.0041015148162842,
      "logps/chosen": -0.698437511920929,
      "logps/rejected": -1.6337890625,
      "loss": 1.0676,
      "nll_loss": 0.9791015386581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06987304985523224,
      "rewards/margins": 0.09343872219324112,
      "rewards/rejected": -0.16335448622703552,
      "step": 1730
    },
    {
      "epoch": 0.22476264289866305,
      "grad_norm": 1.703654433886517,
      "learning_rate": 1.9178532059415367e-06,
      "log_odds_chosen": 1.2189452648162842,
      "log_odds_ratio": -0.48491209745407104,
      "logits/chosen": -1.108007788658142,
      "logits/rejected": -0.9486328363418579,
      "logps/chosen": -0.784375011920929,
      "logps/rejected": -1.6884765625,
      "loss": 1.0794,
      "nll_loss": 1.020117163658142,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07846679538488388,
      "rewards/margins": 0.09031982719898224,
      "rewards/rejected": -0.16867676377296448,
      "step": 1740
    },
    {
      "epoch": 0.22605438222566687,
      "grad_norm": 1.3197303152885576,
      "learning_rate": 1.9123657749350298e-06,
      "log_odds_chosen": 1.359375,
      "log_odds_ratio": -0.41845703125,
      "logits/chosen": -1.1828124523162842,
      "logits/rejected": -0.95703125,
      "logps/chosen": -0.707812488079071,
      "logps/rejected": -1.681640625,
      "loss": 1.0527,
      "nll_loss": 0.916210949420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07077636569738388,
      "rewards/margins": 0.09749756008386612,
      "rewards/rejected": -0.168212890625,
      "step": 1750
    },
    {
      "epoch": 0.22734612155267067,
      "grad_norm": 1.3752457761798864,
      "learning_rate": 1.9069251784911844e-06,
      "log_odds_chosen": 1.460302710533142,
      "log_odds_ratio": -0.4194580018520355,
      "logits/chosen": -1.1455078125,
      "logits/rejected": -0.974609375,
      "logps/chosen": -0.842968761920929,
      "logps/rejected": -1.952734351158142,
      "loss": 1.0929,
      "nll_loss": 1.1062500476837158,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.084228515625,
      "rewards/margins": 0.11113281548023224,
      "rewards/rejected": -0.19521483778953552,
      "step": 1760
    },
    {
      "epoch": 0.22863786087967447,
      "grad_norm": 1.3357777395021317,
      "learning_rate": 1.9015307541661132e-06,
      "log_odds_chosen": 1.176171898841858,
      "log_odds_ratio": -0.46562498807907104,
      "logits/chosen": -1.218359351158142,
      "logits/rejected": -1.0373046398162842,
      "logps/chosen": -0.7621093988418579,
      "logps/rejected": -1.5998046398162842,
      "loss": 1.095,
      "nll_loss": 1.002539038658142,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07619629055261612,
      "rewards/margins": 0.08369751274585724,
      "rewards/rejected": -0.15999755263328552,
      "step": 1770
    },
    {
      "epoch": 0.2299296002066783,
      "grad_norm": 1.3132989058982572,
      "learning_rate": 1.8961818525599089e-06,
      "log_odds_chosen": 1.2180664539337158,
      "log_odds_ratio": -0.4473632872104645,
      "logits/chosen": -1.1767578125,
      "logits/rejected": -0.947460949420929,
      "logps/chosen": -0.7367187738418579,
      "logps/rejected": -1.625390648841858,
      "loss": 1.0728,
      "nll_loss": 0.9925781488418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07366943359375,
      "rewards/margins": 0.08895263820886612,
      "rewards/rejected": -0.16269531846046448,
      "step": 1780
    },
    {
      "epoch": 0.2312213395336821,
      "grad_norm": 1.316861722581661,
      "learning_rate": 1.890877836988262e-06,
      "log_odds_chosen": 1.240820288658142,
      "log_odds_ratio": -0.40068358182907104,
      "logits/chosen": -1.1417968273162842,
      "logits/rejected": -0.9369140863418579,
      "logps/chosen": -0.746289074420929,
      "logps/rejected": -1.6218750476837158,
      "loss": 1.0663,
      "nll_loss": 1.0400390625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0745849609375,
      "rewards/margins": 0.08765868842601776,
      "rewards/rejected": -0.16218261420726776,
      "step": 1790
    },
    {
      "epoch": 0.23251307886068592,
      "grad_norm": 1.7931092708976446,
      "learning_rate": 1.8856180831641269e-06,
      "log_odds_chosen": 1.3117187023162842,
      "log_odds_ratio": -0.4071289002895355,
      "logits/chosen": -1.225000023841858,
      "logits/rejected": -0.9652343988418579,
      "logps/chosen": -0.7142578363418579,
      "logps/rejected": -1.6662108898162842,
      "loss": 1.0676,
      "nll_loss": 0.956250011920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07144775241613388,
      "rewards/margins": 0.0953521728515625,
      "rewards/rejected": -0.1666259765625,
      "step": 1800
    },
    {
      "epoch": 0.23380481818768972,
      "grad_norm": 1.3106318056930855,
      "learning_rate": 1.8804019788890737e-06,
      "log_odds_chosen": 1.5377929210662842,
      "log_odds_ratio": -0.3639160096645355,
      "logits/chosen": -1.1648437976837158,
      "logits/rejected": -0.8792968988418579,
      "logps/chosen": -0.720898449420929,
      "logps/rejected": -1.8449218273162842,
      "loss": 1.0564,
      "nll_loss": 1.0869140625,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07213135063648224,
      "rewards/margins": 0.11234740912914276,
      "rewards/rejected": -0.18452148139476776,
      "step": 1810
    },
    {
      "epoch": 0.23509655751469352,
      "grad_norm": 1.4595181023876223,
      "learning_rate": 1.8752289237539816e-06,
      "log_odds_chosen": 1.2355468273162842,
      "log_odds_ratio": -0.45244139432907104,
      "logits/chosen": -1.2228515148162842,
      "logits/rejected": -1.011132836341858,
      "logps/chosen": -0.7105468511581421,
      "logps/rejected": -1.6240234375,
      "loss": 1.0574,
      "nll_loss": 0.9613281488418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07095947116613388,
      "rewards/margins": 0.09142456203699112,
      "rewards/rejected": -0.16242675483226776,
      "step": 1820
    },
    {
      "epoch": 0.23638829684169735,
      "grad_norm": 1.3966000265636949,
      "learning_rate": 1.8700983288487376e-06,
      "log_odds_chosen": 1.0509765148162842,
      "log_odds_ratio": -0.4830566346645355,
      "logits/chosen": -1.1072266101837158,
      "logits/rejected": -0.9283202886581421,
      "logps/chosen": -0.8089843988418579,
      "logps/rejected": -1.568750023841858,
      "loss": 1.0709,
      "nll_loss": 1.095117211341858,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.08085937798023224,
      "rewards/margins": 0.07611083984375,
      "rewards/rejected": -0.15700682997703552,
      "step": 1830
    },
    {
      "epoch": 0.23768003616870115,
      "grad_norm": 1.3894198159912972,
      "learning_rate": 1.8650096164806275e-06,
      "log_odds_chosen": 1.0227539539337158,
      "log_odds_ratio": -0.50341796875,
      "logits/chosen": -1.0837891101837158,
      "logits/rejected": -0.9574218988418579,
      "logps/chosen": -0.83203125,
      "logps/rejected": -1.5613281726837158,
      "loss": 1.0795,
      "nll_loss": 1.0791015625,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.08322753757238388,
      "rewards/margins": 0.07283325493335724,
      "rewards/rejected": -0.1561279296875,
      "step": 1840
    },
    {
      "epoch": 0.23897177549570497,
      "grad_norm": 1.4364656975984647,
      "learning_rate": 1.8599622199011084e-06,
      "log_odds_chosen": 1.266821265220642,
      "log_odds_ratio": -0.4259277284145355,
      "logits/chosen": -1.181249976158142,
      "logits/rejected": -0.9615234136581421,
      "logps/chosen": -0.7847656011581421,
      "logps/rejected": -1.7003905773162842,
      "loss": 1.0789,
      "nll_loss": 1.0439453125,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.0784912109375,
      "rewards/margins": 0.09159698337316513,
      "rewards/rejected": -0.17014160752296448,
      "step": 1850
    },
    {
      "epoch": 0.24026351482270877,
      "grad_norm": 1.6303492430495117,
      "learning_rate": 1.854955583040673e-06,
      "log_odds_chosen": 1.181738257408142,
      "log_odds_ratio": -0.44550782442092896,
      "logits/chosen": -1.158789038658142,
      "logits/rejected": -0.9644531011581421,
      "logps/chosen": -0.7568359375,
      "logps/rejected": -1.599023461341858,
      "loss": 1.0719,
      "nll_loss": 1.017578125,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07564697414636612,
      "rewards/margins": 0.08420410007238388,
      "rewards/rejected": -0.15993651747703552,
      "step": 1860
    },
    {
      "epoch": 0.2415552541497126,
      "grad_norm": 1.4370695114557293,
      "learning_rate": 1.849989160251521e-06,
      "log_odds_chosen": 0.994921863079071,
      "log_odds_ratio": -0.484375,
      "logits/chosen": -1.1892578601837158,
      "logits/rejected": -0.957812488079071,
      "logps/chosen": -0.8203125,
      "logps/rejected": -1.5419921875,
      "loss": 1.0772,
      "nll_loss": 1.005273461341858,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.08205566555261612,
      "rewards/margins": 0.07221679389476776,
      "rewards/rejected": -0.15427246689796448,
      "step": 1870
    },
    {
      "epoch": 0.2428469934767164,
      "grad_norm": 1.2712485063950865,
      "learning_rate": 1.8450624160577701e-06,
      "log_odds_chosen": 1.143396019935608,
      "log_odds_ratio": -0.4758056700229645,
      "logits/chosen": -1.118749976158142,
      "logits/rejected": -0.9794921875,
      "logps/chosen": -0.7958984375,
      "logps/rejected": -1.635156273841858,
      "loss": 1.074,
      "nll_loss": 1.043359398841858,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07965087890625,
      "rewards/margins": 0.083892822265625,
      "rewards/rejected": -0.16367188096046448,
      "step": 1880
    },
    {
      "epoch": 0.2441387328037202,
      "grad_norm": 1.5110079177651148,
      "learning_rate": 1.8401748249129445e-06,
      "log_odds_chosen": 1.0840332508087158,
      "log_odds_ratio": -0.4921875,
      "logits/chosen": -1.1482422351837158,
      "logits/rejected": -0.9322265386581421,
      "logps/chosen": -0.7978515625,
      "logps/rejected": -1.603515625,
      "loss": 1.0865,
      "nll_loss": 1.116601586341858,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.079833984375,
      "rewards/margins": 0.08058013767004013,
      "rewards/rejected": -0.16044922173023224,
      "step": 1890
    },
    {
      "epoch": 0.24543047213072403,
      "grad_norm": 1.2946941022031773,
      "learning_rate": 1.8353258709644938e-06,
      "log_odds_chosen": 1.1730468273162842,
      "log_odds_ratio": -0.4471191465854645,
      "logits/chosen": -1.217382788658142,
      "logits/rejected": -0.9847656488418579,
      "logps/chosen": -0.7236328125,
      "logps/rejected": -1.5470702648162842,
      "loss": 1.0934,
      "nll_loss": 0.953320324420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07232666015625,
      "rewards/margins": 0.08240966498851776,
      "rewards/rejected": -0.15468749403953552,
      "step": 1900
    },
    {
      "epoch": 0.24672221145772782,
      "grad_norm": 1.3602102605280648,
      "learning_rate": 1.830515047825102e-06,
      "log_odds_chosen": 1.148461937904358,
      "log_odds_ratio": -0.4573730528354645,
      "logits/chosen": -1.116601586341858,
      "logits/rejected": -0.982421875,
      "logps/chosen": -0.744140625,
      "logps/rejected": -1.5080077648162842,
      "loss": 1.0933,
      "nll_loss": 1.037500023841858,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0743408203125,
      "rewards/margins": 0.07627563178539276,
      "rewards/rejected": -0.15083007514476776,
      "step": 1910
    },
    {
      "epoch": 0.24801395078473165,
      "grad_norm": 1.331413905721886,
      "learning_rate": 1.8257418583505536e-06,
      "log_odds_chosen": 1.2104003429412842,
      "log_odds_ratio": -0.45576173067092896,
      "logits/chosen": -1.1257812976837158,
      "logits/rejected": -0.9183593988418579,
      "logps/chosen": -0.74267578125,
      "logps/rejected": -1.592382788658142,
      "loss": 1.0579,
      "nll_loss": 1.063867211341858,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07421875,
      "rewards/margins": 0.08487548679113388,
      "rewards/rejected": -0.15922851860523224,
      "step": 1920
    },
    {
      "epoch": 0.24930569011173545,
      "grad_norm": 1.2763876400651855,
      "learning_rate": 1.8210058144239416e-06,
      "log_odds_chosen": 1.206640601158142,
      "log_odds_ratio": -0.47124022245407104,
      "logits/chosen": -1.147070288658142,
      "logits/rejected": -0.9585937261581421,
      "logps/chosen": -0.7783203125,
      "logps/rejected": -1.694726586341858,
      "loss": 1.0572,
      "nll_loss": 1.011132836341858,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07786865532398224,
      "rewards/margins": 0.09166260063648224,
      "rewards/rejected": -0.16943359375,
      "step": 1930
    },
    {
      "epoch": 0.25059742943873925,
      "grad_norm": 1.5617480677474052,
      "learning_rate": 1.816306436745999e-06,
      "log_odds_chosen": 1.2263672351837158,
      "log_odds_ratio": -0.49946290254592896,
      "logits/chosen": -1.127539038658142,
      "logits/rejected": -0.9693359136581421,
      "logps/chosen": -0.8187500238418579,
      "logps/rejected": -1.736718773841858,
      "loss": 1.1076,
      "nll_loss": 1.0691406726837158,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08187256008386612,
      "rewards/margins": 0.09178467094898224,
      "rewards/rejected": -0.173583984375,
      "step": 1940
    },
    {
      "epoch": 0.2518891687657431,
      "grad_norm": 1.2825733313227454,
      "learning_rate": 1.8116432546313529e-06,
      "log_odds_chosen": 1.30859375,
      "log_odds_ratio": -0.4261230528354645,
      "logits/chosen": -1.137304663658142,
      "logits/rejected": -0.9576171636581421,
      "logps/chosen": -0.741015613079071,
      "logps/rejected": -1.6667969226837158,
      "loss": 1.0635,
      "nll_loss": 0.9488281011581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0740966796875,
      "rewards/margins": 0.09265746921300888,
      "rewards/rejected": -0.16665038466453552,
      "step": 1950
    },
    {
      "epoch": 0.2531809080927469,
      "grad_norm": 1.341096409932551,
      "learning_rate": 1.8070158058105026e-06,
      "log_odds_chosen": 1.140344262123108,
      "log_odds_ratio": -0.476806640625,
      "logits/chosen": -1.109765648841858,
      "logits/rejected": -0.9212890863418579,
      "logps/chosen": -0.8140624761581421,
      "logps/rejected": -1.639062523841858,
      "loss": 1.0966,
      "nll_loss": 1.0810546875,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0814208984375,
      "rewards/margins": 0.08255767822265625,
      "rewards/rejected": -0.16398926079273224,
      "step": 1960
    },
    {
      "epoch": 0.2544726474197507,
      "grad_norm": 1.3760458560050721,
      "learning_rate": 1.8024236362373315e-06,
      "log_odds_chosen": 1.361883521080017,
      "log_odds_ratio": -0.42573243379592896,
      "logits/chosen": -1.138671875,
      "logits/rejected": -0.8843749761581421,
      "logps/chosen": -0.8173828125,
      "logps/rejected": -1.817968726158142,
      "loss": 1.0426,
      "nll_loss": 1.0187499523162842,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.08168945461511612,
      "rewards/margins": 0.10009308159351349,
      "rewards/rejected": -0.18183593451976776,
      "step": 1970
    },
    {
      "epoch": 0.2557643867467545,
      "grad_norm": 1.9365934338647697,
      "learning_rate": 1.7978662999019787e-06,
      "log_odds_chosen": 1.171411156654358,
      "log_odds_ratio": -0.4984374940395355,
      "logits/chosen": -1.1443359851837158,
      "logits/rejected": -0.9878906011581421,
      "logps/chosen": -0.839062511920929,
      "logps/rejected": -1.694921851158142,
      "loss": 1.0862,
      "nll_loss": 1.016015648841858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.08388672024011612,
      "rewards/margins": 0.08565368503332138,
      "rewards/rejected": -0.16950683295726776,
      "step": 1980
    },
    {
      "epoch": 0.25705612607375833,
      "grad_norm": 1.5127076094213072,
      "learning_rate": 1.793343358648881e-06,
      "log_odds_chosen": 1.217504858970642,
      "log_odds_ratio": -0.44682615995407104,
      "logits/chosen": -1.149023413658142,
      "logits/rejected": -0.9585937261581421,
      "logps/chosen": -0.7626953125,
      "logps/rejected": -1.6318359375,
      "loss": 1.029,
      "nll_loss": 1.012109398841858,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07624511420726776,
      "rewards/margins": 0.08691100776195526,
      "rewards/rejected": -0.16323241591453552,
      "step": 1990
    },
    {
      "epoch": 0.2583478654007621,
      "grad_norm": 1.2773700243690813,
      "learning_rate": 1.7888543819998317e-06,
      "log_odds_chosen": 1.44482421875,
      "log_odds_ratio": -0.38325196504592896,
      "logits/chosen": -1.1619141101837158,
      "logits/rejected": -0.9017578363418579,
      "logps/chosen": -0.7593749761581421,
      "logps/rejected": -1.831445336341858,
      "loss": 1.043,
      "nll_loss": 0.937304675579071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07591553032398224,
      "rewards/margins": 0.10738525539636612,
      "rewards/rejected": -0.18320313096046448,
      "step": 2000
    },
    {
      "epoch": 0.2596396047277659,
      "grad_norm": 1.3164715223864245,
      "learning_rate": 1.7843989469818819e-06,
      "log_odds_chosen": 1.339257836341858,
      "log_odds_ratio": -0.42207032442092896,
      "logits/chosen": -1.0916016101837158,
      "logits/rejected": -0.920703113079071,
      "logps/chosen": -0.791210949420929,
      "logps/rejected": -1.7917969226837158,
      "loss": 1.0776,
      "nll_loss": 1.115234375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07915039360523224,
      "rewards/margins": 0.10001220554113388,
      "rewards/rejected": -0.17917481064796448,
      "step": 2010
    },
    {
      "epoch": 0.26093134405476975,
      "grad_norm": 1.3922193226398354,
      "learning_rate": 1.779976637959939e-06,
      "log_odds_chosen": 1.28173828125,
      "log_odds_ratio": -0.43896484375,
      "logits/chosen": -1.14453125,
      "logits/rejected": -0.9814453125,
      "logps/chosen": -0.774121105670929,
      "logps/rejected": -1.723046898841858,
      "loss": 1.079,
      "nll_loss": 1.0144531726837158,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07741699367761612,
      "rewards/margins": 0.09483642876148224,
      "rewards/rejected": -0.17231445014476776,
      "step": 2020
    },
    {
      "epoch": 0.2622230833817736,
      "grad_norm": 1.5867735946732342,
      "learning_rate": 1.7755870464739012e-06,
      "log_odds_chosen": 1.1649901866912842,
      "log_odds_ratio": -0.42607420682907104,
      "logits/chosen": -1.136328101158142,
      "logits/rejected": -0.965624988079071,
      "logps/chosen": -0.7455078363418579,
      "logps/rejected": -1.5554687976837158,
      "loss": 1.0473,
      "nll_loss": 1.048828125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07449951022863388,
      "rewards/margins": 0.08096160739660263,
      "rewards/rejected": -0.15556640923023224,
      "step": 2030
    },
    {
      "epoch": 0.26351482270877735,
      "grad_norm": 1.4962573424371364,
      "learning_rate": 1.7712297710801907e-06,
      "log_odds_chosen": 1.169531226158142,
      "log_odds_ratio": -0.4840331971645355,
      "logits/chosen": -1.218359351158142,
      "logits/rejected": -1.0125000476837158,
      "logps/chosen": -0.7705078125,
      "logps/rejected": -1.623437523841858,
      "loss": 1.0736,
      "nll_loss": 0.9886718988418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07701416313648224,
      "rewards/margins": 0.08531494438648224,
      "rewards/rejected": -0.162353515625,
      "step": 2040
    },
    {
      "epoch": 0.2648065620357812,
      "grad_norm": 1.5367137020866226,
      "learning_rate": 1.7669044171975444e-06,
      "log_odds_chosen": 1.3913085460662842,
      "log_odds_ratio": -0.40239256620407104,
      "logits/chosen": -1.234765648841858,
      "logits/rejected": -1.016992211341858,
      "logps/chosen": -0.766894519329071,
      "logps/rejected": -1.778906226158142,
      "loss": 1.0515,
      "nll_loss": 0.9380859136581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07664795219898224,
      "rewards/margins": 0.10111083835363388,
      "rewards/rejected": -0.17788085341453552,
      "step": 2050
    },
    {
      "epoch": 0.266098301362785,
      "grad_norm": 1.4442294155730115,
      "learning_rate": 1.7626105969569268e-06,
      "log_odds_chosen": 1.34033203125,
      "log_odds_ratio": -0.4219726622104645,
      "logits/chosen": -1.0908203125,
      "logits/rejected": -0.9658203125,
      "logps/chosen": -0.7353515625,
      "logps/rejected": -1.712499976158142,
      "loss": 1.0659,
      "nll_loss": 1.110937476158142,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07352294772863388,
      "rewards/margins": 0.09757690131664276,
      "rewards/rejected": -0.17121581733226776,
      "step": 2060
    },
    {
      "epoch": 0.2673900406897888,
      "grad_norm": 1.7331386731707017,
      "learning_rate": 1.758347929055432e-06,
      "log_odds_chosen": 1.366601586341858,
      "log_odds_ratio": -0.41008299589157104,
      "logits/chosen": -1.1552734375,
      "logits/rejected": -0.961718738079071,
      "logps/chosen": -0.7796875238418579,
      "logps/rejected": -1.7746093273162842,
      "loss": 1.0332,
      "nll_loss": 1.0185546875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07795409858226776,
      "rewards/margins": 0.09932861477136612,
      "rewards/rejected": -0.17719726264476776,
      "step": 2070
    },
    {
      "epoch": 0.2686817800167926,
      "grad_norm": 1.5711426411889913,
      "learning_rate": 1.7541160386140582e-06,
      "log_odds_chosen": 1.40380859375,
      "log_odds_ratio": -0.36376953125,
      "logits/chosen": -1.2099609375,
      "logits/rejected": -0.9908202886581421,
      "logps/chosen": -0.707226574420929,
      "logps/rejected": -1.6857421398162842,
      "loss": 1.0501,
      "nll_loss": 1.0068359375,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.07076416164636612,
      "rewards/margins": 0.09784545749425888,
      "rewards/rejected": -0.16860350966453552,
      "step": 2080
    },
    {
      "epoch": 0.26997351934379643,
      "grad_norm": 1.37906452701969,
      "learning_rate": 1.7499145570392284e-06,
      "log_odds_chosen": 1.233007788658142,
      "log_odds_ratio": -0.427001953125,
      "logits/chosen": -1.1882812976837158,
      "logits/rejected": -0.9697265625,
      "logps/chosen": -0.701953113079071,
      "logps/rejected": -1.5773437023162842,
      "loss": 1.0535,
      "nll_loss": 1.002343773841858,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07008056342601776,
      "rewards/margins": 0.08765258640050888,
      "rewards/rejected": -0.15778808295726776,
      "step": 2090
    },
    {
      "epoch": 0.27126525867080026,
      "grad_norm": 1.4674573972993576,
      "learning_rate": 1.745743121887939e-06,
      "log_odds_chosen": 1.5597655773162842,
      "log_odds_ratio": -0.3653808534145355,
      "logits/chosen": -1.19921875,
      "logits/rejected": -0.957226574420929,
      "logps/chosen": -0.779101550579071,
      "logps/rejected": -1.916406273841858,
      "loss": 1.0486,
      "nll_loss": 0.984570324420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07790527492761612,
      "rewards/margins": 0.11383056640625,
      "rewards/rejected": -0.19157715141773224,
      "step": 2100
    },
    {
      "epoch": 0.27255699799780403,
      "grad_norm": 1.4652223187305242,
      "learning_rate": 1.7416013767364324e-06,
      "log_odds_chosen": 1.378881812095642,
      "log_odds_ratio": -0.43085938692092896,
      "logits/chosen": -1.1853516101837158,
      "logits/rejected": -1.0126953125,
      "logps/chosen": -0.8238281011581421,
      "logps/rejected": -1.8527343273162842,
      "loss": 1.0598,
      "nll_loss": 1.058007836341858,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0823974609375,
      "rewards/margins": 0.10298462212085724,
      "rewards/rejected": -0.18537597358226776,
      "step": 2110
    },
    {
      "epoch": 0.27384873732480786,
      "grad_norm": 1.7711240330762235,
      "learning_rate": 1.7374889710522776e-06,
      "log_odds_chosen": 1.299902319908142,
      "log_odds_ratio": -0.4232421815395355,
      "logits/chosen": -1.157812476158142,
      "logits/rejected": -0.964062511920929,
      "logps/chosen": -0.732421875,
      "logps/rejected": -1.6749999523162842,
      "loss": 1.0622,
      "nll_loss": 1.0480468273162842,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07327880710363388,
      "rewards/margins": 0.09409485012292862,
      "rewards/rejected": -0.16745606064796448,
      "step": 2120
    },
    {
      "epoch": 0.2751404766518117,
      "grad_norm": 1.3030813369326975,
      "learning_rate": 1.7334055600697579e-06,
      "log_odds_chosen": 1.449316382408142,
      "log_odds_ratio": -0.40205079317092896,
      "logits/chosen": -1.219140648841858,
      "logits/rejected": -0.9908202886581421,
      "logps/chosen": -0.729687511920929,
      "logps/rejected": -1.809960961341858,
      "loss": 1.0731,
      "nll_loss": 1.0060546398162842,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07297363132238388,
      "rewards/margins": 0.10809326171875,
      "rewards/rejected": -0.18115234375,
      "step": 2130
    },
    {
      "epoch": 0.27643221597881545,
      "grad_norm": 1.464640572436433,
      "learning_rate": 1.7293508046684678e-06,
      "log_odds_chosen": 1.437353491783142,
      "log_odds_ratio": -0.43017578125,
      "logits/chosen": -1.209375023841858,
      "logits/rejected": -1.0234375,
      "logps/chosen": -0.751660168170929,
      "logps/rejected": -1.811914086341858,
      "loss": 1.0532,
      "nll_loss": 1.046875,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07513427734375,
      "rewards/margins": 0.106201171875,
      "rewards/rejected": -0.18129882216453552,
      "step": 2140
    },
    {
      "epoch": 0.2777239553058193,
      "grad_norm": 1.4928033483255845,
      "learning_rate": 1.7253243712550145e-06,
      "log_odds_chosen": 1.5045897960662842,
      "log_odds_ratio": -0.3724609315395355,
      "logits/chosen": -1.076171875,
      "logits/rejected": -0.906054675579071,
      "logps/chosen": -0.720019519329071,
      "logps/rejected": -1.7900390625,
      "loss": 1.0298,
      "nll_loss": 1.0207030773162842,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07199706882238388,
      "rewards/margins": 0.10687255859375,
      "rewards/rejected": -0.17897948622703552,
      "step": 2150
    },
    {
      "epoch": 0.2790156946328231,
      "grad_norm": 1.5632429685519984,
      "learning_rate": 1.7213259316477406e-06,
      "log_odds_chosen": 1.410058617591858,
      "log_odds_ratio": -0.4019531309604645,
      "logits/chosen": -1.174414038658142,
      "logits/rejected": -0.977343738079071,
      "logps/chosen": -0.732226550579071,
      "logps/rejected": -1.7531249523162842,
      "loss": 1.0438,
      "nll_loss": 1.083398461341858,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07319335639476776,
      "rewards/margins": 0.10199890285730362,
      "rewards/rejected": -0.17524413764476776,
      "step": 2160
    },
    {
      "epoch": 0.2803074339598269,
      "grad_norm": 1.4310232321792595,
      "learning_rate": 1.7173551629643674e-06,
      "log_odds_chosen": 1.403564453125,
      "log_odds_ratio": -0.38969725370407104,
      "logits/chosen": -1.287109375,
      "logits/rejected": -1.0421874523162842,
      "logps/chosen": -0.7118164300918579,
      "logps/rejected": -1.684179663658142,
      "loss": 1.0517,
      "nll_loss": 1.043554663658142,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07115478813648224,
      "rewards/margins": 0.09721069037914276,
      "rewards/rejected": -0.16848143935203552,
      "step": 2170
    },
    {
      "epoch": 0.2815991732868307,
      "grad_norm": 1.3435552892158844,
      "learning_rate": 1.713411747512477e-06,
      "log_odds_chosen": 1.390039086341858,
      "log_odds_ratio": -0.413818359375,
      "logits/chosen": -1.232421875,
      "logits/rejected": -1.023828148841858,
      "logps/chosen": -0.720410168170929,
      "logps/rejected": -1.706640601158142,
      "loss": 1.0531,
      "nll_loss": 0.954296886920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07198486477136612,
      "rewards/margins": 0.09860839694738388,
      "rewards/rejected": -0.17070312798023224,
      "step": 2180
    },
    {
      "epoch": 0.28289091261383453,
      "grad_norm": 1.3543097172089642,
      "learning_rate": 1.709495372682753e-06,
      "log_odds_chosen": 1.364648461341858,
      "log_odds_ratio": -0.41474610567092896,
      "logits/chosen": -1.180273413658142,
      "logits/rejected": -0.9876953363418579,
      "logps/chosen": -0.7391601800918579,
      "logps/rejected": -1.73046875,
      "loss": 1.0347,
      "nll_loss": 0.991406261920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07390137016773224,
      "rewards/margins": 0.099273681640625,
      "rewards/rejected": -0.17304687201976776,
      "step": 2190
    },
    {
      "epoch": 0.28418265194083836,
      "grad_norm": 1.450327152039063,
      "learning_rate": 1.7056057308448832e-06,
      "log_odds_chosen": 1.3577148914337158,
      "log_odds_ratio": -0.4122558534145355,
      "logits/chosen": -1.173437476158142,
      "logits/rejected": -0.9654296636581421,
      "logps/chosen": -0.7728515863418579,
      "logps/rejected": -1.755859375,
      "loss": 1.0583,
      "nll_loss": 1.1042969226837158,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07724609225988388,
      "rewards/margins": 0.09824218600988388,
      "rewards/rejected": -0.17539063096046448,
      "step": 2200
    },
    {
      "epoch": 0.28547439126784213,
      "grad_norm": 1.6354500208184082,
      "learning_rate": 1.701742519246068e-06,
      "log_odds_chosen": 1.3793456554412842,
      "log_odds_ratio": -0.40581053495407104,
      "logits/chosen": -1.097265601158142,
      "logits/rejected": -0.9345703125,
      "logps/chosen": -0.736328125,
      "logps/rejected": -1.742773413658142,
      "loss": 1.0481,
      "nll_loss": 1.067968726158142,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0736083984375,
      "rewards/margins": 0.10062255710363388,
      "rewards/rejected": -0.17421874403953552,
      "step": 2210
    },
    {
      "epoch": 0.28676613059484596,
      "grad_norm": 1.398866809923403,
      "learning_rate": 1.6979054399120355e-06,
      "log_odds_chosen": 1.503320336341858,
      "log_odds_ratio": -0.4133056700229645,
      "logits/chosen": -1.166406273841858,
      "logits/rejected": -0.9244140386581421,
      "logps/chosen": -0.7568359375,
      "logps/rejected": -1.8800780773162842,
      "loss": 1.0551,
      "nll_loss": 1.0281250476837158,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07568359375,
      "rewards/margins": 0.11248626559972763,
      "rewards/rejected": -0.18818359076976776,
      "step": 2220
    },
    {
      "epoch": 0.2880578699218498,
      "grad_norm": 3.253046351692678,
      "learning_rate": 1.6940941995505069e-06,
      "log_odds_chosen": 1.4011719226837158,
      "log_odds_ratio": -0.40546876192092896,
      "logits/chosen": -1.1457030773162842,
      "logits/rejected": -0.936718761920929,
      "logps/chosen": -0.7574218511581421,
      "logps/rejected": -1.775781273841858,
      "loss": 1.0325,
      "nll_loss": 1.094335913658142,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07568359375,
      "rewards/margins": 0.10195312649011612,
      "rewards/rejected": -0.17766113579273224,
      "step": 2230
    },
    {
      "epoch": 0.28934960924885356,
      "grad_norm": 1.3927117003877405,
      "learning_rate": 1.6903085094570331e-06,
      "log_odds_chosen": 1.321923851966858,
      "log_odds_ratio": -0.4161621034145355,
      "logits/chosen": -1.1980469226837158,
      "logits/rejected": -1.0119140148162842,
      "logps/chosen": -0.75537109375,
      "logps/rejected": -1.731835961341858,
      "loss": 1.0522,
      "nll_loss": 1.0460937023162842,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07557372748851776,
      "rewards/margins": 0.09777526557445526,
      "rewards/rejected": -0.17336425185203552,
      "step": 2240
    },
    {
      "epoch": 0.2906413485758574,
      "grad_norm": 1.5831060295374126,
      "learning_rate": 1.6865480854231356e-06,
      "log_odds_chosen": 1.4000976085662842,
      "log_odds_ratio": -0.41240233182907104,
      "logits/chosen": -1.188085913658142,
      "logits/rejected": -0.983203113079071,
      "logps/chosen": -0.786914050579071,
      "logps/rejected": -1.790624976158142,
      "loss": 1.0816,
      "nll_loss": 1.0236327648162842,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07871093600988388,
      "rewards/margins": 0.10028076171875,
      "rewards/rejected": -0.178955078125,
      "step": 2250
    },
    {
      "epoch": 0.2919330879028612,
      "grad_norm": 1.663941379685356,
      "learning_rate": 1.682812647646685e-06,
      "log_odds_chosen": 1.664648413658142,
      "log_odds_ratio": -0.34648436307907104,
      "logits/chosen": -1.209570288658142,
      "logits/rejected": -0.9781249761581421,
      "logps/chosen": -0.6900390386581421,
      "logps/rejected": -1.9246094226837158,
      "loss": 1.028,
      "nll_loss": 0.960156261920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06901855766773224,
      "rewards/margins": 0.12341918796300888,
      "rewards/rejected": -0.19233398139476776,
      "step": 2260
    },
    {
      "epoch": 0.29322482722986504,
      "grad_norm": 1.4352924893648698,
      "learning_rate": 1.6791019206444541e-06,
      "log_odds_chosen": 1.498925805091858,
      "log_odds_ratio": -0.45512694120407104,
      "logits/chosen": -1.1960937976837158,
      "logits/rejected": -0.9359375238418579,
      "logps/chosen": -0.775195300579071,
      "logps/rejected": -1.912500023841858,
      "loss": 1.0363,
      "nll_loss": 1.000585913658142,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0775146484375,
      "rewards/margins": 0.11378173530101776,
      "rewards/rejected": -0.19123534858226776,
      "step": 2270
    },
    {
      "epoch": 0.2945165665568688,
      "grad_norm": 1.4049718397651765,
      "learning_rate": 1.675415633166782e-06,
      "log_odds_chosen": 1.373925805091858,
      "log_odds_ratio": -0.4488525390625,
      "logits/chosen": -1.2433593273162842,
      "logits/rejected": -1.0574219226837158,
      "logps/chosen": -0.7152343988418579,
      "logps/rejected": -1.710546851158142,
      "loss": 1.0473,
      "nll_loss": 0.938671886920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07159423828125,
      "rewards/margins": 0.0995330810546875,
      "rewards/rejected": -0.17106933891773224,
      "step": 2280
    },
    {
      "epoch": 0.29580830588387264,
      "grad_norm": 1.430606464052193,
      "learning_rate": 1.6717535181142914e-06,
      "log_odds_chosen": 1.393408179283142,
      "log_odds_ratio": -0.4036621153354645,
      "logits/chosen": -1.149023413658142,
      "logits/rejected": -0.957226574420929,
      "logps/chosen": -0.7818359136581421,
      "logps/rejected": -1.805078148841858,
      "loss": 1.0498,
      "nll_loss": 1.043554663658142,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07816161960363388,
      "rewards/margins": 0.10239867866039276,
      "rewards/rejected": -0.18049316108226776,
      "step": 2290
    },
    {
      "epoch": 0.29710004521087646,
      "grad_norm": 1.464839047786352,
      "learning_rate": 1.668115312456598e-06,
      "log_odds_chosen": 1.475976586341858,
      "log_odds_ratio": -0.36674803495407104,
      "logits/chosen": -1.299414038658142,
      "logits/rejected": -0.9701172113418579,
      "logps/chosen": -0.7010742425918579,
      "logps/rejected": -1.742578148841858,
      "loss": 1.0238,
      "nll_loss": 0.96484375,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07010497897863388,
      "rewards/margins": 0.10404052585363388,
      "rewards/rejected": -0.17421874403953552,
      "step": 2300
    },
    {
      "epoch": 0.29839178453788023,
      "grad_norm": 1.383961073373241,
      "learning_rate": 1.6645007571529578e-06,
      "log_odds_chosen": 1.392822265625,
      "log_odds_ratio": -0.4227050840854645,
      "logits/chosen": -1.1535155773162842,
      "logits/rejected": -0.978320300579071,
      "logps/chosen": -0.816210925579071,
      "logps/rejected": -1.820703148841858,
      "loss": 1.0509,
      "nll_loss": 1.028906226158142,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.08172607421875,
      "rewards/margins": 0.10060425102710724,
      "rewards/rejected": -0.18220214545726776,
      "step": 2310
    },
    {
      "epoch": 0.29968352386488406,
      "grad_norm": 1.4690858074866915,
      "learning_rate": 1.6609095970747992e-06,
      "log_odds_chosen": 1.2936522960662842,
      "log_odds_ratio": -0.40742188692092896,
      "logits/chosen": -1.091210961341858,
      "logits/rejected": -0.9072265625,
      "logps/chosen": -0.720898449420929,
      "logps/rejected": -1.6134765148162842,
      "loss": 1.0573,
      "nll_loss": 0.9898437261581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07207031548023224,
      "rewards/margins": 0.08919677883386612,
      "rewards/rejected": -0.16118164360523224,
      "step": 2320
    },
    {
      "epoch": 0.3009752631918879,
      "grad_norm": 1.4149358445174556,
      "learning_rate": 1.6573415809300833e-06,
      "log_odds_chosen": 1.281835913658142,
      "log_odds_ratio": -0.41411131620407104,
      "logits/chosen": -1.1648437976837158,
      "logits/rejected": -1.005468726158142,
      "logps/chosen": -0.733593761920929,
      "logps/rejected": -1.634374976158142,
      "loss": 1.0494,
      "nll_loss": 1.0197265148162842,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07342529296875,
      "rewards/margins": 0.08999023586511612,
      "rewards/rejected": -0.16340331733226776,
      "step": 2330
    },
    {
      "epoch": 0.3022670025188917,
      "grad_norm": 1.3938562110076298,
      "learning_rate": 1.6537964611894462e-06,
      "log_odds_chosen": 1.2960937023162842,
      "log_odds_ratio": -0.4034667909145355,
      "logits/chosen": -1.236914038658142,
      "logits/rejected": -1.028710961341858,
      "logps/chosen": -0.771679699420929,
      "logps/rejected": -1.6945312023162842,
      "loss": 1.0385,
      "nll_loss": 0.9970703125,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07716064155101776,
      "rewards/margins": 0.09230957180261612,
      "rewards/rejected": -0.16938476264476776,
      "step": 2340
    },
    {
      "epoch": 0.3035587418458955,
      "grad_norm": 1.6894515563160155,
      "learning_rate": 1.6502739940140692e-06,
      "log_odds_chosen": 1.2419922351837158,
      "log_odds_ratio": -0.40234375,
      "logits/chosen": -1.1535155773162842,
      "logits/rejected": -0.986132800579071,
      "logps/chosen": -0.6810547113418579,
      "logps/rejected": -1.532812476158142,
      "loss": 1.064,
      "nll_loss": 1.0050780773162842,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06813964992761612,
      "rewards/margins": 0.08524779975414276,
      "rewards/rejected": -0.15327148139476776,
      "step": 2350
    },
    {
      "epoch": 0.3048504811728993,
      "grad_norm": 1.4917376954340076,
      "learning_rate": 1.6467739391852364e-06,
      "log_odds_chosen": 1.118310570716858,
      "log_odds_ratio": -0.4642089903354645,
      "logits/chosen": -1.198632836341858,
      "logits/rejected": -1.002539038658142,
      "logps/chosen": -0.7500976324081421,
      "logps/rejected": -1.544531226158142,
      "loss": 1.0377,
      "nll_loss": 0.988476574420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.074951171875,
      "rewards/margins": 0.07948608696460724,
      "rewards/rejected": -0.15444335341453552,
      "step": 2360
    },
    {
      "epoch": 0.30614222049990314,
      "grad_norm": 1.9800899582075846,
      "learning_rate": 1.6432960600355221e-06,
      "log_odds_chosen": 1.436914086341858,
      "log_odds_ratio": -0.41191405057907104,
      "logits/chosen": -1.142968773841858,
      "logits/rejected": -0.9443359375,
      "logps/chosen": -0.8306640386581421,
      "logps/rejected": -1.9162108898162842,
      "loss": 1.0382,
      "nll_loss": 1.0291016101837158,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.08303222805261612,
      "rewards/margins": 0.10868225246667862,
      "rewards/rejected": -0.19167479872703552,
      "step": 2370
    },
    {
      "epoch": 0.3074339598269069,
      "grad_norm": 1.343456393550254,
      "learning_rate": 1.6398401233815756e-06,
      "log_odds_chosen": 1.602441430091858,
      "log_odds_ratio": -0.36518555879592896,
      "logits/chosen": -1.311132788658142,
      "logits/rejected": -0.9898437261581421,
      "logps/chosen": -0.764453113079071,
      "logps/rejected": -1.962890625,
      "loss": 1.0407,
      "nll_loss": 0.953125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07648925483226776,
      "rewards/margins": 0.11967773735523224,
      "rewards/rejected": -0.19626465439796448,
      "step": 2380
    },
    {
      "epoch": 0.30872569915391074,
      "grad_norm": 1.4840370862815913,
      "learning_rate": 1.6364058994584524e-06,
      "log_odds_chosen": 1.3615233898162842,
      "log_odds_ratio": -0.4132324159145355,
      "logits/chosen": -1.1716797351837158,
      "logits/rejected": -0.9453125,
      "logps/chosen": -0.7710937261581421,
      "logps/rejected": -1.804296851158142,
      "loss": 1.0514,
      "nll_loss": 1.094140648841858,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07709960639476776,
      "rewards/margins": 0.10357666015625,
      "rewards/rejected": -0.1805419921875,
      "step": 2390
    },
    {
      "epoch": 0.31001743848091456,
      "grad_norm": 1.646153300373328,
      "learning_rate": 1.6329931618554522e-06,
      "log_odds_chosen": 1.5543944835662842,
      "log_odds_ratio": -0.4095703065395355,
      "logits/chosen": -1.183203101158142,
      "logits/rejected": -1.0109374523162842,
      "logps/chosen": -0.7749999761581421,
      "logps/rejected": -1.941015601158142,
      "loss": 1.094,
      "nll_loss": 1.0146484375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07749023288488388,
      "rewards/margins": 0.1166839599609375,
      "rewards/rejected": -0.19414062798023224,
      "step": 2400
    },
    {
      "epoch": 0.31130917780791834,
      "grad_norm": 1.50876978762928,
      "learning_rate": 1.6296016874534209e-06,
      "log_odds_chosen": 1.3102538585662842,
      "log_odds_ratio": -0.43989259004592896,
      "logits/chosen": -1.162109375,
      "logits/rejected": -0.9833984375,
      "logps/chosen": -0.779492199420929,
      "logps/rejected": -1.750390648841858,
      "loss": 1.0635,
      "nll_loss": 1.0673828125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07797851413488388,
      "rewards/margins": 0.09709472954273224,
      "rewards/rejected": -0.175048828125,
      "step": 2410
    },
    {
      "epoch": 0.31260091713492216,
      "grad_norm": 1.4433249867946605,
      "learning_rate": 1.6262312563634835e-06,
      "log_odds_chosen": 1.472070336341858,
      "log_odds_ratio": -0.38288575410842896,
      "logits/chosen": -1.1824219226837158,
      "logits/rejected": -0.98046875,
      "logps/chosen": -0.7474609613418579,
      "logps/rejected": -1.8054687976837158,
      "loss": 1.0462,
      "nll_loss": 1.0339844226837158,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07474365085363388,
      "rewards/margins": 0.10581054538488388,
      "rewards/rejected": -0.18051758408546448,
      "step": 2420
    },
    {
      "epoch": 0.313892656461926,
      "grad_norm": 1.4611690322509405,
      "learning_rate": 1.6228816518671587e-06,
      "log_odds_chosen": 1.360742211341858,
      "log_odds_ratio": -0.42255860567092896,
      "logits/chosen": -1.186914086341858,
      "logits/rejected": -0.9955078363418579,
      "logps/chosen": -0.7376953363418579,
      "logps/rejected": -1.7277343273162842,
      "loss": 1.0245,
      "nll_loss": 1.0105469226837158,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07375488430261612,
      "rewards/margins": 0.09897460788488388,
      "rewards/rejected": -0.17270508408546448,
      "step": 2430
    },
    {
      "epoch": 0.3151843957889298,
      "grad_norm": 1.4892979215794329,
      "learning_rate": 1.619552660357832e-06,
      "log_odds_chosen": 1.344091773033142,
      "log_odds_ratio": -0.4166503846645355,
      "logits/chosen": -1.1281249523162842,
      "logits/rejected": -0.9535156488418579,
      "logps/chosen": -0.783398449420929,
      "logps/rejected": -1.78515625,
      "loss": 1.0317,
      "nll_loss": 0.9794921875,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07840576022863388,
      "rewards/margins": 0.10019226372241974,
      "rewards/rejected": -0.178466796875,
      "step": 2440
    },
    {
      "epoch": 0.3164761351159336,
      "grad_norm": 1.5231614106480502,
      "learning_rate": 1.616244071283537e-06,
      "log_odds_chosen": 1.4494507312774658,
      "log_odds_ratio": -0.4336914122104645,
      "logits/chosen": -1.185937523841858,
      "logits/rejected": -0.98046875,
      "logps/chosen": -0.7406250238418579,
      "logps/rejected": -1.802343726158142,
      "loss": 1.0278,
      "nll_loss": 0.9419921636581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07410888373851776,
      "rewards/margins": 0.10613403469324112,
      "rewards/rejected": -0.18012695014476776,
      "step": 2450
    },
    {
      "epoch": 0.3177678744429374,
      "grad_norm": 1.6994086077204105,
      "learning_rate": 1.6129556770910235e-06,
      "log_odds_chosen": 1.408203125,
      "log_odds_ratio": -0.4393066465854645,
      "logits/chosen": -1.2296874523162842,
      "logits/rejected": -0.9857422113418579,
      "logps/chosen": -0.830273449420929,
      "logps/rejected": -1.896093726158142,
      "loss": 1.0262,
      "nll_loss": 1.027929663658142,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.08304443210363388,
      "rewards/margins": 0.10650634765625,
      "rewards/rejected": -0.18950195610523224,
      "step": 2460
    },
    {
      "epoch": 0.31905961376994124,
      "grad_norm": 1.4240021638150706,
      "learning_rate": 1.6096872731710673e-06,
      "log_odds_chosen": 1.3523437976837158,
      "log_odds_ratio": -0.4209960997104645,
      "logits/chosen": -1.3083984851837158,
      "logits/rejected": -1.0935547351837158,
      "logps/chosen": -0.7364257574081421,
      "logps/rejected": -1.703710913658142,
      "loss": 1.0716,
      "nll_loss": 1.0187499523162842,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07359619438648224,
      "rewards/margins": 0.09668274223804474,
      "rewards/rejected": -0.17038574814796448,
      "step": 2470
    },
    {
      "epoch": 0.320351353096945,
      "grad_norm": 1.8312501857733896,
      "learning_rate": 1.6064386578049978e-06,
      "log_odds_chosen": 1.3611328601837158,
      "log_odds_ratio": -0.4229980409145355,
      "logits/chosen": -1.1912109851837158,
      "logits/rejected": -0.983593761920929,
      "logps/chosen": -0.774609386920929,
      "logps/rejected": -1.762109398841858,
      "loss": 1.0292,
      "nll_loss": 0.977343738079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07742919772863388,
      "rewards/margins": 0.09885253757238388,
      "rewards/rejected": -0.17629393935203552,
      "step": 2480
    },
    {
      "epoch": 0.32164309242394884,
      "grad_norm": 1.4612159161532716,
      "learning_rate": 1.6032096321124046e-06,
      "log_odds_chosen": 1.294335961341858,
      "log_odds_ratio": -0.4075927734375,
      "logits/chosen": -1.1730468273162842,
      "logits/rejected": -1.015234351158142,
      "logps/chosen": -0.7837890386581421,
      "logps/rejected": -1.701171875,
      "loss": 1.0493,
      "nll_loss": 1.0441405773162842,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07840576022863388,
      "rewards/margins": 0.09180907905101776,
      "rewards/rejected": -0.17019042372703552,
      "step": 2490
    },
    {
      "epoch": 0.32293483175095267,
      "grad_norm": 1.4460549593736796,
      "learning_rate": 1.6e-06,
      "log_odds_chosen": 1.229833960533142,
      "log_odds_ratio": -0.45458984375,
      "logits/chosen": -1.2169921398162842,
      "logits/rejected": -1.0431640148162842,
      "logps/chosen": -0.7837890386581421,
      "logps/rejected": -1.668359398841858,
      "loss": 1.0188,
      "nll_loss": 0.958789050579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07844237983226776,
      "rewards/margins": 0.08848266303539276,
      "rewards/rejected": -0.16679687798023224,
      "step": 2500
    },
    {
      "epoch": 0.3242265710779565,
      "grad_norm": 1.5193996446299936,
      "learning_rate": 1.5968095681115984e-06,
      "log_odds_chosen": 1.462988257408142,
      "log_odds_ratio": -0.39848631620407104,
      "logits/chosen": -1.159765601158142,
      "logits/rejected": -0.9498046636581421,
      "logps/chosen": -0.774218738079071,
      "logps/rejected": -1.853124976158142,
      "loss": 1.0451,
      "nll_loss": 1.0232422351837158,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07744140923023224,
      "rewards/margins": 0.10784912109375,
      "rewards/rejected": -0.18513183295726776,
      "step": 2510
    },
    {
      "epoch": 0.32551831040496026,
      "grad_norm": 1.3876887215351676,
      "learning_rate": 1.5936381457791914e-06,
      "log_odds_chosen": 1.424218773841858,
      "log_odds_ratio": -0.4034179747104645,
      "logits/chosen": -1.2722656726837158,
      "logits/rejected": -1.065039038658142,
      "logps/chosen": -0.7193359136581421,
      "logps/rejected": -1.751562476158142,
      "loss": 1.025,
      "nll_loss": 0.9185546636581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07187499850988388,
      "rewards/margins": 0.10329131782054901,
      "rewards/rejected": -0.17526856064796448,
      "step": 2520
    },
    {
      "epoch": 0.3268100497319641,
      "grad_norm": 2.253820601782652,
      "learning_rate": 1.590485544975088e-06,
      "log_odds_chosen": 1.548730492591858,
      "log_odds_ratio": -0.38081055879592896,
      "logits/chosen": -1.2734375,
      "logits/rejected": -1.056640625,
      "logps/chosen": -0.719531238079071,
      "logps/rejected": -1.853515625,
      "loss": 1.0494,
      "nll_loss": 0.9873046875,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07192382961511612,
      "rewards/margins": 0.11336670070886612,
      "rewards/rejected": -0.18544921278953552,
      "step": 2530
    },
    {
      "epoch": 0.3281017890589679,
      "grad_norm": 1.3747681344638403,
      "learning_rate": 1.5873515802650901e-06,
      "log_odds_chosen": 1.33984375,
      "log_odds_ratio": -0.41450196504592896,
      "logits/chosen": -1.237890601158142,
      "logits/rejected": -1.0400390625,
      "logps/chosen": -0.7544921636581421,
      "logps/rejected": -1.750390648841858,
      "loss": 1.0303,
      "nll_loss": 0.9515625238418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07539062201976776,
      "rewards/margins": 0.09970702975988388,
      "rewards/rejected": -0.17519530653953552,
      "step": 2540
    },
    {
      "epoch": 0.3293935283859717,
      "grad_norm": 1.4181531333436277,
      "learning_rate": 1.584236068762679e-06,
      "log_odds_chosen": 1.4600098133087158,
      "log_odds_ratio": -0.40961915254592896,
      "logits/chosen": -1.2615234851837158,
      "logits/rejected": -1.0164062976837158,
      "logps/chosen": -0.7635742425918579,
      "logps/rejected": -1.851171851158142,
      "loss": 1.0298,
      "nll_loss": 0.9390624761581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07634277641773224,
      "rewards/margins": 0.108795166015625,
      "rewards/rejected": -0.18498535454273224,
      "step": 2550
    },
    {
      "epoch": 0.3306852677129755,
      "grad_norm": 1.5370784846255565,
      "learning_rate": 1.5811388300841894e-06,
      "log_odds_chosen": 1.3122069835662842,
      "log_odds_ratio": -0.45976561307907104,
      "logits/chosen": -1.178125023841858,
      "logits/rejected": -1.0466797351837158,
      "logps/chosen": -0.705273449420929,
      "logps/rejected": -1.6423828601837158,
      "loss": 1.0235,
      "nll_loss": 0.9564453363418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07044677436351776,
      "rewards/margins": 0.09376678615808487,
      "rewards/rejected": -0.16425780951976776,
      "step": 2560
    },
    {
      "epoch": 0.33197700703997934,
      "grad_norm": 1.47951200592382,
      "learning_rate": 1.5780596863049431e-06,
      "log_odds_chosen": 1.221411108970642,
      "log_odds_ratio": -0.4374023377895355,
      "logits/chosen": -1.2003905773162842,
      "logits/rejected": -0.991015613079071,
      "logps/chosen": -0.7542968988418579,
      "logps/rejected": -1.672460913658142,
      "loss": 1.0153,
      "nll_loss": 0.9287109375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07545165717601776,
      "rewards/margins": 0.09169311821460724,
      "rewards/rejected": -0.1671142578125,
      "step": 2570
    },
    {
      "epoch": 0.3332687463669831,
      "grad_norm": 1.8432236859206257,
      "learning_rate": 1.5749984619163156e-06,
      "log_odds_chosen": 1.197241187095642,
      "log_odds_ratio": -0.4608398377895355,
      "logits/chosen": -1.240234375,
      "logits/rejected": -0.9732421636581421,
      "logps/chosen": -0.7623046636581421,
      "logps/rejected": -1.622460961341858,
      "loss": 1.0356,
      "nll_loss": 0.964648425579071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07619629055261612,
      "rewards/margins": 0.08606567233800888,
      "rewards/rejected": -0.16225585341453552,
      "step": 2580
    },
    {
      "epoch": 0.33456048569398694,
      "grad_norm": 1.7574783045605753,
      "learning_rate": 1.5719549837837187e-06,
      "log_odds_chosen": 1.5114257335662842,
      "log_odds_ratio": -0.3783203065395355,
      "logits/chosen": -1.216210961341858,
      "logits/rejected": -1.0011718273162842,
      "logps/chosen": -0.7455078363418579,
      "logps/rejected": -1.850000023841858,
      "loss": 1.0331,
      "nll_loss": 1.0146484375,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07457275688648224,
      "rewards/margins": 0.11044921725988388,
      "rewards/rejected": -0.18508300185203552,
      "step": 2590
    },
    {
      "epoch": 0.33585222502099077,
      "grad_norm": 1.5235642516222496,
      "learning_rate": 1.5689290811054722e-06,
      "log_odds_chosen": 1.3966796398162842,
      "log_odds_ratio": -0.3941406309604645,
      "logits/chosen": -1.181054711341858,
      "logits/rejected": -1.017578125,
      "logps/chosen": -0.756640613079071,
      "logps/rejected": -1.760156273841858,
      "loss": 1.0286,
      "nll_loss": 0.9996093511581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07568359375,
      "rewards/margins": 0.10050048679113388,
      "rewards/rejected": -0.17607422173023224,
      "step": 2600
    },
    {
      "epoch": 0.3371439643479946,
      "grad_norm": 1.5596353514751462,
      "learning_rate": 1.5659205853725426e-06,
      "log_odds_chosen": 1.3064453601837158,
      "log_odds_ratio": -0.44648438692092896,
      "logits/chosen": -1.241601586341858,
      "logits/rejected": -1.0642578601837158,
      "logps/chosen": -0.809374988079071,
      "logps/rejected": -1.818359375,
      "loss": 1.0485,
      "nll_loss": 1.0349609851837158,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.08096923679113388,
      "rewards/margins": 0.10084228217601776,
      "rewards/rejected": -0.18173828721046448,
      "step": 2610
    },
    {
      "epoch": 0.33843570367499837,
      "grad_norm": 1.5099100866144541,
      "learning_rate": 1.562929330329127e-06,
      "log_odds_chosen": 1.318457007408142,
      "log_odds_ratio": -0.477294921875,
      "logits/chosen": -1.249414086341858,
      "logits/rejected": -1.040429711341858,
      "logps/chosen": -0.846484363079071,
      "logps/rejected": -1.859375,
      "loss": 1.0281,
      "nll_loss": 0.986328125,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.08461914211511612,
      "rewards/margins": 0.101226806640625,
      "rewards/rejected": -0.18581542372703552,
      "step": 2620
    },
    {
      "epoch": 0.3397274430020022,
      "grad_norm": 1.3594384557751003,
      "learning_rate": 1.5599551519340636e-06,
      "log_odds_chosen": 1.3944823741912842,
      "log_odds_ratio": -0.3941406309604645,
      "logits/chosen": -1.3113281726837158,
      "logits/rejected": -1.0810546875,
      "logps/chosen": -0.747851550579071,
      "logps/rejected": -1.773828148841858,
      "loss": 1.0291,
      "nll_loss": 0.961132824420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07470703125,
      "rewards/margins": 0.10255126655101776,
      "rewards/rejected": -0.17724609375,
      "step": 2630
    },
    {
      "epoch": 0.341019182329006,
      "grad_norm": 1.6169140376702476,
      "learning_rate": 1.556997888323046e-06,
      "log_odds_chosen": 1.3738281726837158,
      "log_odds_ratio": -0.40864259004592896,
      "logits/chosen": -1.1798827648162842,
      "logits/rejected": -0.9527343511581421,
      "logps/chosen": -0.710644543170929,
      "logps/rejected": -1.657617211341858,
      "loss": 1.0137,
      "nll_loss": 0.9888671636581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.071044921875,
      "rewards/margins": 0.09481658786535263,
      "rewards/rejected": -0.16582031548023224,
      "step": 2640
    },
    {
      "epoch": 0.3423109216560098,
      "grad_norm": 1.5496554391757391,
      "learning_rate": 1.5540573797716226e-06,
      "log_odds_chosen": 1.3556640148162842,
      "log_odds_ratio": -0.41093748807907104,
      "logits/chosen": -1.2980468273162842,
      "logits/rejected": -1.0771484375,
      "logps/chosen": -0.783398449420929,
      "logps/rejected": -1.751562476158142,
      "loss": 1.0216,
      "nll_loss": 0.9828125238418579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07835693657398224,
      "rewards/margins": 0.09660644829273224,
      "rewards/rejected": -0.17499999701976776,
      "step": 2650
    },
    {
      "epoch": 0.3436026609830136,
      "grad_norm": 1.5257605761637454,
      "learning_rate": 1.5511334686589623e-06,
      "log_odds_chosen": 1.259863257408142,
      "log_odds_ratio": -0.413330078125,
      "logits/chosen": -1.1994140148162842,
      "logits/rejected": -1.051367163658142,
      "logps/chosen": -0.714550793170929,
      "logps/rejected": -1.585546851158142,
      "loss": 1.0243,
      "nll_loss": 1.015527367591858,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07139892876148224,
      "rewards/margins": 0.08719329535961151,
      "rewards/rejected": -0.15866699814796448,
      "step": 2660
    },
    {
      "epoch": 0.34489440031001745,
      "grad_norm": 1.406882352670602,
      "learning_rate": 1.548225999432367e-06,
      "log_odds_chosen": 1.4289062023162842,
      "log_odds_ratio": -0.4266113340854645,
      "logits/chosen": -1.284765601158142,
      "logits/rejected": -1.0205078125,
      "logps/chosen": -0.7904297113418579,
      "logps/rejected": -1.848046898841858,
      "loss": 1.0515,
      "nll_loss": 1.0007812976837158,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07911376655101776,
      "rewards/margins": 0.105712890625,
      "rewards/rejected": -0.1849365234375,
      "step": 2670
    },
    {
      "epoch": 0.3461861396370213,
      "grad_norm": 1.5168007043821992,
      "learning_rate": 1.5453348185725114e-06,
      "log_odds_chosen": 1.562109351158142,
      "log_odds_ratio": -0.3645996153354645,
      "logits/chosen": -1.234960913658142,
      "logits/rejected": -1.0291016101837158,
      "logps/chosen": -0.75390625,
      "logps/rejected": -1.8937499523162842,
      "loss": 1.0265,
      "nll_loss": 0.9947265386581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07539062201976776,
      "rewards/margins": 0.11407470703125,
      "rewards/rejected": -0.189453125,
      "step": 2680
    },
    {
      "epoch": 0.34747787896402504,
      "grad_norm": 1.5393745432766317,
      "learning_rate": 1.542459774559398e-06,
      "log_odds_chosen": 1.302343726158142,
      "log_odds_ratio": -0.46269530057907104,
      "logits/chosen": -1.310546875,
      "logits/rejected": -1.095117211341858,
      "logps/chosen": -0.7939453125,
      "logps/rejected": -1.715234398841858,
      "loss": 1.0303,
      "nll_loss": 1.002539038658142,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07939453423023224,
      "rewards/margins": 0.09210205078125,
      "rewards/rejected": -0.1715087890625,
      "step": 2690
    },
    {
      "epoch": 0.34876961829102887,
      "grad_norm": 1.379322100160873,
      "learning_rate": 1.539600717839002e-06,
      "log_odds_chosen": 1.3251464366912842,
      "log_odds_ratio": -0.43413084745407104,
      "logits/chosen": -1.197265625,
      "logits/rejected": -0.9779297113418579,
      "logps/chosen": -0.760937511920929,
      "logps/rejected": -1.7371094226837158,
      "loss": 1.0159,
      "nll_loss": 0.99609375,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07613525539636612,
      "rewards/margins": 0.0975494384765625,
      "rewards/rejected": -0.17363281548023224,
      "step": 2700
    },
    {
      "epoch": 0.3500613576180327,
      "grad_norm": 2.601522461925953,
      "learning_rate": 1.536757500790597e-06,
      "log_odds_chosen": 1.43408203125,
      "log_odds_ratio": -0.4046386778354645,
      "logits/chosen": -1.3097655773162842,
      "logits/rejected": -1.0666015148162842,
      "logps/chosen": -0.723828136920929,
      "logps/rejected": -1.7900390625,
      "loss": 1.0355,
      "nll_loss": 0.9310547113418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07233886420726776,
      "rewards/margins": 0.10648803412914276,
      "rewards/rejected": -0.17893067002296448,
      "step": 2710
    },
    {
      "epoch": 0.35135309694503647,
      "grad_norm": 1.340594321855894,
      "learning_rate": 1.5339299776947407e-06,
      "log_odds_chosen": 1.3108398914337158,
      "log_odds_ratio": -0.4166015684604645,
      "logits/chosen": -1.176171898841858,
      "logits/rejected": -0.9583984613418579,
      "logps/chosen": -0.7547851800918579,
      "logps/rejected": -1.6886718273162842,
      "loss": 1.01,
      "nll_loss": 1.010156273841858,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07540283352136612,
      "rewards/margins": 0.0933837890625,
      "rewards/rejected": -0.16899414360523224,
      "step": 2720
    },
    {
      "epoch": 0.3526448362720403,
      "grad_norm": 1.6438008709360286,
      "learning_rate": 1.5311180047019054e-06,
      "log_odds_chosen": 1.527929663658142,
      "log_odds_ratio": -0.39677733182907104,
      "logits/chosen": -1.1982421875,
      "logits/rejected": -0.9703124761581421,
      "logps/chosen": -0.7738281488418579,
      "logps/rejected": -1.921875,
      "loss": 1.032,
      "nll_loss": 0.999804675579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07733154296875,
      "rewards/margins": 0.11467895656824112,
      "rewards/rejected": -0.19204100966453552,
      "step": 2730
    },
    {
      "epoch": 0.3539365755990441,
      "grad_norm": 1.8687393440252311,
      "learning_rate": 1.5283214398017402e-06,
      "log_odds_chosen": 1.238867163658142,
      "log_odds_ratio": -0.5,
      "logits/chosen": -1.2541015148162842,
      "logits/rejected": -1.025390625,
      "logps/chosen": -0.802539050579071,
      "logps/rejected": -1.7292969226837158,
      "loss": 1.0277,
      "nll_loss": 1.0478515625,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.08022461086511612,
      "rewards/margins": 0.09252776950597763,
      "rewards/rejected": -0.172607421875,
      "step": 2740
    },
    {
      "epoch": 0.35522831492604795,
      "grad_norm": 1.5147936009635052,
      "learning_rate": 1.5255401427929477e-06,
      "log_odds_chosen": 1.236059546470642,
      "log_odds_ratio": -0.43378907442092896,
      "logits/chosen": -1.211523413658142,
      "logits/rejected": -1.0498046875,
      "logps/chosen": -0.6805664300918579,
      "logps/rejected": -1.523046851158142,
      "loss": 1.0336,
      "nll_loss": 0.922656238079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06805419921875,
      "rewards/margins": 0.08431701362133026,
      "rewards/rejected": -0.1524658203125,
      "step": 2750
    },
    {
      "epoch": 0.3565200542530517,
      "grad_norm": 1.5854956762156527,
      "learning_rate": 1.5227739752537617e-06,
      "log_odds_chosen": 1.560546875,
      "log_odds_ratio": -0.37226563692092896,
      "logits/chosen": -1.178320288658142,
      "logits/rejected": -0.9458984136581421,
      "logps/chosen": -0.732421875,
      "logps/rejected": -1.8601562976837158,
      "loss": 1.0424,
      "nll_loss": 0.9208984375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07319335639476776,
      "rewards/margins": 0.11280517280101776,
      "rewards/rejected": -0.18608398735523224,
      "step": 2760
    },
    {
      "epoch": 0.35781179358005555,
      "grad_norm": 1.4269528294063227,
      "learning_rate": 1.5200228005130127e-06,
      "log_odds_chosen": 1.502539038658142,
      "log_odds_ratio": -0.3880859315395355,
      "logits/chosen": -1.2521483898162842,
      "logits/rejected": -1.0333983898162842,
      "logps/chosen": -0.8287109136581421,
      "logps/rejected": -1.955468773841858,
      "loss": 1.0055,
      "nll_loss": 1.0158202648162842,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.08286132663488388,
      "rewards/margins": 0.11251220852136612,
      "rewards/rejected": -0.19560547173023224,
      "step": 2770
    },
    {
      "epoch": 0.3591035329070594,
      "grad_norm": 1.532521391356933,
      "learning_rate": 1.5172864836217631e-06,
      "log_odds_chosen": 1.211523413658142,
      "log_odds_ratio": -0.46875,
      "logits/chosen": -1.1306641101837158,
      "logits/rejected": -0.9546874761581421,
      "logps/chosen": -0.787304699420929,
      "logps/rejected": -1.6687500476837158,
      "loss": 1.025,
      "nll_loss": 0.9818359613418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0787353515625,
      "rewards/margins": 0.08814086765050888,
      "rewards/rejected": -0.16689452528953552,
      "step": 2780
    },
    {
      "epoch": 0.36039527223406315,
      "grad_norm": 1.5506901976608174,
      "learning_rate": 1.514564891325506e-06,
      "log_odds_chosen": 1.505859375,
      "log_odds_ratio": -0.39824217557907104,
      "logits/chosen": -1.228124976158142,
      "logits/rejected": -0.991015613079071,
      "logps/chosen": -0.740429699420929,
      "logps/rejected": -1.8113281726837158,
      "loss": 1.0235,
      "nll_loss": 0.9931640625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.07403564453125,
      "rewards/margins": 0.10703124850988388,
      "rewards/rejected": -0.1810302734375,
      "step": 2790
    },
    {
      "epoch": 0.361687011561067,
      "grad_norm": 1.3388784430979392,
      "learning_rate": 1.5118578920369086e-06,
      "log_odds_chosen": 1.507104516029358,
      "log_odds_ratio": -0.398681640625,
      "logits/chosen": -1.1941406726837158,
      "logits/rejected": -0.9976562261581421,
      "logps/chosen": -0.748828113079071,
      "logps/rejected": -1.831445336341858,
      "loss": 1.015,
      "nll_loss": 0.9349609613418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07479248195886612,
      "rewards/margins": 0.10837402194738388,
      "rewards/rejected": -0.18308106064796448,
      "step": 2800
    },
    {
      "epoch": 0.3629787508880708,
      "grad_norm": 1.6617358254763972,
      "learning_rate": 1.5091653558090898e-06,
      "log_odds_chosen": 1.4683105945587158,
      "log_odds_ratio": -0.4205566346645355,
      "logits/chosen": -1.284765601158142,
      "logits/rejected": -1.062890648841858,
      "logps/chosen": -0.7701171636581421,
      "logps/rejected": -1.829492211341858,
      "loss": 1.0107,
      "nll_loss": 0.9599609375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07709960639476776,
      "rewards/margins": 0.10589905083179474,
      "rewards/rejected": -0.182861328125,
      "step": 2810
    },
    {
      "epoch": 0.36427049021507457,
      "grad_norm": 1.448693430057611,
      "learning_rate": 1.506487154309419e-06,
      "log_odds_chosen": 1.3786132335662842,
      "log_odds_ratio": -0.3921875059604645,
      "logits/chosen": -1.16015625,
      "logits/rejected": -0.979296863079071,
      "logps/chosen": -0.687304675579071,
      "logps/rejected": -1.6398437023162842,
      "loss": 1.0153,
      "nll_loss": 0.91796875,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.0687255859375,
      "rewards/margins": 0.09523925930261612,
      "rewards/rejected": -0.16408690810203552,
      "step": 2820
    },
    {
      "epoch": 0.3655622295420784,
      "grad_norm": 1.4163825937690617,
      "learning_rate": 1.5038231607938247e-06,
      "log_odds_chosen": 1.44873046875,
      "log_odds_ratio": -0.4397949278354645,
      "logits/chosen": -1.1611328125,
      "logits/rejected": -0.991992175579071,
      "logps/chosen": -0.741992175579071,
      "logps/rejected": -1.742578148841858,
      "loss": 1.0058,
      "nll_loss": 0.9515625238418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07429198920726776,
      "rewards/margins": 0.100006103515625,
      "rewards/rejected": -0.17434081435203552,
      "step": 2830
    },
    {
      "epoch": 0.3668539688690822,
      "grad_norm": 1.773613483592378,
      "learning_rate": 1.501173250081603e-06,
      "log_odds_chosen": 1.4287109375,
      "log_odds_ratio": -0.3788085877895355,
      "logits/chosen": -1.2431640625,
      "logits/rejected": -1.025781273841858,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -1.738867163658142,
      "loss": 1.0283,
      "nll_loss": 1.0115234851837158,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07065429538488388,
      "rewards/margins": 0.10333709418773651,
      "rewards/rejected": -0.17402343451976776,
      "step": 2840
    },
    {
      "epoch": 0.36814570819608605,
      "grad_norm": 1.5918987751394618,
      "learning_rate": 1.4985372985307103e-06,
      "log_odds_chosen": 1.2820312976837158,
      "log_odds_ratio": -0.4369140565395355,
      "logits/chosen": -1.2033202648162842,
      "logits/rejected": -1.0458984375,
      "logps/chosen": -0.7724609375,
      "logps/rejected": -1.717187523841858,
      "loss": 1.0158,
      "nll_loss": 0.9458984136581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07724609225988388,
      "rewards/margins": 0.094512939453125,
      "rewards/rejected": -0.1717529296875,
      "step": 2850
    },
    {
      "epoch": 0.3694374475230898,
      "grad_norm": 1.5245879752168185,
      "learning_rate": 1.4959151840135313e-06,
      "log_odds_chosen": 1.411474585533142,
      "log_odds_ratio": -0.4097656309604645,
      "logits/chosen": -1.1181640625,
      "logits/rejected": -0.9574218988418579,
      "logps/chosen": -0.7955077886581421,
      "logps/rejected": -1.8371093273162842,
      "loss": 1.0241,
      "nll_loss": 1.0597655773162842,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07957763969898224,
      "rewards/margins": 0.104095458984375,
      "rewards/rejected": -0.18366698920726776,
      "step": 2860
    },
    {
      "epoch": 0.37072918685009365,
      "grad_norm": 1.3437615394154236,
      "learning_rate": 1.4933067858931148e-06,
      "log_odds_chosen": 1.2717773914337158,
      "log_odds_ratio": -0.4296875,
      "logits/chosen": -1.278906226158142,
      "logits/rejected": -1.026757836341858,
      "logps/chosen": -0.693164050579071,
      "logps/rejected": -1.5900390148162842,
      "loss": 0.9985,
      "nll_loss": 0.9185546636581421,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06931152194738388,
      "rewards/margins": 0.08977966010570526,
      "rewards/rejected": -0.15895995497703552,
      "step": 2870
    },
    {
      "epoch": 0.3720209261770975,
      "grad_norm": 1.5114703261404312,
      "learning_rate": 1.4907119849998597e-06,
      "log_odds_chosen": 1.190637230873108,
      "log_odds_ratio": -0.48320311307907104,
      "logits/chosen": -1.271875023841858,
      "logits/rejected": -1.0808594226837158,
      "logps/chosen": -0.786914050579071,
      "logps/rejected": -1.677148461341858,
      "loss": 1.021,
      "nll_loss": 1.005273461341858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07872314751148224,
      "rewards/margins": 0.08900909125804901,
      "rewards/rejected": -0.16767577826976776,
      "step": 2880
    },
    {
      "epoch": 0.37331266550410125,
      "grad_norm": 1.587789549253314,
      "learning_rate": 1.488130663608649e-06,
      "log_odds_chosen": 1.495703101158142,
      "log_odds_ratio": -0.42597657442092896,
      "logits/chosen": -1.232812523841858,
      "logits/rejected": -1.0099608898162842,
      "logps/chosen": -0.717578113079071,
      "logps/rejected": -1.842187523841858,
      "loss": 1.0279,
      "nll_loss": 0.957226574420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07175292819738388,
      "rewards/margins": 0.11240234225988388,
      "rewards/rejected": -0.18413086235523224,
      "step": 2890
    },
    {
      "epoch": 0.3746044048311051,
      "grad_norm": 1.5928047512354997,
      "learning_rate": 1.4855627054164149e-06,
      "log_odds_chosen": 1.270898461341858,
      "log_odds_ratio": -0.4200683534145355,
      "logits/chosen": -1.3156249523162842,
      "logits/rejected": -1.067968726158142,
      "logps/chosen": -0.8037109375,
      "logps/rejected": -1.708593726158142,
      "loss": 1.0292,
      "nll_loss": 1.0212891101837158,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.08037109673023224,
      "rewards/margins": 0.09041748195886612,
      "rewards/rejected": -0.17087402939796448,
      "step": 2900
    },
    {
      "epoch": 0.3758961441581089,
      "grad_norm": 1.8110139955924969,
      "learning_rate": 1.4830079955201294e-06,
      "log_odds_chosen": 1.489843726158142,
      "log_odds_ratio": -0.39375001192092896,
      "logits/chosen": -1.272070288658142,
      "logits/rejected": -1.0476562976837158,
      "logps/chosen": -0.72998046875,
      "logps/rejected": -1.777929663658142,
      "loss": 1.0086,
      "nll_loss": 0.8931640386581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07294921576976776,
      "rewards/margins": 0.1048583984375,
      "rewards/rejected": -0.17795410752296448,
      "step": 2910
    },
    {
      "epoch": 0.37718788348511273,
      "grad_norm": 1.668166996320744,
      "learning_rate": 1.4804664203952103e-06,
      "log_odds_chosen": 1.6106445789337158,
      "log_odds_ratio": -0.37373048067092896,
      "logits/chosen": -1.293554663658142,
      "logits/rejected": -1.021093726158142,
      "logps/chosen": -0.7568359375,
      "logps/rejected": -1.9578125476837158,
      "loss": 1.0023,
      "nll_loss": 0.966992199420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07572021335363388,
      "rewards/margins": 0.12015380710363388,
      "rewards/rejected": -0.19584961235523224,
      "step": 2920
    },
    {
      "epoch": 0.3784796228121165,
      "grad_norm": 1.6791351283783518,
      "learning_rate": 1.4779378678743327e-06,
      "log_odds_chosen": 1.41796875,
      "log_odds_ratio": -0.4283691346645355,
      "logits/chosen": -1.2785155773162842,
      "logits/rejected": -1.065820336341858,
      "logps/chosen": -0.7250000238418579,
      "logps/rejected": -1.7253906726837158,
      "loss": 1.0158,
      "nll_loss": 0.9916015863418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07255859673023224,
      "rewards/margins": 0.099853515625,
      "rewards/rejected": -0.17243652045726776,
      "step": 2930
    },
    {
      "epoch": 0.3797713621391203,
      "grad_norm": 1.8648969342795538,
      "learning_rate": 1.4754222271266348e-06,
      "log_odds_chosen": 1.320898413658142,
      "log_odds_ratio": -0.3905029296875,
      "logits/chosen": -1.270898461341858,
      "logits/rejected": -1.0224609375,
      "logps/chosen": -0.7982422113418579,
      "logps/rejected": -1.7566406726837158,
      "loss": 1.0473,
      "nll_loss": 0.997265636920929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07978515326976776,
      "rewards/margins": 0.09592285007238388,
      "rewards/rejected": -0.17568358778953552,
      "step": 2940
    },
    {
      "epoch": 0.38106310146612415,
      "grad_norm": 1.761201927303849,
      "learning_rate": 1.4729193886373175e-06,
      "log_odds_chosen": 1.363671898841858,
      "log_odds_ratio": -0.38330078125,
      "logits/chosen": -1.2166016101837158,
      "logits/rejected": -1.0730469226837158,
      "logps/chosen": -0.75146484375,
      "logps/rejected": -1.734765648841858,
      "loss": 0.9995,
      "nll_loss": 0.9400390386581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07514648139476776,
      "rewards/margins": 0.09832763671875,
      "rewards/rejected": -0.17338867485523224,
      "step": 2950
    },
    {
      "epoch": 0.3823548407931279,
      "grad_norm": 1.3949921314428897,
      "learning_rate": 1.4704292441876156e-06,
      "log_odds_chosen": 1.377343773841858,
      "log_odds_ratio": -0.43623048067092896,
      "logits/chosen": -1.1896483898162842,
      "logits/rejected": -1.06640625,
      "logps/chosen": -0.7225586175918579,
      "logps/rejected": -1.7541015148162842,
      "loss": 1.0266,
      "nll_loss": 0.9585937261581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07225342094898224,
      "rewards/margins": 0.10338135063648224,
      "rewards/rejected": -0.17551270127296448,
      "step": 2960
    },
    {
      "epoch": 0.38364658012013175,
      "grad_norm": 1.8030693308145072,
      "learning_rate": 1.4679516868351474e-06,
      "log_odds_chosen": 1.4705078601837158,
      "log_odds_ratio": -0.3963378965854645,
      "logits/chosen": -1.196679711341858,
      "logits/rejected": -1.0164062976837158,
      "logps/chosen": -0.740429699420929,
      "logps/rejected": -1.8025391101837158,
      "loss": 1.0231,
      "nll_loss": 0.937304675579071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.0740966796875,
      "rewards/margins": 0.10624389350414276,
      "rewards/rejected": -0.18039551377296448,
      "step": 2970
    },
    {
      "epoch": 0.3849383194471356,
      "grad_norm": 1.4769906211012922,
      "learning_rate": 1.4654866108946234e-06,
      "log_odds_chosen": 1.230078101158142,
      "log_odds_ratio": -0.443603515625,
      "logits/chosen": -1.2890625,
      "logits/rejected": -1.11328125,
      "logps/chosen": -0.7509765625,
      "logps/rejected": -1.618749976158142,
      "loss": 1.0199,
      "nll_loss": 0.913281261920929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07510986179113388,
      "rewards/margins": 0.08683471381664276,
      "rewards/rejected": -0.16184082627296448,
      "step": 2980
    },
    {
      "epoch": 0.38623005877413935,
      "grad_norm": 1.6965568036298624,
      "learning_rate": 1.4630339119189101e-06,
      "log_odds_chosen": 1.381445288658142,
      "log_odds_ratio": -0.4249023497104645,
      "logits/chosen": -1.1982421875,
      "logits/rejected": -1.0109374523162842,
      "logps/chosen": -0.7247070074081421,
      "logps/rejected": -1.727929711341858,
      "loss": 0.9742,
      "nll_loss": 0.9462890625,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07252196967601776,
      "rewards/margins": 0.10051269829273224,
      "rewards/rejected": -0.17304687201976776,
      "step": 2990
    },
    {
      "epoch": 0.3875217981011432,
      "grad_norm": 1.9339540346435529,
      "learning_rate": 1.4605934866804429e-06,
      "log_odds_chosen": 1.356201171875,
      "log_odds_ratio": -0.40576171875,
      "logits/chosen": -1.3712890148162842,
      "logits/rejected": -1.155664086341858,
      "logps/chosen": -0.681835949420929,
      "logps/rejected": -1.6511719226837158,
      "loss": 1.0077,
      "nll_loss": 0.918749988079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06821288913488388,
      "rewards/margins": 0.09696350246667862,
      "rewards/rejected": -0.16520996391773224,
      "step": 3000
    },
    {
      "epoch": 0.388813537428147,
      "grad_norm": 1.7533317818193308,
      "learning_rate": 1.4581652331529784e-06,
      "log_odds_chosen": 1.305761694908142,
      "log_odds_ratio": -0.42333984375,
      "logits/chosen": -1.1140625476837158,
      "logits/rejected": -0.9781249761581421,
      "logps/chosen": -0.7259765863418579,
      "logps/rejected": -1.657812476158142,
      "loss": 1.006,
      "nll_loss": 1.0107421875,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07260742038488388,
      "rewards/margins": 0.09302368015050888,
      "rewards/rejected": -0.16555175185203552,
      "step": 3010
    },
    {
      "epoch": 0.39010527675515083,
      "grad_norm": 1.468852927176469,
      "learning_rate": 1.4557490504936778e-06,
      "log_odds_chosen": 1.431054711341858,
      "log_odds_ratio": -0.39836424589157104,
      "logits/chosen": -1.2599608898162842,
      "logits/rejected": -1.068750023841858,
      "logps/chosen": -0.801953136920929,
      "logps/rejected": -1.8585937023162842,
      "loss": 1.0284,
      "nll_loss": 0.974609375,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.08026123046875,
      "rewards/margins": 0.10570068657398224,
      "rewards/rejected": -0.18588867783546448,
      "step": 3020
    },
    {
      "epoch": 0.3913970160821546,
      "grad_norm": 1.5365042199375425,
      "learning_rate": 1.453344839025519e-06,
      "log_odds_chosen": 1.542871117591858,
      "log_odds_ratio": -0.3916015625,
      "logits/chosen": -1.201171875,
      "logits/rejected": -1.0085937976837158,
      "logps/chosen": -0.720507800579071,
      "logps/rejected": -1.81640625,
      "loss": 1.0042,
      "nll_loss": 0.9208008050918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07208251953125,
      "rewards/margins": 0.10950622707605362,
      "rewards/rejected": -0.18161621689796448,
      "step": 3030
    },
    {
      "epoch": 0.39268875540915843,
      "grad_norm": 1.7801592044391912,
      "learning_rate": 1.4509525002200234e-06,
      "log_odds_chosen": 1.3759765625,
      "log_odds_ratio": -0.41865235567092896,
      "logits/chosen": -1.2156250476837158,
      "logits/rejected": -1.0615234375,
      "logps/chosen": -0.736132800579071,
      "logps/rejected": -1.728515625,
      "loss": 1.0142,
      "nll_loss": 0.9937499761581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07357177883386612,
      "rewards/margins": 0.09931640326976776,
      "rewards/rejected": -0.17287597060203552,
      "step": 3040
    },
    {
      "epoch": 0.39398049473616226,
      "grad_norm": 1.6593237307277826,
      "learning_rate": 1.4485719366802965e-06,
      "log_odds_chosen": 1.5216796398162842,
      "log_odds_ratio": -0.378173828125,
      "logits/chosen": -1.2998046875,
      "logits/rejected": -1.0529296398162842,
      "logps/chosen": -0.7447265386581421,
      "logps/rejected": -1.880859375,
      "loss": 0.9992,
      "nll_loss": 0.9482421875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07445068657398224,
      "rewards/margins": 0.1136474609375,
      "rewards/rejected": -0.188232421875,
      "step": 3050
    },
    {
      "epoch": 0.39527223406316603,
      "grad_norm": 1.5433039740808436,
      "learning_rate": 1.4462030521243742e-06,
      "log_odds_chosen": 1.35107421875,
      "log_odds_ratio": -0.4595703184604645,
      "logits/chosen": -1.2443358898162842,
      "logits/rejected": -1.0556640625,
      "logps/chosen": -0.7548828125,
      "logps/rejected": -1.7546875476837158,
      "loss": 1.0066,
      "nll_loss": 0.930859386920929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07548828423023224,
      "rewards/margins": 0.09993286430835724,
      "rewards/rejected": -0.17551270127296448,
      "step": 3060
    },
    {
      "epoch": 0.39656397339016985,
      "grad_norm": 30.646687219608673,
      "learning_rate": 1.443845751368867e-06,
      "log_odds_chosen": 1.623046875,
      "log_odds_ratio": -0.38652342557907104,
      "logits/chosen": -1.203710913658142,
      "logits/rejected": -0.9990234375,
      "logps/chosen": -0.7427734136581421,
      "logps/rejected": -1.904687523841858,
      "loss": 1.0121,
      "nll_loss": 1.058007836341858,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07425536960363388,
      "rewards/margins": 0.11622314155101776,
      "rewards/rejected": -0.19045409560203552,
      "step": 3070
    },
    {
      "epoch": 0.3978557127171737,
      "grad_norm": 3.8635557864130523,
      "learning_rate": 1.4414999403128943e-06,
      "log_odds_chosen": 1.771875023841858,
      "log_odds_ratio": -0.37373048067092896,
      "logits/chosen": -1.2492187023162842,
      "logits/rejected": -0.983593761920929,
      "logps/chosen": -0.7616211175918579,
      "logps/rejected": -2.075390577316284,
      "loss": 1.0185,
      "nll_loss": 0.9644531011581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.07620849460363388,
      "rewards/margins": 0.13140869140625,
      "rewards/rejected": -0.20766600966453552,
      "step": 3080
    },
    {
      "epoch": 0.3991474520441775,
      "grad_norm": 1.5740548084697006,
      "learning_rate": 1.439165525922309e-06,
      "log_odds_chosen": 1.757421851158142,
      "log_odds_ratio": -0.33479005098342896,
      "logits/chosen": -1.242773413658142,
      "logits/rejected": -1.0138671398162842,
      "logps/chosen": -0.7007812261581421,
      "logps/rejected": -2.0078125,
      "loss": 1.0081,
      "nll_loss": 0.940234363079071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07001952826976776,
      "rewards/margins": 0.13093261420726776,
      "rewards/rejected": -0.200927734375,
      "step": 3090
    },
    {
      "epoch": 0.4004391913711813,
      "grad_norm": 1.4410177225831737,
      "learning_rate": 1.4368424162141992e-06,
      "log_odds_chosen": 1.442773461341858,
      "log_odds_ratio": -0.4041503965854645,
      "logits/chosen": -1.208984375,
      "logits/rejected": -1.026953101158142,
      "logps/chosen": -0.7623046636581421,
      "logps/rejected": -1.851953148841858,
      "loss": 0.9811,
      "nll_loss": 0.9560546875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07622070610523224,
      "rewards/margins": 0.10897216945886612,
      "rewards/rejected": -0.18522949516773224,
      "step": 3100
    },
    {
      "epoch": 0.4017309306981851,
      "grad_norm": 1.600319707869519,
      "learning_rate": 1.434530520241665e-06,
      "log_odds_chosen": 1.4052245616912842,
      "log_odds_ratio": -0.42792969942092896,
      "logits/chosen": -1.2490234375,
      "logits/rejected": -1.0236327648162842,
      "logps/chosen": -0.727734386920929,
      "logps/rejected": -1.755468726158142,
      "loss": 0.9927,
      "nll_loss": 0.9751952886581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07282714545726776,
      "rewards/margins": 0.1026611328125,
      "rewards/rejected": -0.17546387016773224,
      "step": 3110
    },
    {
      "epoch": 0.40302267002518893,
      "grad_norm": 1.997515048461857,
      "learning_rate": 1.4322297480788657e-06,
      "log_odds_chosen": 1.597802758216858,
      "log_odds_ratio": -0.38432615995407104,
      "logits/chosen": -1.3214843273162842,
      "logits/rejected": -1.0304687023162842,
      "logps/chosen": -0.730273425579071,
      "logps/rejected": -1.93359375,
      "loss": 0.9899,
      "nll_loss": 0.9505859613418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07308349758386612,
      "rewards/margins": 0.12014313042163849,
      "rewards/rejected": -0.19338378310203552,
      "step": 3120
    },
    {
      "epoch": 0.4043144093521927,
      "grad_norm": 1.7222233813060632,
      "learning_rate": 1.4299400108063247e-06,
      "log_odds_chosen": 1.6300780773162842,
      "log_odds_ratio": -0.3740234375,
      "logits/chosen": -1.316015601158142,
      "logits/rejected": -1.110937476158142,
      "logps/chosen": -0.770703136920929,
      "logps/rejected": -1.9806640148162842,
      "loss": 0.9945,
      "nll_loss": 0.906445324420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07701416313648224,
      "rewards/margins": 0.12120361626148224,
      "rewards/rejected": -0.19829101860523224,
      "step": 3130
    },
    {
      "epoch": 0.40560614867919653,
      "grad_norm": 1.4898834138623338,
      "learning_rate": 1.4276612204964992e-06,
      "log_odds_chosen": 1.4451172351837158,
      "log_odds_ratio": -0.401123046875,
      "logits/chosen": -1.208398461341858,
      "logits/rejected": -1.021484375,
      "logps/chosen": -0.7417968511581421,
      "logps/rejected": -1.78515625,
      "loss": 0.998,
      "nll_loss": 0.948046863079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07415771484375,
      "rewards/margins": 0.10443115234375,
      "rewards/rejected": -0.17854003608226776,
      "step": 3140
    },
    {
      "epoch": 0.40689788800620036,
      "grad_norm": 1.8764285052879957,
      "learning_rate": 1.4253932901995967e-06,
      "log_odds_chosen": 1.435937523841858,
      "log_odds_ratio": -0.3905273377895355,
      "logits/chosen": -1.215429663658142,
      "logits/rejected": -0.9873046875,
      "logps/chosen": -0.738476574420929,
      "logps/rejected": -1.7820312976837158,
      "loss": 1.0202,
      "nll_loss": 0.956835925579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07386474311351776,
      "rewards/margins": 0.10433349758386612,
      "rewards/rejected": -0.17812499403953552,
      "step": 3150
    },
    {
      "epoch": 0.4081896273332042,
      "grad_norm": 1.4844998195761614,
      "learning_rate": 1.42313613392964e-06,
      "log_odds_chosen": 1.8781249523162842,
      "log_odds_ratio": -0.33305662870407104,
      "logits/chosen": -1.228124976158142,
      "logits/rejected": -1.0353515148162842,
      "logps/chosen": -0.690234363079071,
      "logps/rejected": -2.1039061546325684,
      "loss": 0.9789,
      "nll_loss": 0.920703113079071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06904296576976776,
      "rewards/margins": 0.14133301377296448,
      "rewards/rejected": -0.21044921875,
      "step": 3160
    },
    {
      "epoch": 0.40948136666020796,
      "grad_norm": 1.5384722571595832,
      "learning_rate": 1.4208896666507756e-06,
      "log_odds_chosen": 1.4221680164337158,
      "log_odds_ratio": -0.3759765625,
      "logits/chosen": -1.146875023841858,
      "logits/rejected": -0.9697265625,
      "logps/chosen": -0.7386718988418579,
      "logps/rejected": -1.7550780773162842,
      "loss": 0.9971,
      "nll_loss": 0.936718761920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.0738525390625,
      "rewards/margins": 0.10155639797449112,
      "rewards/rejected": -0.17543944716453552,
      "step": 3170
    },
    {
      "epoch": 0.4107731059872118,
      "grad_norm": 1.8645588867089167,
      "learning_rate": 1.4186538042638173e-06,
      "log_odds_chosen": 1.452539086341858,
      "log_odds_ratio": -0.3577636778354645,
      "logits/chosen": -1.2380859851837158,
      "logits/rejected": -0.9916015863418579,
      "logps/chosen": -0.760546863079071,
      "logps/rejected": -1.8113281726837158,
      "loss": 1.0138,
      "nll_loss": 0.9955078363418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07600097358226776,
      "rewards/margins": 0.10518798977136612,
      "rewards/rejected": -0.18125000596046448,
      "step": 3180
    },
    {
      "epoch": 0.4120648453142156,
      "grad_norm": 2.013911821697341,
      "learning_rate": 1.416428463593022e-06,
      "log_odds_chosen": 1.6748046875,
      "log_odds_ratio": -0.37687987089157104,
      "logits/chosen": -1.255273461341858,
      "logits/rejected": -1.0324218273162842,
      "logps/chosen": -0.70458984375,
      "logps/rejected": -1.9296875,
      "loss": 1.0041,
      "nll_loss": 0.9306640625,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.07039795070886612,
      "rewards/margins": 0.12253417819738388,
      "rewards/rejected": -0.19296875596046448,
      "step": 3190
    },
    {
      "epoch": 0.4133565846412194,
      "grad_norm": 1.7711389963585158,
      "learning_rate": 1.414213562373095e-06,
      "log_odds_chosen": 1.500585913658142,
      "log_odds_ratio": -0.40410155057907104,
      "logits/chosen": -1.3330078125,
      "logits/rejected": -1.0400390625,
      "logps/chosen": -0.7689453363418579,
      "logps/rejected": -1.8917968273162842,
      "loss": 1.0133,
      "nll_loss": 0.986132800579071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07685546576976776,
      "rewards/margins": 0.11232604831457138,
      "rewards/rejected": -0.18916015326976776,
      "step": 3200
    },
    {
      "epoch": 0.4146483239682232,
      "grad_norm": 1.4487056262222646,
      "learning_rate": 1.4120090192364154e-06,
      "log_odds_chosen": 1.649999976158142,
      "log_odds_ratio": -0.35148924589157104,
      "logits/chosen": -1.1847655773162842,
      "logits/rejected": -0.922656238079071,
      "logps/chosen": -0.73046875,
      "logps/rejected": -1.91015625,
      "loss": 0.9977,
      "nll_loss": 0.9173828363418579,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.07308349758386612,
      "rewards/margins": 0.11784668266773224,
      "rewards/rejected": -0.19096679985523224,
      "step": 3210
    },
    {
      "epoch": 0.41594006329522704,
      "grad_norm": 1.4071233375340308,
      "learning_rate": 1.4098147537004828e-06,
      "log_odds_chosen": 1.5021483898162842,
      "log_odds_ratio": -0.37761229276657104,
      "logits/chosen": -1.232031226158142,
      "logits/rejected": -1.021093726158142,
      "logps/chosen": -0.72607421875,
      "logps/rejected": -1.828515648841858,
      "loss": 1.0145,
      "nll_loss": 0.985156238079071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07259521633386612,
      "rewards/margins": 0.11009521782398224,
      "rewards/rejected": -0.18288573622703552,
      "step": 3220
    },
    {
      "epoch": 0.4172318026222308,
      "grad_norm": 2.536392549878833,
      "learning_rate": 1.4076306861555735e-06,
      "log_odds_chosen": 1.250952124595642,
      "log_odds_ratio": -0.43413084745407104,
      "logits/chosen": -1.279882788658142,
      "logits/rejected": -1.123437523841858,
      "logps/chosen": -0.724414050579071,
      "logps/rejected": -1.6007812023162842,
      "loss": 0.9747,
      "nll_loss": 0.8988281488418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07242431491613388,
      "rewards/margins": 0.08768920600414276,
      "rewards/rejected": -0.16008301079273224,
      "step": 3230
    },
    {
      "epoch": 0.41852354194923463,
      "grad_norm": 1.5315128990542002,
      "learning_rate": 1.405456737852613e-06,
      "log_odds_chosen": 1.464257836341858,
      "log_odds_ratio": -0.43671876192092896,
      "logits/chosen": -1.2443358898162842,
      "logits/rejected": -1.0705077648162842,
      "logps/chosen": -0.732617199420929,
      "logps/rejected": -1.8230469226837158,
      "loss": 0.992,
      "nll_loss": 0.9107421636581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07330322265625,
      "rewards/margins": 0.10887451469898224,
      "rewards/rejected": -0.18217773735523224,
      "step": 3240
    },
    {
      "epoch": 0.41981528127623846,
      "grad_norm": 2.2011303414892787,
      "learning_rate": 1.4032928308912468e-06,
      "log_odds_chosen": 1.4099609851837158,
      "log_odds_ratio": -0.3797851502895355,
      "logits/chosen": -1.29296875,
      "logits/rejected": -1.0974609851837158,
      "logps/chosen": -0.683789074420929,
      "logps/rejected": -1.6769530773162842,
      "loss": 1.0125,
      "nll_loss": 0.949023425579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06838379055261612,
      "rewards/margins": 0.09927978366613388,
      "rewards/rejected": -0.16757813096046448,
      "step": 3250
    },
    {
      "epoch": 0.4211070206032423,
      "grad_norm": 1.9007130940647943,
      "learning_rate": 1.4011388882081175e-06,
      "log_odds_chosen": 1.462304711341858,
      "log_odds_ratio": -0.4241699278354645,
      "logits/chosen": -1.2666015625,
      "logits/rejected": -1.049218773841858,
      "logps/chosen": -0.770312488079071,
      "logps/rejected": -1.822265625,
      "loss": 1.0176,
      "nll_loss": 1.009179711341858,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07703857123851776,
      "rewards/margins": 0.10509643703699112,
      "rewards/rejected": -0.18205566704273224,
      "step": 3260
    },
    {
      "epoch": 0.42239875993024606,
      "grad_norm": 1.5002841689366821,
      "learning_rate": 1.3989948335653378e-06,
      "log_odds_chosen": 1.3546874523162842,
      "log_odds_ratio": -0.41289061307907104,
      "logits/chosen": -1.197656273841858,
      "logits/rejected": -0.9867187738418579,
      "logps/chosen": -0.752734363079071,
      "logps/rejected": -1.7400391101837158,
      "loss": 1.0029,
      "nll_loss": 0.987500011920929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07526855170726776,
      "rewards/margins": 0.098602294921875,
      "rewards/rejected": -0.17390136420726776,
      "step": 3270
    },
    {
      "epoch": 0.4236904992572499,
      "grad_norm": 1.7112780257285214,
      "learning_rate": 1.3968605915391564e-06,
      "log_odds_chosen": 1.649804711341858,
      "log_odds_ratio": -0.3883300721645355,
      "logits/chosen": -1.267968773841858,
      "logits/rejected": -1.032617211341858,
      "logps/chosen": -0.7740234136581421,
      "logps/rejected": -2.006640672683716,
      "loss": 0.9912,
      "nll_loss": 1.0107421875,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07745361328125,
      "rewards/margins": 0.12327881157398224,
      "rewards/rejected": -0.20048828423023224,
      "step": 3280
    },
    {
      "epoch": 0.4249822385842537,
      "grad_norm": 1.4277594122843376,
      "learning_rate": 1.3947360875088132e-06,
      "log_odds_chosen": 1.554296851158142,
      "log_odds_ratio": -0.37785643339157104,
      "logits/chosen": -1.3087890148162842,
      "logits/rejected": -1.046875,
      "logps/chosen": -0.743945300579071,
      "logps/rejected": -1.8914062976837158,
      "loss": 0.9939,
      "nll_loss": 0.8890625238418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07438965141773224,
      "rewards/margins": 0.11470947414636612,
      "rewards/rejected": -0.18901367485523224,
      "step": 3290
    },
    {
      "epoch": 0.4262739779112575,
      "grad_norm": 2.037220577223024,
      "learning_rate": 1.3926212476455828e-06,
      "log_odds_chosen": 1.112695336341858,
      "log_odds_ratio": -0.504199206829071,
      "logits/chosen": -1.317968726158142,
      "logits/rejected": -1.1056640148162842,
      "logps/chosen": -0.7802734375,
      "logps/rejected": -1.603124976158142,
      "loss": 0.9924,
      "nll_loss": 0.9869140386581421,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07801513373851776,
      "rewards/margins": 0.08224181830883026,
      "rewards/rejected": -0.16020508110523224,
      "step": 3300
    },
    {
      "epoch": 0.4275657172382613,
      "grad_norm": 1.4860013380904586,
      "learning_rate": 1.3905159989019964e-06,
      "log_odds_chosen": 1.600195288658142,
      "log_odds_ratio": -0.3551269471645355,
      "logits/chosen": -1.240234375,
      "logits/rejected": -1.0125000476837158,
      "logps/chosen": -0.7679687738418579,
      "logps/rejected": -1.9558594226837158,
      "loss": 0.9807,
      "nll_loss": 0.955273449420929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0767822265625,
      "rewards/margins": 0.11893310397863388,
      "rewards/rejected": -0.19572754204273224,
      "step": 3310
    },
    {
      "epoch": 0.42885745656526514,
      "grad_norm": 1.4769956768968087,
      "learning_rate": 1.3884202690012465e-06,
      "log_odds_chosen": 1.264794945716858,
      "log_odds_ratio": -0.45146483182907104,
      "logits/chosen": -1.261328101158142,
      "logits/rejected": -1.101171851158142,
      "logps/chosen": -0.744433581829071,
      "logps/rejected": -1.663671851158142,
      "loss": 1.0005,
      "nll_loss": 0.939257800579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07441405951976776,
      "rewards/margins": 0.09189758449792862,
      "rewards/rejected": -0.16635742783546448,
      "step": 3320
    },
    {
      "epoch": 0.43014919589226897,
      "grad_norm": 1.9868726645866255,
      "learning_rate": 1.3863339864267636e-06,
      "log_odds_chosen": 1.4054687023162842,
      "log_odds_ratio": -0.4319824278354645,
      "logits/chosen": -1.2042968273162842,
      "logits/rejected": -1.0361328125,
      "logps/chosen": -0.721484363079071,
      "logps/rejected": -1.731054663658142,
      "loss": 1.0059,
      "nll_loss": 0.958203136920929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0721435546875,
      "rewards/margins": 0.10098876804113388,
      "rewards/rejected": -0.1732177734375,
      "step": 3330
    },
    {
      "epoch": 0.43144093521927274,
      "grad_norm": 1.455663891143005,
      "learning_rate": 1.3842570804119655e-06,
      "log_odds_chosen": 1.5056641101837158,
      "log_odds_ratio": -0.3729003965854645,
      "logits/chosen": -1.259765625,
      "logits/rejected": -1.020898461341858,
      "logps/chosen": -0.691699206829071,
      "logps/rejected": -1.7431640625,
      "loss": 0.9764,
      "nll_loss": 0.8968750238418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06910400092601776,
      "rewards/margins": 0.10523681342601776,
      "rewards/rejected": -0.17431640625,
      "step": 3340
    },
    {
      "epoch": 0.43273267454627656,
      "grad_norm": 1.867354991127315,
      "learning_rate": 1.3821894809301763e-06,
      "log_odds_chosen": 1.60107421875,
      "log_odds_ratio": -0.35107421875,
      "logits/chosen": -1.3634765148162842,
      "logits/rejected": -1.0798828601837158,
      "logps/chosen": -0.756640613079071,
      "logps/rejected": -1.9464843273162842,
      "loss": 0.9949,
      "nll_loss": 0.896679699420929,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.07564697414636612,
      "rewards/margins": 0.11897583305835724,
      "rewards/rejected": -0.19462890923023224,
      "step": 3350
    },
    {
      "epoch": 0.4340244138732804,
      "grad_norm": 1.6571299514806361,
      "learning_rate": 1.3801311186847081e-06,
      "log_odds_chosen": 1.253515601158142,
      "log_odds_ratio": -0.43964844942092896,
      "logits/chosen": -1.310546875,
      "logits/rejected": -1.116601586341858,
      "logps/chosen": -0.736132800579071,
      "logps/rejected": -1.6316406726837158,
      "loss": 0.9926,
      "nll_loss": 0.9134765863418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07362060248851776,
      "rewards/margins": 0.0895233154296875,
      "rewards/rejected": -0.16318359971046448,
      "step": 3360
    },
    {
      "epoch": 0.43531615320028416,
      "grad_norm": 1.4443000732041174,
      "learning_rate": 1.378081925099109e-06,
      "log_odds_chosen": 1.5732421875,
      "log_odds_ratio": -0.3878417909145355,
      "logits/chosen": -1.1833984851837158,
      "logits/rejected": -0.9828125238418579,
      "logps/chosen": -0.7865234613418579,
      "logps/rejected": -1.9015624523162842,
      "loss": 1.0104,
      "nll_loss": 1.043554663658142,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.07858886569738388,
      "rewards/margins": 0.111572265625,
      "rewards/rejected": -0.19013671576976776,
      "step": 3370
    },
    {
      "epoch": 0.436607892527288,
      "grad_norm": 1.4691098191321261,
      "learning_rate": 1.376041832307563e-06,
      "log_odds_chosen": 1.4033203125,
      "log_odds_ratio": -0.43999022245407104,
      "logits/chosen": -1.224218726158142,
      "logits/rejected": -1.0353515148162842,
      "logps/chosen": -0.758593738079071,
      "logps/rejected": -1.730859398841858,
      "loss": 0.9915,
      "nll_loss": 0.9267578125,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07591553032398224,
      "rewards/margins": 0.09709472954273224,
      "rewards/rejected": -0.17292480170726776,
      "step": 3380
    },
    {
      "epoch": 0.4378996318542918,
      "grad_norm": 1.5769081578967865,
      "learning_rate": 1.3740107731454524e-06,
      "log_odds_chosen": 1.360107421875,
      "log_odds_ratio": -0.4132080078125,
      "logits/chosen": -1.2521483898162842,
      "logits/rejected": -1.025781273841858,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -1.6728515625,
      "loss": 0.9721,
      "nll_loss": 0.9189453125,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07159423828125,
      "rewards/margins": 0.09572906792163849,
      "rewards/rejected": -0.16740722954273224,
      "step": 3390
    },
    {
      "epoch": 0.43919137118129564,
      "grad_norm": 1.5895196057493086,
      "learning_rate": 1.3719886811400705e-06,
      "log_odds_chosen": 1.4857909679412842,
      "log_odds_ratio": -0.43706053495407104,
      "logits/chosen": -1.1318359375,
      "logits/rejected": -1.0166015625,
      "logps/chosen": -0.767382800579071,
      "logps/rejected": -1.8742187023162842,
      "loss": 0.9792,
      "nll_loss": 0.8919922113418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07666015625,
      "rewards/margins": 0.11079101264476776,
      "rewards/rejected": -0.18742676079273224,
      "step": 3400
    },
    {
      "epoch": 0.4404831105082994,
      "grad_norm": 1.4264229184390609,
      "learning_rate": 1.3699754905014834e-06,
      "log_odds_chosen": 1.683984398841858,
      "log_odds_ratio": -0.33134764432907104,
      "logits/chosen": -1.313085913658142,
      "logits/rejected": -0.988476574420929,
      "logps/chosen": -0.727343738079071,
      "logps/rejected": -1.935156226158142,
      "loss": 0.9851,
      "nll_loss": 0.9525390863418579,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.07279052585363388,
      "rewards/margins": 0.12055663764476776,
      "rewards/rejected": -0.19340820610523224,
      "step": 3410
    },
    {
      "epoch": 0.44177484983530324,
      "grad_norm": 1.5944561487582725,
      "learning_rate": 1.3679711361135388e-06,
      "log_odds_chosen": 1.188867211341858,
      "log_odds_ratio": -0.4444824159145355,
      "logits/chosen": -1.158789038658142,
      "logits/rejected": -1.01953125,
      "logps/chosen": -0.7587890625,
      "logps/rejected": -1.613671898841858,
      "loss": 0.9932,
      "nll_loss": 0.9876953363418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07579346001148224,
      "rewards/margins": 0.08562012016773224,
      "rewards/rejected": -0.16147461533546448,
      "step": 3420
    },
    {
      "epoch": 0.44306658916230707,
      "grad_norm": 1.5607214327080359,
      "learning_rate": 1.3659755535250212e-06,
      "log_odds_chosen": 1.2274658679962158,
      "log_odds_ratio": -0.45024412870407104,
      "logits/chosen": -1.273046851158142,
      "logits/rejected": -1.0730469226837158,
      "logps/chosen": -0.6767578125,
      "logps/rejected": -1.564453125,
      "loss": 1.0148,
      "nll_loss": 0.856640636920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06768798828125,
      "rewards/margins": 0.08869628608226776,
      "rewards/rejected": -0.15639647841453552,
      "step": 3430
    },
    {
      "epoch": 0.44435832848931084,
      "grad_norm": 1.946900712613248,
      "learning_rate": 1.3639886789409469e-06,
      "log_odds_chosen": 1.2673461437225342,
      "log_odds_ratio": -0.4585937559604645,
      "logits/chosen": -1.2781250476837158,
      "logits/rejected": -1.0427734851837158,
      "logps/chosen": -0.789843738079071,
      "logps/rejected": -1.744140625,
      "loss": 1.0153,
      "nll_loss": 0.9462890625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07900390774011612,
      "rewards/margins": 0.09552917629480362,
      "rewards/rejected": -0.17446288466453552,
      "step": 3440
    },
    {
      "epoch": 0.44565006781631467,
      "grad_norm": 1.8064466076001375,
      "learning_rate": 1.3620104492139977e-06,
      "log_odds_chosen": 1.329492211341858,
      "log_odds_ratio": -0.45048826932907104,
      "logits/chosen": -1.197265625,
      "logits/rejected": -0.9955078363418579,
      "logps/chosen": -0.748730480670929,
      "logps/rejected": -1.748437523841858,
      "loss": 1.0072,
      "nll_loss": 0.970703125,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07485351711511612,
      "rewards/margins": 0.09989013522863388,
      "rewards/rejected": -0.17485351860523224,
      "step": 3450
    },
    {
      "epoch": 0.4469418071433185,
      "grad_norm": 1.5834900703149317,
      "learning_rate": 1.3600408018360918e-06,
      "log_odds_chosen": 1.181542992591858,
      "log_odds_ratio": -0.42534178495407104,
      "logits/chosen": -1.3029296398162842,
      "logits/rejected": -1.1005859375,
      "logps/chosen": -0.747851550579071,
      "logps/rejected": -1.588281273841858,
      "loss": 0.9978,
      "nll_loss": 0.9234374761581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07480468600988388,
      "rewards/margins": 0.08403930813074112,
      "rewards/rejected": -0.15886230766773224,
      "step": 3460
    },
    {
      "epoch": 0.44823354647032226,
      "grad_norm": 1.6682907659008914,
      "learning_rate": 1.3580796749300878e-06,
      "log_odds_chosen": 1.4191405773162842,
      "log_odds_ratio": -0.41987305879592896,
      "logits/chosen": -1.3595702648162842,
      "logits/rejected": -1.1365234851837158,
      "logps/chosen": -0.75439453125,
      "logps/rejected": -1.8046875,
      "loss": 0.9446,
      "nll_loss": 0.9306640625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07542724907398224,
      "rewards/margins": 0.10507812350988388,
      "rewards/rejected": -0.18061523139476776,
      "step": 3470
    },
    {
      "epoch": 0.4495252857973261,
      "grad_norm": 1.9590984426955198,
      "learning_rate": 1.3561270072416209e-06,
      "log_odds_chosen": 1.522363305091858,
      "log_odds_ratio": -0.38847655057907104,
      "logits/chosen": -1.3064453601837158,
      "logits/rejected": -1.1121094226837158,
      "logps/chosen": -0.70849609375,
      "logps/rejected": -1.802148461341858,
      "loss": 0.97,
      "nll_loss": 0.8902343511581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07088623195886612,
      "rewards/margins": 0.10939331352710724,
      "rewards/rejected": -0.18034668266773224,
      "step": 3480
    },
    {
      "epoch": 0.4508170251243299,
      "grad_norm": 1.5367845973776255,
      "learning_rate": 1.3541827381310652e-06,
      "log_odds_chosen": 1.468359351158142,
      "log_odds_ratio": -0.3921875059604645,
      "logits/chosen": -1.2546875476837158,
      "logits/rejected": -1.0363280773162842,
      "logps/chosen": -0.72900390625,
      "logps/rejected": -1.783203125,
      "loss": 0.9764,
      "nll_loss": 0.9673827886581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0728759765625,
      "rewards/margins": 0.10546875,
      "rewards/rejected": -0.17841796576976776,
      "step": 3490
    },
    {
      "epoch": 0.45210876445133374,
      "grad_norm": 1.4975918544029976,
      "learning_rate": 1.3522468075656264e-06,
      "log_odds_chosen": 1.7684447765350342,
      "log_odds_ratio": -0.3681640625,
      "logits/chosen": -1.2958984375,
      "logits/rejected": -0.9927734136581421,
      "logps/chosen": -0.758593738079071,
      "logps/rejected": -2.089062452316284,
      "loss": 0.995,
      "nll_loss": 0.960156261920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07591553032398224,
      "rewards/margins": 0.13293762505054474,
      "rewards/rejected": -0.20883789658546448,
      "step": 3500
    },
    {
      "epoch": 0.4534005037783375,
      "grad_norm": 1.8392749892168345,
      "learning_rate": 1.3503191561115553e-06,
      "log_odds_chosen": 1.729101538658142,
      "log_odds_ratio": -0.3299316465854645,
      "logits/chosen": -1.3603515625,
      "logits/rejected": -1.070898413658142,
      "logps/chosen": -0.669140636920929,
      "logps/rejected": -1.9343750476837158,
      "loss": 0.9729,
      "nll_loss": 0.9056640863418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06694336235523224,
      "rewards/margins": 0.12644653022289276,
      "rewards/rejected": -0.19345703721046448,
      "step": 3510
    },
    {
      "epoch": 0.45469224310534134,
      "grad_norm": 1.5708301563171019,
      "learning_rate": 1.348399724926484e-06,
      "log_odds_chosen": 1.309667944908142,
      "log_odds_ratio": -0.4463867247104645,
      "logits/chosen": -1.176367163658142,
      "logits/rejected": -1.0146484375,
      "logps/chosen": -0.73828125,
      "logps/rejected": -1.673828125,
      "loss": 0.9458,
      "nll_loss": 0.925585925579071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07380370795726776,
      "rewards/margins": 0.09362182766199112,
      "rewards/rejected": -0.16767577826976776,
      "step": 3520
    },
    {
      "epoch": 0.45598398243234517,
      "grad_norm": 10.822215354023651,
      "learning_rate": 1.346488455751882e-06,
      "log_odds_chosen": 1.657812476158142,
      "log_odds_ratio": -0.3833984434604645,
      "logits/chosen": -1.347265601158142,
      "logits/rejected": -1.0673828125,
      "logps/chosen": -0.739453136920929,
      "logps/rejected": -1.965234398841858,
      "loss": 0.9682,
      "nll_loss": 0.913867175579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07388915866613388,
      "rewards/margins": 0.12266846001148224,
      "rewards/rejected": -0.196533203125,
      "step": 3530
    },
    {
      "epoch": 0.45727572175934894,
      "grad_norm": 1.5064817100665473,
      "learning_rate": 1.3445852909056286e-06,
      "log_odds_chosen": 1.7374999523162842,
      "log_odds_ratio": -0.3180175721645355,
      "logits/chosen": -1.299414038658142,
      "logits/rejected": -1.0625,
      "logps/chosen": -0.691601574420929,
      "logps/rejected": -1.9171874523162842,
      "loss": 0.9808,
      "nll_loss": 0.893359363079071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06911621242761612,
      "rewards/margins": 0.12258300930261612,
      "rewards/rejected": -0.191650390625,
      "step": 3540
    },
    {
      "epoch": 0.45856746108635277,
      "grad_norm": 2.0338982417265328,
      "learning_rate": 1.3426901732747024e-06,
      "log_odds_chosen": 1.295019507408142,
      "log_odds_ratio": -0.4264160096645355,
      "logits/chosen": -1.216796875,
      "logits/rejected": -1.053125023841858,
      "logps/chosen": -0.8638671636581421,
      "logps/rejected": -1.802734375,
      "loss": 0.9915,
      "nll_loss": 0.988574206829071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.08629150688648224,
      "rewards/margins": 0.09398193657398224,
      "rewards/rejected": -0.18017578125,
      "step": 3550
    },
    {
      "epoch": 0.4598592004133566,
      "grad_norm": 1.5442721340441574,
      "learning_rate": 1.3408030463079818e-06,
      "log_odds_chosen": 1.4619140625,
      "log_odds_ratio": -0.415771484375,
      "logits/chosen": -1.31640625,
      "logits/rejected": -1.041406273841858,
      "logps/chosen": -0.796679675579071,
      "logps/rejected": -1.904687523841858,
      "loss": 0.9902,
      "nll_loss": 0.943164050579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07969971001148224,
      "rewards/margins": 0.11088867485523224,
      "rewards/rejected": -0.19074706733226776,
      "step": 3560
    },
    {
      "epoch": 0.4611509397403604,
      "grad_norm": 1.6511161926682998,
      "learning_rate": 1.3389238540091568e-06,
      "log_odds_chosen": 1.4822266101837158,
      "log_odds_ratio": -0.364501953125,
      "logits/chosen": -1.2468750476837158,
      "logits/rejected": -1.034570336341858,
      "logps/chosen": -0.7601562738418579,
      "logps/rejected": -1.8359375,
      "loss": 1.0021,
      "nll_loss": 0.9892578125,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.07597656548023224,
      "rewards/margins": 0.10747070610523224,
      "rewards/rejected": -0.18359375,
      "step": 3570
    },
    {
      "epoch": 0.4624426790673642,
      "grad_norm": 1.6704847548055055,
      "learning_rate": 1.337052540929751e-06,
      "log_odds_chosen": 1.5302734375,
      "log_odds_ratio": -0.36748045682907104,
      "logits/chosen": -1.113867163658142,
      "logits/rejected": -0.9564453363418579,
      "logps/chosen": -0.726757824420929,
      "logps/rejected": -1.8337891101837158,
      "loss": 0.9826,
      "nll_loss": 0.9755859375,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.0726318359375,
      "rewards/margins": 0.11074218899011612,
      "rewards/rejected": -0.18339844048023224,
      "step": 3580
    },
    {
      "epoch": 0.463734418394368,
      "grad_norm": 1.7927372955274787,
      "learning_rate": 1.33518905216225e-06,
      "log_odds_chosen": 1.5802733898162842,
      "log_odds_ratio": -0.3642578125,
      "logits/chosen": -1.251953125,
      "logits/rejected": -1.0265624523162842,
      "logps/chosen": -0.715624988079071,
      "logps/rejected": -1.8503906726837158,
      "loss": 0.9714,
      "nll_loss": 0.960156261920929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.071533203125,
      "rewards/margins": 0.11337890475988388,
      "rewards/rejected": -0.18496093153953552,
      "step": 3590
    },
    {
      "epoch": 0.46502615772137185,
      "grad_norm": 1.6464046185979526,
      "learning_rate": 1.3333333333333332e-06,
      "log_odds_chosen": 1.6052734851837158,
      "log_odds_ratio": -0.3460449278354645,
      "logits/chosen": -1.400781273841858,
      "logits/rejected": -1.055078148841858,
      "logps/chosen": -0.718554675579071,
      "logps/rejected": -1.877343773841858,
      "loss": 0.9996,
      "nll_loss": 0.916015625,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07185058295726776,
      "rewards/margins": 0.11573486030101776,
      "rewards/rejected": -0.18759766221046448,
      "step": 3600
    },
    {
      "epoch": 0.4663178970483756,
      "grad_norm": 1.9312834775052827,
      "learning_rate": 1.3314853305972122e-06,
      "log_odds_chosen": 1.324121117591858,
      "log_odds_ratio": -0.41923826932907104,
      "logits/chosen": -1.13671875,
      "logits/rejected": -0.9970703125,
      "logps/chosen": -0.7392578125,
      "logps/rejected": -1.653710961341858,
      "loss": 0.9768,
      "nll_loss": 0.9449218511581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07392577826976776,
      "rewards/margins": 0.09138794243335724,
      "rewards/rejected": -0.165283203125,
      "step": 3610
    },
    {
      "epoch": 0.46760963637537944,
      "grad_norm": 1.9891494392053055,
      "learning_rate": 1.3296449906290671e-06,
      "log_odds_chosen": 1.204980492591858,
      "log_odds_ratio": -0.4417480528354645,
      "logits/chosen": -1.2517578601837158,
      "logits/rejected": -1.107812523841858,
      "logps/chosen": -0.758984386920929,
      "logps/rejected": -1.648828148841858,
      "loss": 0.9986,
      "nll_loss": 1.002539038658142,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07586669921875,
      "rewards/margins": 0.088958740234375,
      "rewards/rejected": -0.16494140028953552,
      "step": 3620
    },
    {
      "epoch": 0.46890137570238327,
      "grad_norm": 9.661617872243768,
      "learning_rate": 1.3278122606185844e-06,
      "log_odds_chosen": 1.480078101158142,
      "log_odds_ratio": -0.394287109375,
      "logits/chosen": -1.2937500476837158,
      "logits/rejected": -1.0714843273162842,
      "logps/chosen": -0.710156261920929,
      "logps/rejected": -1.78515625,
      "loss": 0.9949,
      "nll_loss": 0.9234374761581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.07099609076976776,
      "rewards/margins": 0.10740967094898224,
      "rewards/rejected": -0.17841796576976776,
      "step": 3630
    },
    {
      "epoch": 0.47019311502938704,
      "grad_norm": 1.563972260148418,
      "learning_rate": 1.3259870882635918e-06,
      "log_odds_chosen": 1.7492187023162842,
      "log_odds_ratio": -0.3333496153354645,
      "logits/chosen": -1.3972656726837158,
      "logits/rejected": -1.099218726158142,
      "logps/chosen": -0.727246105670929,
      "logps/rejected": -2.0238280296325684,
      "loss": 0.9828,
      "nll_loss": 0.922656238079071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.07277832180261612,
      "rewards/margins": 0.12969970703125,
      "rewards/rejected": -0.202392578125,
      "step": 3640
    },
    {
      "epoch": 0.47148485435639087,
      "grad_norm": 1.6261822344604553,
      "learning_rate": 1.3241694217637886e-06,
      "log_odds_chosen": 1.506250023841858,
      "log_odds_ratio": -0.35009765625,
      "logits/chosen": -1.2687499523162842,
      "logits/rejected": -1.046289086341858,
      "logps/chosen": -0.686816394329071,
      "logps/rejected": -1.73828125,
      "loss": 0.9811,
      "nll_loss": 0.902539074420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06868896633386612,
      "rewards/margins": 0.10502929985523224,
      "rewards/rejected": -0.17390136420726776,
      "step": 3650
    },
    {
      "epoch": 0.4727765936833947,
      "grad_norm": 1.5306094079513928,
      "learning_rate": 1.3223592098145723e-06,
      "log_odds_chosen": 1.362890601158142,
      "log_odds_ratio": -0.4374023377895355,
      "logits/chosen": -1.264062523841858,
      "logits/rejected": -1.050195336341858,
      "logps/chosen": -0.7564452886581421,
      "logps/rejected": -1.7625000476837158,
      "loss": 0.9631,
      "nll_loss": 0.930468738079071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07564697414636612,
      "rewards/margins": 0.10084228217601776,
      "rewards/rejected": -0.1763916015625,
      "step": 3660
    },
    {
      "epoch": 0.4740683330103985,
      "grad_norm": 1.9890572232639956,
      "learning_rate": 1.3205564016009555e-06,
      "log_odds_chosen": 1.374609351158142,
      "log_odds_ratio": -0.436767578125,
      "logits/chosen": -1.2498047351837158,
      "logits/rejected": -1.08203125,
      "logps/chosen": -0.767773449420929,
      "logps/rejected": -1.7785155773162842,
      "loss": 0.9848,
      "nll_loss": 0.943554699420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07679443061351776,
      "rewards/margins": 0.10096435248851776,
      "rewards/rejected": -0.17768554389476776,
      "step": 3670
    },
    {
      "epoch": 0.4753600723374023,
      "grad_norm": 1.6751524808376392,
      "learning_rate": 1.318760946791574e-06,
      "log_odds_chosen": 1.462499976158142,
      "log_odds_ratio": -0.42333984375,
      "logits/chosen": -1.1130859851837158,
      "logits/rejected": -1.001562476158142,
      "logps/chosen": -0.7181640863418579,
      "logps/rejected": -1.8195312023162842,
      "loss": 0.9776,
      "nll_loss": 0.8794921636581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07185058295726776,
      "rewards/margins": 0.11024780571460724,
      "rewards/rejected": -0.181884765625,
      "step": 3680
    },
    {
      "epoch": 0.4766518116644061,
      "grad_norm": 1.4818300146646157,
      "learning_rate": 1.316972795532786e-06,
      "log_odds_chosen": 1.6818358898162842,
      "log_odds_ratio": -0.33989256620407104,
      "logits/chosen": -1.283593773841858,
      "logits/rejected": -1.0583984851837158,
      "logps/chosen": -0.71923828125,
      "logps/rejected": -1.9562499523162842,
      "loss": 0.9722,
      "nll_loss": 0.9150390625,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07188721001148224,
      "rewards/margins": 0.12364502251148224,
      "rewards/rejected": -0.19562987983226776,
      "step": 3690
    },
    {
      "epoch": 0.47794355099140995,
      "grad_norm": 1.6905819193403553,
      "learning_rate": 1.3151918984428582e-06,
      "log_odds_chosen": 1.639257788658142,
      "log_odds_ratio": -0.3614257872104645,
      "logits/chosen": -1.27734375,
      "logits/rejected": -1.065039038658142,
      "logps/chosen": -0.740234375,
      "logps/rejected": -1.932031273841858,
      "loss": 0.9599,
      "nll_loss": 0.8980468511581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07399902492761612,
      "rewards/margins": 0.11922607570886612,
      "rewards/rejected": -0.19321289658546448,
      "step": 3700
    },
    {
      "epoch": 0.4792352903184137,
      "grad_norm": 1.5854113641156875,
      "learning_rate": 1.313418206606237e-06,
      "log_odds_chosen": 1.3996093273162842,
      "log_odds_ratio": -0.414306640625,
      "logits/chosen": -1.2830078601837158,
      "logits/rejected": -1.0802733898162842,
      "logps/chosen": -0.684374988079071,
      "logps/rejected": -1.662500023841858,
      "loss": 0.9791,
      "nll_loss": 0.8685547113418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06844482570886612,
      "rewards/margins": 0.09786377102136612,
      "rewards/rejected": -0.16630859673023224,
      "step": 3710
    },
    {
      "epoch": 0.48052702964541755,
      "grad_norm": 1.6239080007040285,
      "learning_rate": 1.3116516715679057e-06,
      "log_odds_chosen": 1.3839843273162842,
      "log_odds_ratio": -0.40498048067092896,
      "logits/chosen": -1.289453148841858,
      "logits/rejected": -1.112890601158142,
      "logps/chosen": -0.745312511920929,
      "logps/rejected": -1.7216796875,
      "loss": 0.9557,
      "nll_loss": 0.949414074420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.074462890625,
      "rewards/margins": 0.09761963039636612,
      "rewards/rejected": -0.172119140625,
      "step": 3720
    },
    {
      "epoch": 0.4818187689724214,
      "grad_norm": 1.6771426969508356,
      "learning_rate": 1.3098922453278258e-06,
      "log_odds_chosen": 1.171972632408142,
      "log_odds_ratio": -0.43574219942092896,
      "logits/chosen": -1.291601538658142,
      "logits/rejected": -1.116796851158142,
      "logps/chosen": -0.7010742425918579,
      "logps/rejected": -1.496484398841858,
      "loss": 0.9386,
      "nll_loss": 0.903515636920929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07011719048023224,
      "rewards/margins": 0.07961425930261612,
      "rewards/rejected": -0.149658203125,
      "step": 3730
    },
    {
      "epoch": 0.4831105082994252,
      "grad_norm": 4.290459544391673,
      "learning_rate": 1.3081398803354573e-06,
      "log_odds_chosen": 1.739843726158142,
      "log_odds_ratio": -0.36572265625,
      "logits/chosen": -1.2585937976837158,
      "logits/rejected": -1.087890625,
      "logps/chosen": -0.7457031011581421,
      "logps/rejected": -2.0406250953674316,
      "loss": 0.9685,
      "nll_loss": 0.9154297113418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07463379204273224,
      "rewards/margins": 0.12946777045726776,
      "rewards/rejected": -0.20419922471046448,
      "step": 3740
    },
    {
      "epoch": 0.484402247626429,
      "grad_norm": 1.8444294877099479,
      "learning_rate": 1.3063945294843617e-06,
      "log_odds_chosen": 1.2117919921875,
      "log_odds_ratio": -0.44316405057907104,
      "logits/chosen": -1.278710961341858,
      "logits/rejected": -1.074804663658142,
      "logps/chosen": -0.7578125,
      "logps/rejected": -1.6457030773162842,
      "loss": 0.9453,
      "nll_loss": 0.922070324420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07585449516773224,
      "rewards/margins": 0.08876800537109375,
      "rewards/rejected": -0.16459961235523224,
      "step": 3750
    },
    {
      "epoch": 0.4856939869534328,
      "grad_norm": 2.100214613837535,
      "learning_rate": 1.3046561461068843e-06,
      "log_odds_chosen": 1.4404296875,
      "log_odds_ratio": -0.42402344942092896,
      "logits/chosen": -1.270898461341858,
      "logits/rejected": -1.0693359375,
      "logps/chosen": -0.737109363079071,
      "logps/rejected": -1.79296875,
      "loss": 0.9706,
      "nll_loss": 0.994824230670929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07366943359375,
      "rewards/margins": 0.10550536960363388,
      "rewards/rejected": -0.17922362685203552,
      "step": 3760
    },
    {
      "epoch": 0.4869857262804366,
      "grad_norm": 1.7129865512806293,
      "learning_rate": 1.3029246839689124e-06,
      "log_odds_chosen": 1.615234375,
      "log_odds_ratio": -0.390625,
      "logits/chosen": -1.353124976158142,
      "logits/rejected": -1.149999976158142,
      "logps/chosen": -0.7398437261581421,
      "logps/rejected": -1.9230468273162842,
      "loss": 0.9636,
      "nll_loss": 0.90625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07401122897863388,
      "rewards/margins": 0.11821899563074112,
      "rewards/rejected": -0.19223633408546448,
      "step": 3770
    },
    {
      "epoch": 0.4882774656074404,
      "grad_norm": 2.0378856053337464,
      "learning_rate": 1.3012000972647109e-06,
      "log_odds_chosen": 1.659082055091858,
      "log_odds_ratio": -0.38300782442092896,
      "logits/chosen": -1.26171875,
      "logits/rejected": -1.0486328601837158,
      "logps/chosen": -0.726757824420929,
      "logps/rejected": -1.970703125,
      "loss": 0.9932,
      "nll_loss": 0.9671875238418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07269287109375,
      "rewards/margins": 0.124481201171875,
      "rewards/rejected": -0.19716796278953552,
      "step": 3780
    },
    {
      "epoch": 0.4895692049344442,
      "grad_norm": 1.6560517751635104,
      "learning_rate": 1.299482340611832e-06,
      "log_odds_chosen": 1.6589844226837158,
      "log_odds_ratio": -0.3722167909145355,
      "logits/chosen": -1.2373046875,
      "logits/rejected": -1.021093726158142,
      "logps/chosen": -0.732617199420929,
      "logps/rejected": -1.9382812976837158,
      "loss": 0.991,
      "nll_loss": 0.9781249761581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07326660305261612,
      "rewards/margins": 0.12054443359375,
      "rewards/rejected": -0.19377441704273224,
      "step": 3790
    },
    {
      "epoch": 0.49086094426144805,
      "grad_norm": 7.633727592655389,
      "learning_rate": 1.2977713690461003e-06,
      "log_odds_chosen": 1.539697289466858,
      "log_odds_ratio": -0.4036621153354645,
      "logits/chosen": -1.259765625,
      "logits/rejected": -1.078125,
      "logps/chosen": -0.8271484375,
      "logps/rejected": -1.990234375,
      "loss": 0.9671,
      "nll_loss": 1.021093726158142,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.08278808742761612,
      "rewards/margins": 0.11631164699792862,
      "rewards/rejected": -0.198974609375,
      "step": 3800
    },
    {
      "epoch": 0.4921526835884519,
      "grad_norm": 1.6293300737992338,
      "learning_rate": 1.296067138016669e-06,
      "log_odds_chosen": 1.5295898914337158,
      "log_odds_ratio": -0.3731933534145355,
      "logits/chosen": -1.3302733898162842,
      "logits/rejected": -1.094335913658142,
      "logps/chosen": -0.669140636920929,
      "logps/rejected": -1.7859375476837158,
      "loss": 0.9839,
      "nll_loss": 0.8970702886581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06694336235523224,
      "rewards/margins": 0.11159972846508026,
      "rewards/rejected": -0.17856445908546448,
      "step": 3810
    },
    {
      "epoch": 0.49344442291545565,
      "grad_norm": 1.8527145758168402,
      "learning_rate": 1.294369603381147e-06,
      "log_odds_chosen": 1.337304711341858,
      "log_odds_ratio": -0.4295410215854645,
      "logits/chosen": -1.29296875,
      "logits/rejected": -1.0671875476837158,
      "logps/chosen": -0.7134765386581421,
      "logps/rejected": -1.667578101158142,
      "loss": 0.9588,
      "nll_loss": 0.90625,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07133789360523224,
      "rewards/margins": 0.09539184719324112,
      "rewards/rejected": -0.16669921576976776,
      "step": 3820
    },
    {
      "epoch": 0.4947361622424595,
      "grad_norm": 1.5132568465612,
      "learning_rate": 1.2926787214007981e-06,
      "log_odds_chosen": 1.443945288658142,
      "log_odds_ratio": -0.3914550840854645,
      "logits/chosen": -1.3347656726837158,
      "logits/rejected": -1.1123046875,
      "logps/chosen": -0.6856445074081421,
      "logps/rejected": -1.7091796398162842,
      "loss": 0.9792,
      "nll_loss": 0.947460949420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06854248046875,
      "rewards/margins": 0.10241089016199112,
      "rewards/rejected": -0.17097167670726776,
      "step": 3830
    },
    {
      "epoch": 0.4960279015694633,
      "grad_norm": 1.5774562875666789,
      "learning_rate": 1.2909944487358056e-06,
      "log_odds_chosen": 1.307836890220642,
      "log_odds_ratio": -0.41044920682907104,
      "logits/chosen": -1.1892578601837158,
      "logits/rejected": -1.013281226158142,
      "logps/chosen": -0.71533203125,
      "logps/rejected": -1.6201171875,
      "loss": 0.9634,
      "nll_loss": 0.931445300579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07154540717601776,
      "rewards/margins": 0.09071274101734161,
      "rewards/rejected": -0.16230468451976776,
      "step": 3840
    },
    {
      "epoch": 0.4973196408964671,
      "grad_norm": 1.6710527826349575,
      "learning_rate": 1.2893167424406084e-06,
      "log_odds_chosen": 1.5478515625,
      "log_odds_ratio": -0.37836915254592896,
      "logits/chosen": -1.216796875,
      "logits/rejected": -0.9771484136581421,
      "logps/chosen": -0.720703125,
      "logps/rejected": -1.855078101158142,
      "loss": 0.9396,
      "nll_loss": 0.9150390625,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07196044921875,
      "rewards/margins": 0.11341552436351776,
      "rewards/rejected": -0.18540039658546448,
      "step": 3850
    },
    {
      "epoch": 0.4986113802234709,
      "grad_norm": 1.654640734200055,
      "learning_rate": 1.2876455599593008e-06,
      "log_odds_chosen": 1.214746117591858,
      "log_odds_ratio": -0.4530273377895355,
      "logits/chosen": -1.2744140625,
      "logits/rejected": -1.093164086341858,
      "logps/chosen": -0.7601562738418579,
      "logps/rejected": -1.651953101158142,
      "loss": 0.9607,
      "nll_loss": 0.953906238079071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0760498046875,
      "rewards/margins": 0.08902587741613388,
      "rewards/rejected": -0.16513672471046448,
      "step": 3860
    },
    {
      "epoch": 0.49990311955047473,
      "grad_norm": 1.794492198439638,
      "learning_rate": 1.285980859121099e-06,
      "log_odds_chosen": 1.396484375,
      "log_odds_ratio": -0.40141600370407104,
      "logits/chosen": -1.1886718273162842,
      "logits/rejected": -1.044921875,
      "logps/chosen": -0.7486327886581421,
      "logps/rejected": -1.72265625,
      "loss": 0.9691,
      "nll_loss": 1.0076172351837158,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07480468600988388,
      "rewards/margins": 0.09727325290441513,
      "rewards/rejected": -0.17209473252296448,
      "step": 3870
    },
    {
      "epoch": 0.5011948588774785,
      "grad_norm": 1.9048132737433332,
      "learning_rate": 1.2843225981358712e-06,
      "log_odds_chosen": 1.25146484375,
      "log_odds_ratio": -0.4400878846645355,
      "logits/chosen": -1.186914086341858,
      "logits/rejected": -1.027929663658142,
      "logps/chosen": -0.754687488079071,
      "logps/rejected": -1.629296898841858,
      "loss": 0.9612,
      "nll_loss": 0.9076172113418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07550048828125,
      "rewards/margins": 0.08762817084789276,
      "rewards/rejected": -0.16313476860523224,
      "step": 3880
    },
    {
      "epoch": 0.5024865982044824,
      "grad_norm": 1.723197788437254,
      "learning_rate": 1.2826707355897317e-06,
      "log_odds_chosen": 1.443750023841858,
      "log_odds_ratio": -0.383056640625,
      "logits/chosen": -1.3359375,
      "logits/rejected": -1.10546875,
      "logps/chosen": -0.7367187738418579,
      "logps/rejected": -1.764062523841858,
      "loss": 0.9326,
      "nll_loss": 0.9017578363418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07366943359375,
      "rewards/margins": 0.10273437201976776,
      "rewards/rejected": -0.17656250298023224,
      "step": 3890
    },
    {
      "epoch": 0.5037783375314862,
      "grad_norm": 1.921703740563146,
      "learning_rate": 1.281025230440697e-06,
      "log_odds_chosen": 1.5695312023162842,
      "log_odds_ratio": -0.3865722715854645,
      "logits/chosen": -1.1396484375,
      "logits/rejected": -0.989453136920929,
      "logps/chosen": -0.745312511920929,
      "logps/rejected": -1.8605468273162842,
      "loss": 0.9646,
      "nll_loss": 0.9380859136581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07454834133386612,
      "rewards/margins": 0.11152343451976776,
      "rewards/rejected": -0.18598632514476776,
      "step": 3900
    },
    {
      "epoch": 0.5050700768584899,
      "grad_norm": 1.5901434623404858,
      "learning_rate": 1.2793860420144025e-06,
      "log_odds_chosen": 1.3917968273162842,
      "log_odds_ratio": -0.4380859434604645,
      "logits/chosen": -1.132226586341858,
      "logits/rejected": -0.9947265386581421,
      "logps/chosen": -0.7291015386581421,
      "logps/rejected": -1.7470703125,
      "loss": 0.971,
      "nll_loss": 0.8880859613418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07292480766773224,
      "rewards/margins": 0.10167236626148224,
      "rewards/rejected": -0.174560546875,
      "step": 3910
    },
    {
      "epoch": 0.5063618161854938,
      "grad_norm": 1.6583533669306973,
      "learning_rate": 1.2777531299998798e-06,
      "log_odds_chosen": 1.51953125,
      "log_odds_ratio": -0.37822264432907104,
      "logits/chosen": -1.324609398841858,
      "logits/rejected": -1.132226586341858,
      "logps/chosen": -0.7906249761581421,
      "logps/rejected": -1.9191405773162842,
      "loss": 0.9573,
      "nll_loss": 0.9593750238418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07908935844898224,
      "rewards/margins": 0.11300048977136612,
      "rewards/rejected": -0.19189453125,
      "step": 3920
    },
    {
      "epoch": 0.5076535555124976,
      "grad_norm": 1.759715289142672,
      "learning_rate": 1.2761264544453928e-06,
      "log_odds_chosen": 1.302343726158142,
      "log_odds_ratio": -0.40727537870407104,
      "logits/chosen": -1.267578125,
      "logits/rejected": -1.1169922351837158,
      "logps/chosen": -0.739453136920929,
      "logps/rejected": -1.648046851158142,
      "loss": 0.9887,
      "nll_loss": 0.8998047113418579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07391357421875,
      "rewards/margins": 0.09085693210363388,
      "rewards/rejected": -0.1649169921875,
      "step": 3930
    },
    {
      "epoch": 0.5089452948395013,
      "grad_norm": 1.702777392436381,
      "learning_rate": 1.2745059757543324e-06,
      "log_odds_chosen": 1.4255859851837158,
      "log_odds_ratio": -0.40361326932907104,
      "logits/chosen": -1.313085913658142,
      "logits/rejected": -1.052734375,
      "logps/chosen": -0.7525390386581421,
      "logps/rejected": -1.7800781726837158,
      "loss": 1.0051,
      "nll_loss": 0.9945312738418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07530517876148224,
      "rewards/margins": 0.10268554836511612,
      "rewards/rejected": -0.17795410752296448,
      "step": 3940
    },
    {
      "epoch": 0.5102370341665052,
      "grad_norm": 1.708938372854009,
      "learning_rate": 1.272891654681168e-06,
      "log_odds_chosen": 1.4802734851837158,
      "log_odds_ratio": -0.40239256620407104,
      "logits/chosen": -1.2628905773162842,
      "logits/rejected": -1.1023437976837158,
      "logps/chosen": -0.734570324420929,
      "logps/rejected": -1.7761719226837158,
      "loss": 0.9565,
      "nll_loss": 0.921679675579071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07351074367761612,
      "rewards/margins": 0.10418701171875,
      "rewards/rejected": -0.17753906548023224,
      "step": 3950
    },
    {
      "epoch": 0.511528773493509,
      "grad_norm": 1.664215034166162,
      "learning_rate": 1.2712834523274563e-06,
      "log_odds_chosen": 1.6281859874725342,
      "log_odds_ratio": -0.4032226502895355,
      "logits/chosen": -1.2742187976837158,
      "logits/rejected": -1.0419921875,
      "logps/chosen": -0.789843738079071,
      "logps/rejected": -2.0283203125,
      "loss": 0.9736,
      "nll_loss": 1.003515601158142,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07895507663488388,
      "rewards/margins": 0.12362518161535263,
      "rewards/rejected": -0.20263671875,
      "step": 3960
    },
    {
      "epoch": 0.5128205128205128,
      "grad_norm": 1.5671953083599743,
      "learning_rate": 1.2696813301379032e-06,
      "log_odds_chosen": 1.3952147960662842,
      "log_odds_ratio": -0.38935548067092896,
      "logits/chosen": -1.2724609375,
      "logits/rejected": -1.0867187976837158,
      "logps/chosen": -0.7577148675918579,
      "logps/rejected": -1.7587890625,
      "loss": 0.9498,
      "nll_loss": 0.900585949420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07583007961511612,
      "rewards/margins": 0.1000518798828125,
      "rewards/rejected": -0.17600098252296448,
      "step": 3970
    },
    {
      "epoch": 0.5141122521475167,
      "grad_norm": 1.6899342194702278,
      "learning_rate": 1.2680852498964829e-06,
      "log_odds_chosen": 1.682226538658142,
      "log_odds_ratio": -0.3612304627895355,
      "logits/chosen": -1.324804663658142,
      "logits/rejected": -1.0380859375,
      "logps/chosen": -0.747851550579071,
      "logps/rejected": -2.001171827316284,
      "loss": 0.9795,
      "nll_loss": 0.896679699420929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07484130561351776,
      "rewards/margins": 0.12528076767921448,
      "rewards/rejected": -0.20012207329273224,
      "step": 3980
    },
    {
      "epoch": 0.5154039914745204,
      "grad_norm": 2.308014510578964,
      "learning_rate": 1.266495173722607e-06,
      "log_odds_chosen": 1.508142113685608,
      "log_odds_ratio": -0.419921875,
      "logits/chosen": -1.404687523841858,
      "logits/rejected": -1.166406273841858,
      "logps/chosen": -0.726855456829071,
      "logps/rejected": -1.833984375,
      "loss": 0.9486,
      "nll_loss": 0.836132824420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07266845554113388,
      "rewards/margins": 0.11069945991039276,
      "rewards/rejected": -0.18325194716453552,
      "step": 3990
    },
    {
      "epoch": 0.5166957308015242,
      "grad_norm": 1.9106146843217657,
      "learning_rate": 1.2649110640673517e-06,
      "log_odds_chosen": 1.5413086414337158,
      "log_odds_ratio": -0.4026855528354645,
      "logits/chosen": -1.2267577648162842,
      "logits/rejected": -0.9599609375,
      "logps/chosen": -0.7193359136581421,
      "logps/rejected": -1.845312476158142,
      "loss": 0.9347,
      "nll_loss": 0.910351574420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07187499850988388,
      "rewards/margins": 0.11249389499425888,
      "rewards/rejected": -0.1844482421875,
      "step": 4000
    },
    {
      "epoch": 0.5179874701285281,
      "grad_norm": 1.8268150921521455,
      "learning_rate": 1.2633328837097308e-06,
      "log_odds_chosen": 1.5032227039337158,
      "log_odds_ratio": -0.39741212129592896,
      "logits/chosen": -1.285546898841858,
      "logits/rejected": -1.0578124523162842,
      "logps/chosen": -0.7796875238418579,
      "logps/rejected": -1.870703101158142,
      "loss": 0.969,
      "nll_loss": 1.003515601158142,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07799072563648224,
      "rewards/margins": 0.10906372219324112,
      "rewards/rejected": -0.18691405653953552,
      "step": 4010
    },
    {
      "epoch": 0.5192792094555319,
      "grad_norm": 1.7339150863080335,
      "learning_rate": 1.2617605957530233e-06,
      "log_odds_chosen": 1.6003906726837158,
      "log_odds_ratio": -0.3857421875,
      "logits/chosen": -1.2353515625,
      "logits/rejected": -0.9755859375,
      "logps/chosen": -0.7574218511581421,
      "logps/rejected": -1.9578125476837158,
      "loss": 0.9557,
      "nll_loss": 0.9638671875,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07564697414636612,
      "rewards/margins": 0.12000732123851776,
      "rewards/rejected": -0.19558104872703552,
      "step": 4020
    },
    {
      "epoch": 0.5205709487825357,
      "grad_norm": 1.7986431899614321,
      "learning_rate": 1.2601941636211516e-06,
      "log_odds_chosen": 1.5583984851837158,
      "log_odds_ratio": -0.3858886659145355,
      "logits/chosen": -1.289453148841858,
      "logits/rejected": -1.0568358898162842,
      "logps/chosen": -0.7880859375,
      "logps/rejected": -1.968359351158142,
      "loss": 0.9681,
      "nll_loss": 0.869921863079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07882080227136612,
      "rewards/margins": 0.11789550632238388,
      "rewards/rejected": -0.1968994140625,
      "step": 4030
    },
    {
      "epoch": 0.5218626881095395,
      "grad_norm": 1.7077146069754672,
      "learning_rate": 1.2586335510551052e-06,
      "log_odds_chosen": 1.4246094226837158,
      "log_odds_ratio": -0.3975585997104645,
      "logits/chosen": -1.326562523841858,
      "logits/rejected": -1.1326172351837158,
      "logps/chosen": -0.7496093511581421,
      "logps/rejected": -1.7341797351837158,
      "loss": 0.9741,
      "nll_loss": 0.8863281011581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07490234076976776,
      "rewards/margins": 0.09852294623851776,
      "rewards/rejected": -0.17341308295726776,
      "step": 4040
    },
    {
      "epoch": 0.5231544274365433,
      "grad_norm": 1.6564549248139329,
      "learning_rate": 1.2570787221094177e-06,
      "log_odds_chosen": 1.673730492591858,
      "log_odds_ratio": -0.34833985567092896,
      "logits/chosen": -1.3253905773162842,
      "logits/rejected": -1.048437476158142,
      "logps/chosen": -0.737988293170929,
      "logps/rejected": -1.968359351158142,
      "loss": 0.972,
      "nll_loss": 0.946484386920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07375488430261612,
      "rewards/margins": 0.12316284328699112,
      "rewards/rejected": -0.19677734375,
      "step": 4050
    },
    {
      "epoch": 0.5244461667635472,
      "grad_norm": 1.6262938515864036,
      "learning_rate": 1.255529641148689e-06,
      "log_odds_chosen": 1.52899169921875,
      "log_odds_ratio": -0.435791015625,
      "logits/chosen": -1.325781226158142,
      "logits/rejected": -1.1218750476837158,
      "logps/chosen": -0.791015625,
      "logps/rejected": -1.91796875,
      "loss": 0.9353,
      "nll_loss": 0.873828113079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07912597805261612,
      "rewards/margins": 0.11279296875,
      "rewards/rejected": -0.19194336235523224,
      "step": 4060
    },
    {
      "epoch": 0.5257379060905509,
      "grad_norm": 1.69390125026409,
      "learning_rate": 1.2539862728441536e-06,
      "log_odds_chosen": 1.677734375,
      "log_odds_ratio": -0.3800292909145355,
      "logits/chosen": -1.252539038658142,
      "logits/rejected": -1.0242187976837158,
      "logps/chosen": -0.725390613079071,
      "logps/rejected": -1.9845702648162842,
      "loss": 0.9479,
      "nll_loss": 0.9478515386581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07258300483226776,
      "rewards/margins": 0.1258544921875,
      "rewards/rejected": -0.1983642578125,
      "step": 4070
    },
    {
      "epoch": 0.5270296454175547,
      "grad_norm": 1.7134731561175889,
      "learning_rate": 1.252448582170299e-06,
      "log_odds_chosen": 1.720703125,
      "log_odds_ratio": -0.39306640625,
      "logits/chosen": -1.2724609375,
      "logits/rejected": -1.0857422351837158,
      "logps/chosen": -0.7416015863418579,
      "logps/rejected": -2.0308594703674316,
      "loss": 0.9515,
      "nll_loss": 0.9937499761581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0740966796875,
      "rewards/margins": 0.12897948920726776,
      "rewards/rejected": -0.20310059189796448,
      "step": 4080
    },
    {
      "epoch": 0.5283213847445586,
      "grad_norm": 2.160268363756649,
      "learning_rate": 1.2509165344015243e-06,
      "log_odds_chosen": 1.555078148841858,
      "log_odds_ratio": -0.37983399629592896,
      "logits/chosen": -1.3503906726837158,
      "logits/rejected": -1.1496093273162842,
      "logps/chosen": -0.6973632574081421,
      "logps/rejected": -1.7999999523162842,
      "loss": 0.9247,
      "nll_loss": 0.865527331829071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06976318359375,
      "rewards/margins": 0.11015625298023224,
      "rewards/rejected": -0.17990723252296448,
      "step": 4090
    },
    {
      "epoch": 0.5296131240715624,
      "grad_norm": 1.8550735513065018,
      "learning_rate": 1.2493900951088486e-06,
      "log_odds_chosen": 1.506250023841858,
      "log_odds_ratio": -0.3660888671875,
      "logits/chosen": -1.259179711341858,
      "logits/rejected": -1.0925781726837158,
      "logps/chosen": -0.6998046636581421,
      "logps/rejected": -1.802734375,
      "loss": 0.9579,
      "nll_loss": 0.8716796636581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06998290866613388,
      "rewards/margins": 0.11025390774011612,
      "rewards/rejected": -0.18020018935203552,
      "step": 4100
    },
    {
      "epoch": 0.5309048633985661,
      "grad_norm": 15.531161135006124,
      "learning_rate": 1.2478692301566601e-06,
      "log_odds_chosen": 1.513671875,
      "log_odds_ratio": -0.3691650331020355,
      "logits/chosen": -1.29296875,
      "logits/rejected": -1.1248047351837158,
      "logps/chosen": -0.7320312261581421,
      "logps/rejected": -1.83203125,
      "loss": 0.975,
      "nll_loss": 0.9072265625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0732421875,
      "rewards/margins": 0.10983886569738388,
      "rewards/rejected": -0.18305663764476776,
      "step": 4110
    },
    {
      "epoch": 0.53219660272557,
      "grad_norm": 1.6684823411584337,
      "learning_rate": 1.2463539056995116e-06,
      "log_odds_chosen": 1.465429663658142,
      "log_odds_ratio": -0.35991209745407104,
      "logits/chosen": -1.2927734851837158,
      "logits/rejected": -1.0927734375,
      "logps/chosen": -0.7080078125,
      "logps/rejected": -1.75,
      "loss": 0.9715,
      "nll_loss": 0.9478515386581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.07083740085363388,
      "rewards/margins": 0.10416259616613388,
      "rewards/rejected": -0.17487792670726776,
      "step": 4120
    },
    {
      "epoch": 0.5334883420525738,
      "grad_norm": 1.754161482071051,
      "learning_rate": 1.2448440881789541e-06,
      "log_odds_chosen": 1.450585961341858,
      "log_odds_ratio": -0.4059081971645355,
      "logits/chosen": -1.228906273841858,
      "logits/rejected": -1.047460913658142,
      "logps/chosen": -0.7691406011581421,
      "logps/rejected": -1.799218773841858,
      "loss": 0.972,
      "nll_loss": 0.896484375,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.076904296875,
      "rewards/margins": 0.10286865383386612,
      "rewards/rejected": -0.18002930283546448,
      "step": 4130
    },
    {
      "epoch": 0.5347800813795776,
      "grad_norm": 1.7108940328604167,
      "learning_rate": 1.2433397443204184e-06,
      "log_odds_chosen": 1.389013648033142,
      "log_odds_ratio": -0.46796876192092896,
      "logits/chosen": -1.354882836341858,
      "logits/rejected": -1.122460961341858,
      "logps/chosen": -0.7403320074081421,
      "logps/rejected": -1.755273461341858,
      "loss": 0.9602,
      "nll_loss": 0.8814452886581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07404784858226776,
      "rewards/margins": 0.10133972018957138,
      "rewards/rejected": -0.17546387016773224,
      "step": 4140
    },
    {
      "epoch": 0.5360718207065814,
      "grad_norm": 1.7068325338172095,
      "learning_rate": 1.2418408411301324e-06,
      "log_odds_chosen": 1.453515648841858,
      "log_odds_ratio": -0.4480957090854645,
      "logits/chosen": -1.256250023841858,
      "logits/rejected": -1.087304711341858,
      "logps/chosen": -0.7826172113418579,
      "logps/rejected": -1.8806641101837158,
      "loss": 0.9486,
      "nll_loss": 0.88037109375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07828368991613388,
      "rewards/margins": 0.10976562649011612,
      "rewards/rejected": -0.18801268935203552,
      "step": 4150
    },
    {
      "epoch": 0.5373635600335852,
      "grad_norm": 1.8369552525918529,
      "learning_rate": 1.2403473458920844e-06,
      "log_odds_chosen": 1.718164086341858,
      "log_odds_ratio": -0.3936523497104645,
      "logits/chosen": -1.3250000476837158,
      "logits/rejected": -1.0945312976837158,
      "logps/chosen": -0.743359386920929,
      "logps/rejected": -2.033984422683716,
      "loss": 0.9849,
      "nll_loss": 0.911425769329071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07429198920726776,
      "rewards/margins": 0.129150390625,
      "rewards/rejected": -0.2034912109375,
      "step": 4160
    },
    {
      "epoch": 0.538655299360589,
      "grad_norm": 3.355249074580041,
      "learning_rate": 1.2388592261650217e-06,
      "log_odds_chosen": 1.226709008216858,
      "log_odds_ratio": -0.4769043028354645,
      "logits/chosen": -1.2345702648162842,
      "logits/rejected": -1.0197265148162842,
      "logps/chosen": -0.7232421636581421,
      "logps/rejected": -1.5974609851837158,
      "loss": 0.9414,
      "nll_loss": 0.930859386920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07230224460363388,
      "rewards/margins": 0.08745422214269638,
      "rewards/rejected": -0.15971679985523224,
      "step": 4170
    },
    {
      "epoch": 0.5399470386875929,
      "grad_norm": 1.5913611596440715,
      "learning_rate": 1.2373764497794918e-06,
      "log_odds_chosen": 1.4685547351837158,
      "log_odds_ratio": -0.4230712950229645,
      "logits/chosen": -1.3689453601837158,
      "logits/rejected": -1.123632788658142,
      "logps/chosen": -0.7645508050918579,
      "logps/rejected": -1.821874976158142,
      "loss": 0.944,
      "nll_loss": 0.832226574420929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07645263522863388,
      "rewards/margins": 0.10562743991613388,
      "rewards/rejected": -0.18215331435203552,
      "step": 4180
    },
    {
      "epoch": 0.5412387780145966,
      "grad_norm": 1.7352052444388062,
      "learning_rate": 1.2358989848349217e-06,
      "log_odds_chosen": 1.512304663658142,
      "log_odds_ratio": -0.40620118379592896,
      "logits/chosen": -1.3876953125,
      "logits/rejected": -1.1759765148162842,
      "logps/chosen": -0.7337890863418579,
      "logps/rejected": -1.819921851158142,
      "loss": 0.9578,
      "nll_loss": 0.9482421875,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07333984225988388,
      "rewards/margins": 0.10850830376148224,
      "rewards/rejected": -0.181884765625,
      "step": 4190
    },
    {
      "epoch": 0.5425305173416005,
      "grad_norm": 1.898054555657517,
      "learning_rate": 1.2344267996967353e-06,
      "log_odds_chosen": 1.500390648841858,
      "log_odds_ratio": -0.42182618379592896,
      "logits/chosen": -1.2314453125,
      "logits/rejected": -1.0320312976837158,
      "logps/chosen": -0.7984374761581421,
      "logps/rejected": -1.886328101158142,
      "loss": 0.9512,
      "nll_loss": 0.9468749761581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07988281548023224,
      "rewards/margins": 0.10878906399011612,
      "rewards/rejected": -0.18862304091453552,
      "step": 4200
    },
    {
      "epoch": 0.5438222566686043,
      "grad_norm": 2.396807389377443,
      "learning_rate": 1.2329598629935076e-06,
      "log_odds_chosen": 1.3772461414337158,
      "log_odds_ratio": -0.4112792909145355,
      "logits/chosen": -1.2751953601837158,
      "logits/rejected": -1.117773413658142,
      "logps/chosen": -0.785351574420929,
      "logps/rejected": -1.7878906726837158,
      "loss": 0.9621,
      "nll_loss": 0.875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07851562649011612,
      "rewards/margins": 0.10018310695886612,
      "rewards/rejected": -0.17866210639476776,
      "step": 4210
    },
    {
      "epoch": 0.5451139959956081,
      "grad_norm": 1.6609247306659123,
      "learning_rate": 1.2314981436141583e-06,
      "log_odds_chosen": 1.4660155773162842,
      "log_odds_ratio": -0.42109376192092896,
      "logits/chosen": -1.3015625476837158,
      "logits/rejected": -1.1359374523162842,
      "logps/chosen": -0.756640613079071,
      "logps/rejected": -1.8621094226837158,
      "loss": 0.9654,
      "nll_loss": 0.885937511920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07553710788488388,
      "rewards/margins": 0.11065368354320526,
      "rewards/rejected": -0.18632812798023224,
      "step": 4220
    },
    {
      "epoch": 0.5464057353226119,
      "grad_norm": 1.6499107825260557,
      "learning_rate": 1.2300416107051802e-06,
      "log_odds_chosen": 1.591406226158142,
      "log_odds_ratio": -0.3929199278354645,
      "logits/chosen": -1.341796875,
      "logits/rejected": -1.0964844226837158,
      "logps/chosen": -0.735156238079071,
      "logps/rejected": -1.9308593273162842,
      "loss": 0.9433,
      "nll_loss": 0.9150390625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07354736328125,
      "rewards/margins": 0.1195068359375,
      "rewards/rejected": -0.19291992485523224,
      "step": 4230
    },
    {
      "epoch": 0.5476974746496157,
      "grad_norm": 1.7219099630672494,
      "learning_rate": 1.2285902336679024e-06,
      "log_odds_chosen": 1.3551757335662842,
      "log_odds_ratio": -0.424072265625,
      "logits/chosen": -1.2333984375,
      "logits/rejected": -1.0771484375,
      "logps/chosen": -0.749218761920929,
      "logps/rejected": -1.746484398841858,
      "loss": 0.9381,
      "nll_loss": 0.9624999761581421,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07486572116613388,
      "rewards/margins": 0.09979248046875,
      "rewards/rejected": -0.17475585639476776,
      "step": 4240
    },
    {
      "epoch": 0.5489892139766195,
      "grad_norm": 2.2717361755641443,
      "learning_rate": 1.2271439821557926e-06,
      "log_odds_chosen": 1.736718773841858,
      "log_odds_ratio": -0.345703125,
      "logits/chosen": -1.2970702648162842,
      "logits/rejected": -1.0935547351837158,
      "logps/chosen": -0.713085949420929,
      "logps/rejected": -2.0142579078674316,
      "loss": 0.9433,
      "nll_loss": 0.85546875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07127685844898224,
      "rewards/margins": 0.13004150986671448,
      "rewards/rejected": -0.20131835341453552,
      "step": 4250
    },
    {
      "epoch": 0.5502809533036234,
      "grad_norm": 1.6711203087525692,
      "learning_rate": 1.225702826071791e-06,
      "log_odds_chosen": 1.5569336414337158,
      "log_odds_ratio": -0.38579100370407104,
      "logits/chosen": -1.3537108898162842,
      "logits/rejected": -1.1496093273162842,
      "logps/chosen": -0.748046875,
      "logps/rejected": -1.888671875,
      "loss": 0.9537,
      "nll_loss": 0.892382800579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07471923530101776,
      "rewards/margins": 0.11398925632238388,
      "rewards/rejected": -0.18891601264476776,
      "step": 4260
    },
    {
      "epoch": 0.5515726926306271,
      "grad_norm": 1.9465195082907079,
      "learning_rate": 1.2242667355656797e-06,
      "log_odds_chosen": 1.61328125,
      "log_odds_ratio": -0.37275391817092896,
      "logits/chosen": -1.3542969226837158,
      "logits/rejected": -1.0812499523162842,
      "logps/chosen": -0.722460925579071,
      "logps/rejected": -1.923437476158142,
      "loss": 0.964,
      "nll_loss": 0.8935546875,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07229004055261612,
      "rewards/margins": 0.12009277194738388,
      "rewards/rejected": -0.19240722060203552,
      "step": 4270
    },
    {
      "epoch": 0.5528644319576309,
      "grad_norm": 1.794022416802587,
      "learning_rate": 1.2228356810314862e-06,
      "log_odds_chosen": 1.548120141029358,
      "log_odds_ratio": -0.38994139432907104,
      "logits/chosen": -1.158789038658142,
      "logits/rejected": -1.0666015148162842,
      "logps/chosen": -0.7455078363418579,
      "logps/rejected": -1.8976562023162842,
      "loss": 0.9748,
      "nll_loss": 0.9507812261581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0745849609375,
      "rewards/margins": 0.11509399116039276,
      "rewards/rejected": -0.18984374403953552,
      "step": 4280
    },
    {
      "epoch": 0.5541561712846348,
      "grad_norm": 1.667794541601779,
      "learning_rate": 1.2214096331049186e-06,
      "log_odds_chosen": 1.586572289466858,
      "log_odds_ratio": -0.42426759004592896,
      "logits/chosen": -1.2902343273162842,
      "logits/rejected": -1.0851562023162842,
      "logps/chosen": -0.7464843988418579,
      "logps/rejected": -1.9328124523162842,
      "loss": 0.9303,
      "nll_loss": 0.8729492425918579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07462158054113388,
      "rewards/margins": 0.11842040717601776,
      "rewards/rejected": -0.19316406548023224,
      "step": 4290
    },
    {
      "epoch": 0.5554479106116386,
      "grad_norm": 5.7679672973402765,
      "learning_rate": 1.2199885626608373e-06,
      "log_odds_chosen": 1.5720703601837158,
      "log_odds_ratio": -0.37666016817092896,
      "logits/chosen": -1.191796898841858,
      "logits/rejected": -1.029687523841858,
      "logps/chosen": -0.739453136920929,
      "logps/rejected": -1.881250023841858,
      "loss": 0.9693,
      "nll_loss": 0.884570300579071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.07392577826976776,
      "rewards/margins": 0.11418457329273224,
      "rewards/rejected": -0.18825682997703552,
      "step": 4300
    },
    {
      "epoch": 0.5567396499386423,
      "grad_norm": 2.352795709971135,
      "learning_rate": 1.2185724408107546e-06,
      "log_odds_chosen": 1.9900391101837158,
      "log_odds_ratio": -0.3060058653354645,
      "logits/chosen": -1.398828148841858,
      "logits/rejected": -1.124414086341858,
      "logps/chosen": -0.694140613079071,
      "logps/rejected": -2.139843702316284,
      "loss": 0.9342,
      "nll_loss": 0.896484375,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06944580376148224,
      "rewards/margins": 0.14444580674171448,
      "rewards/rejected": -0.21389159560203552,
      "step": 4310
    },
    {
      "epoch": 0.5580313892656462,
      "grad_norm": 1.8931088376604501,
      "learning_rate": 1.2171612389003689e-06,
      "log_odds_chosen": 1.6291015148162842,
      "log_odds_ratio": -0.37236326932907104,
      "logits/chosen": -1.2666015625,
      "logits/rejected": -1.081445336341858,
      "logps/chosen": -0.675488293170929,
      "logps/rejected": -1.8025391101837158,
      "loss": 0.9251,
      "nll_loss": 0.8734375238418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06755371391773224,
      "rewards/margins": 0.11271362006664276,
      "rewards/rejected": -0.18020018935203552,
      "step": 4320
    },
    {
      "epoch": 0.55932312859265,
      "grad_norm": 1.591707692558663,
      "learning_rate": 1.2157549285071297e-06,
      "log_odds_chosen": 1.5688965320587158,
      "log_odds_ratio": -0.37128907442092896,
      "logits/chosen": -1.2509765625,
      "logits/rejected": -1.032617211341858,
      "logps/chosen": -0.707226574420929,
      "logps/rejected": -1.838281273841858,
      "loss": 0.9799,
      "nll_loss": 0.903515636920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07075195014476776,
      "rewards/margins": 0.11311187595129013,
      "rewards/rejected": -0.18386229872703552,
      "step": 4330
    },
    {
      "epoch": 0.5606148679196538,
      "grad_norm": 1.5906474756638334,
      "learning_rate": 1.2143534814378327e-06,
      "log_odds_chosen": 1.462890625,
      "log_odds_ratio": -0.3943847715854645,
      "logits/chosen": -1.361328125,
      "logits/rejected": -1.0583984851837158,
      "logps/chosen": -0.775585949420929,
      "logps/rejected": -1.8507812023162842,
      "loss": 0.9685,
      "nll_loss": 0.9234374761581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.0775146484375,
      "rewards/margins": 0.10768432915210724,
      "rewards/rejected": -0.18525390326976776,
      "step": 4340
    },
    {
      "epoch": 0.5619066072466576,
      "grad_norm": 1.5883088992798324,
      "learning_rate": 1.2129568697262454e-06,
      "log_odds_chosen": 1.5482177734375,
      "log_odds_ratio": -0.4226318299770355,
      "logits/chosen": -1.2458984851837158,
      "logits/rejected": -1.0382812023162842,
      "logps/chosen": -0.7261718511581421,
      "logps/rejected": -1.8603515625,
      "loss": 0.9546,
      "nll_loss": 0.9345703125,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07260742038488388,
      "rewards/margins": 0.11366577446460724,
      "rewards/rejected": -0.18620605766773224,
      "step": 4350
    },
    {
      "epoch": 0.5631983465736614,
      "grad_norm": 1.88948530137162,
      "learning_rate": 1.2115650656307653e-06,
      "log_odds_chosen": 1.5812499523162842,
      "log_odds_ratio": -0.39936524629592896,
      "logits/chosen": -1.4187500476837158,
      "logits/rejected": -1.186914086341858,
      "logps/chosen": -0.7251952886581421,
      "logps/rejected": -1.8468749523162842,
      "loss": 0.9145,
      "nll_loss": 0.8501952886581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07243652641773224,
      "rewards/margins": 0.11224365234375,
      "rewards/rejected": -0.18479004502296448,
      "step": 4360
    },
    {
      "epoch": 0.5644900859006653,
      "grad_norm": 1.585683218221428,
      "learning_rate": 1.210178041632103e-06,
      "log_odds_chosen": 1.537988305091858,
      "log_odds_ratio": -0.390869140625,
      "logits/chosen": -1.225195288658142,
      "logits/rejected": -1.0779297351837158,
      "logps/chosen": -0.7337890863418579,
      "logps/rejected": -1.8644530773162842,
      "loss": 0.9384,
      "nll_loss": 0.992382824420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07340087741613388,
      "rewards/margins": 0.11305542290210724,
      "rewards/rejected": -0.1864013671875,
      "step": 4370
    },
    {
      "epoch": 0.5657818252276691,
      "grad_norm": 2.710797695816908,
      "learning_rate": 1.2087957704309988e-06,
      "log_odds_chosen": 1.834570288658142,
      "log_odds_ratio": -0.3258056640625,
      "logits/chosen": -1.393945336341858,
      "logits/rejected": -1.110742211341858,
      "logps/chosen": -0.7353515625,
      "logps/rejected": -2.06640625,
      "loss": 0.9275,
      "nll_loss": 0.8570312261581421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.07349853217601776,
      "rewards/margins": 0.13310547173023224,
      "rewards/rejected": -0.20668944716453552,
      "step": 4380
    },
    {
      "epoch": 0.5670735645546728,
      "grad_norm": 1.6428952259274932,
      "learning_rate": 1.2074182249459642e-06,
      "log_odds_chosen": 1.867578148841858,
      "log_odds_ratio": -0.311767578125,
      "logits/chosen": -1.3224608898162842,
      "logits/rejected": -1.067968726158142,
      "logps/chosen": -0.703320324420929,
      "logps/rejected": -2.083203077316284,
      "loss": 0.9314,
      "nll_loss": 0.874218761920929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07038573920726776,
      "rewards/margins": 0.13797608017921448,
      "rewards/rejected": -0.20842285454273224,
      "step": 4390
    },
    {
      "epoch": 0.5683653038816767,
      "grad_norm": 1.6735372722272903,
      "learning_rate": 1.2060453783110545e-06,
      "log_odds_chosen": 1.7418944835662842,
      "log_odds_ratio": -0.387939453125,
      "logits/chosen": -1.257421851158142,
      "logits/rejected": -1.0076172351837158,
      "logps/chosen": -0.6776367425918579,
      "logps/rejected": -1.98046875,
      "loss": 0.9327,
      "nll_loss": 0.9339843988418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0677490234375,
      "rewards/margins": 0.13032837212085724,
      "rewards/rejected": -0.19802245497703552,
      "step": 4400
    },
    {
      "epoch": 0.5696570432086805,
      "grad_norm": 1.9883383046376895,
      "learning_rate": 1.2046772038736682e-06,
      "log_odds_chosen": 1.7687499523162842,
      "log_odds_ratio": -0.3750244081020355,
      "logits/chosen": -1.3603515625,
      "logits/rejected": -1.196874976158142,
      "logps/chosen": -0.792773425579071,
      "logps/rejected": -2.0894532203674316,
      "loss": 0.9221,
      "nll_loss": 0.8646484613418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07923583686351776,
      "rewards/margins": 0.12967529892921448,
      "rewards/rejected": -0.208984375,
      "step": 4410
    },
    {
      "epoch": 0.5709487825356843,
      "grad_norm": 2.0432023907066434,
      "learning_rate": 1.2033136751923736e-06,
      "log_odds_chosen": 1.441430687904358,
      "log_odds_ratio": -0.42192381620407104,
      "logits/chosen": -1.253320336341858,
      "logits/rejected": -1.0900390148162842,
      "logps/chosen": -0.7984374761581421,
      "logps/rejected": -1.90234375,
      "loss": 0.9264,
      "nll_loss": 0.8695312738418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07988281548023224,
      "rewards/margins": 0.11024627834558487,
      "rewards/rejected": -0.19028320908546448,
      "step": 4420
    },
    {
      "epoch": 0.5722405218626881,
      "grad_norm": 1.709445948911892,
      "learning_rate": 1.201954766034762e-06,
      "log_odds_chosen": 1.551855444908142,
      "log_odds_ratio": -0.40380859375,
      "logits/chosen": -1.279687523841858,
      "logits/rejected": -1.1318359375,
      "logps/chosen": -0.7669922113418579,
      "logps/rejected": -1.897851586341858,
      "loss": 0.9682,
      "nll_loss": 0.917187511920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07667236030101776,
      "rewards/margins": 0.11307372897863388,
      "rewards/rejected": -0.1898193359375,
      "step": 4430
    },
    {
      "epoch": 0.5735322611896919,
      "grad_norm": 1.4798049494038241,
      "learning_rate": 1.2006004503753285e-06,
      "log_odds_chosen": 1.487695336341858,
      "log_odds_ratio": -0.41552734375,
      "logits/chosen": -1.2332031726837158,
      "logits/rejected": -1.080468773841858,
      "logps/chosen": -0.755078136920929,
      "logps/rejected": -1.828515648841858,
      "loss": 0.9445,
      "nll_loss": 0.9351562261581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07550048828125,
      "rewards/margins": 0.10734252631664276,
      "rewards/rejected": -0.18293456733226776,
      "step": 4440
    },
    {
      "epoch": 0.5748240005166957,
      "grad_norm": 2.771084764429539,
      "learning_rate": 1.1992507023933782e-06,
      "log_odds_chosen": 1.721289038658142,
      "log_odds_ratio": -0.38969725370407104,
      "logits/chosen": -1.3406250476837158,
      "logits/rejected": -1.025976538658142,
      "logps/chosen": -0.7347656488418579,
      "logps/rejected": -2.0511717796325684,
      "loss": 0.9442,
      "nll_loss": 0.931835949420929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07351074367761612,
      "rewards/margins": 0.13165283203125,
      "rewards/rejected": -0.20510253310203552,
      "step": 4450
    },
    {
      "epoch": 0.5761157398436996,
      "grad_norm": 1.7271424290403872,
      "learning_rate": 1.1979054964709597e-06,
      "log_odds_chosen": 1.4455077648162842,
      "log_odds_ratio": -0.4397216737270355,
      "logits/chosen": -1.1970703601837158,
      "logits/rejected": -1.0613281726837158,
      "logps/chosen": -0.72021484375,
      "logps/rejected": -1.765234351158142,
      "loss": 0.9,
      "nll_loss": 0.875195324420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07203368842601776,
      "rewards/margins": 0.10458984225988388,
      "rewards/rejected": -0.17653807997703552,
      "step": 4460
    },
    {
      "epoch": 0.5774074791707033,
      "grad_norm": 2.0017667214720296,
      "learning_rate": 1.1965648071908207e-06,
      "log_odds_chosen": 1.65582275390625,
      "log_odds_ratio": -0.39360350370407104,
      "logits/chosen": -1.3914062976837158,
      "logits/rejected": -1.145898461341858,
      "logps/chosen": -0.713085949420929,
      "logps/rejected": -1.9562499523162842,
      "loss": 0.9397,
      "nll_loss": 0.8662109375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07130126655101776,
      "rewards/margins": 0.12435150146484375,
      "rewards/rejected": -0.19558104872703552,
      "step": 4470
    },
    {
      "epoch": 0.5786992184977071,
      "grad_norm": 1.6552600794243664,
      "learning_rate": 1.1952286093343935e-06,
      "log_odds_chosen": 1.548828125,
      "log_odds_ratio": -0.39306640625,
      "logits/chosen": -1.339453101158142,
      "logits/rejected": -1.1443359851837158,
      "logps/chosen": -0.7134765386581421,
      "logps/rejected": -1.857031226158142,
      "loss": 0.9312,
      "nll_loss": 0.866406261920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07139892876148224,
      "rewards/margins": 0.11435546725988388,
      "rewards/rejected": -0.18581542372703552,
      "step": 4480
    },
    {
      "epoch": 0.579990957824711,
      "grad_norm": 1.5780722160872864,
      "learning_rate": 1.1938968778798005e-06,
      "log_odds_chosen": 1.529882788658142,
      "log_odds_ratio": -0.38593751192092896,
      "logits/chosen": -1.2482421398162842,
      "logits/rejected": -1.052148461341858,
      "logps/chosen": -0.7484375238418579,
      "logps/rejected": -1.857421875,
      "loss": 0.9343,
      "nll_loss": 0.923632800579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07489013671875,
      "rewards/margins": 0.11077880859375,
      "rewards/rejected": -0.18557128310203552,
      "step": 4490
    },
    {
      "epoch": 0.5812826971517148,
      "grad_norm": 1.6172753868208591,
      "learning_rate": 1.1925695879998878e-06,
      "log_odds_chosen": 1.399511694908142,
      "log_odds_ratio": -0.4425048828125,
      "logits/chosen": -1.297460913658142,
      "logits/rejected": -1.0802733898162842,
      "logps/chosen": -0.810742199420929,
      "logps/rejected": -1.8859374523162842,
      "loss": 0.9451,
      "nll_loss": 0.9740234613418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.08107910305261612,
      "rewards/margins": 0.10748291015625,
      "rewards/rejected": -0.1885986328125,
      "step": 4500
    },
    {
      "epoch": 0.5825744364787186,
      "grad_norm": 1.9126350428215306,
      "learning_rate": 1.1912467150602794e-06,
      "log_odds_chosen": 1.795507788658142,
      "log_odds_ratio": -0.3241943418979645,
      "logits/chosen": -1.3220703601837158,
      "logits/rejected": -1.0419921875,
      "logps/chosen": -0.720898449420929,
      "logps/rejected": -2.0542969703674316,
      "loss": 0.9168,
      "nll_loss": 0.90087890625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.07216797024011612,
      "rewards/margins": 0.13309936225414276,
      "rewards/rejected": -0.205322265625,
      "step": 4510
    },
    {
      "epoch": 0.5838661758057224,
      "grad_norm": 1.8376299963777751,
      "learning_rate": 1.189928234617459e-06,
      "log_odds_chosen": 1.4835937023162842,
      "log_odds_ratio": -0.43798828125,
      "logits/chosen": -1.2980468273162842,
      "logits/rejected": -1.058007836341858,
      "logps/chosen": -0.7933593988418579,
      "logps/rejected": -1.8576171398162842,
      "loss": 0.9321,
      "nll_loss": 0.8760741949081421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.079345703125,
      "rewards/margins": 0.1065673828125,
      "rewards/rejected": -0.18576660752296448,
      "step": 4520
    },
    {
      "epoch": 0.5851579151327262,
      "grad_norm": 1.7383810518158855,
      "learning_rate": 1.1886141224168716e-06,
      "log_odds_chosen": 1.754492163658142,
      "log_odds_ratio": -0.33447265625,
      "logits/chosen": -1.2548828125,
      "logits/rejected": -1.0558593273162842,
      "logps/chosen": -0.7138671875,
      "logps/rejected": -2.014453172683716,
      "loss": 0.9516,
      "nll_loss": 0.865039050579071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.07133789360523224,
      "rewards/margins": 0.13015136122703552,
      "rewards/rejected": -0.20161132514476776,
      "step": 4530
    },
    {
      "epoch": 0.5864496544597301,
      "grad_norm": 1.9270731836682518,
      "learning_rate": 1.1873043543910495e-06,
      "log_odds_chosen": 1.409326195716858,
      "log_odds_ratio": -0.42426759004592896,
      "logits/chosen": -1.2667968273162842,
      "logits/rejected": -1.130273461341858,
      "logps/chosen": -0.7564452886581421,
      "logps/rejected": -1.765625,
      "loss": 0.9565,
      "nll_loss": 0.9164062738418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07564697414636612,
      "rewards/margins": 0.10087432712316513,
      "rewards/rejected": -0.17641600966453552,
      "step": 4540
    },
    {
      "epoch": 0.5877413937867338,
      "grad_norm": 2.0481071458390323,
      "learning_rate": 1.1859989066577617e-06,
      "log_odds_chosen": 1.7667968273162842,
      "log_odds_ratio": -0.36767578125,
      "logits/chosen": -1.3308594226837158,
      "logits/rejected": -1.044921875,
      "logps/chosen": -0.7632812261581421,
      "logps/rejected": -2.0738282203674316,
      "loss": 0.9183,
      "nll_loss": 0.9144531488418579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.07630614936351776,
      "rewards/margins": 0.13126221299171448,
      "rewards/rejected": -0.20742186903953552,
      "step": 4550
    },
    {
      "epoch": 0.5890331331137376,
      "grad_norm": 1.7883800871026392,
      "learning_rate": 1.1846977555181846e-06,
      "log_odds_chosen": 1.733789086341858,
      "log_odds_ratio": -0.3633789122104645,
      "logits/chosen": -1.287500023841858,
      "logits/rejected": -1.088281273841858,
      "logps/chosen": -0.69677734375,
      "logps/rejected": -1.9375,
      "loss": 0.9448,
      "nll_loss": 0.8607422113418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06968994438648224,
      "rewards/margins": 0.12413330376148224,
      "rewards/rejected": -0.19401855766773224,
      "step": 4560
    },
    {
      "epoch": 0.5903248724407415,
      "grad_norm": 5.853994795389135,
      "learning_rate": 1.1834008774550946e-06,
      "log_odds_chosen": 1.587304711341858,
      "log_odds_ratio": -0.3641601502895355,
      "logits/chosen": -1.375,
      "logits/rejected": -1.1164062023162842,
      "logps/chosen": -0.6874023675918579,
      "logps/rejected": -1.845703125,
      "loss": 0.9285,
      "nll_loss": 0.7808593511581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06873778998851776,
      "rewards/margins": 0.11588134616613388,
      "rewards/rejected": -0.18459472060203552,
      "step": 4570
    },
    {
      "epoch": 0.5916166117677453,
      "grad_norm": 1.6588117029178708,
      "learning_rate": 1.1821082491310835e-06,
      "log_odds_chosen": 1.680273413658142,
      "log_odds_ratio": -0.39702147245407104,
      "logits/chosen": -1.396875023841858,
      "logits/rejected": -1.162109375,
      "logps/chosen": -0.7720702886581421,
      "logps/rejected": -2.0433592796325684,
      "loss": 0.9621,
      "nll_loss": 0.951367199420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07716064155101776,
      "rewards/margins": 0.12718506157398224,
      "rewards/rejected": -0.20424804091453552,
      "step": 4580
    },
    {
      "epoch": 0.592908351094749,
      "grad_norm": 1.7284757683678504,
      "learning_rate": 1.1808198473867937e-06,
      "log_odds_chosen": 1.6033203601837158,
      "log_odds_ratio": -0.3680664002895355,
      "logits/chosen": -1.314453125,
      "logits/rejected": -1.118749976158142,
      "logps/chosen": -0.715039074420929,
      "logps/rejected": -1.8933594226837158,
      "loss": 0.9396,
      "nll_loss": 0.826367199420929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07149658352136612,
      "rewards/margins": 0.11781005561351776,
      "rewards/rejected": -0.189453125,
      "step": 4590
    },
    {
      "epoch": 0.5942000904217529,
      "grad_norm": 1.8571080585729525,
      "learning_rate": 1.179535649239177e-06,
      "log_odds_chosen": 1.8361327648162842,
      "log_odds_ratio": -0.3235839903354645,
      "logits/chosen": -1.361914038658142,
      "logits/rejected": -1.0947265625,
      "logps/chosen": -0.735058605670929,
      "logps/rejected": -2.10546875,
      "loss": 0.9338,
      "nll_loss": 0.938281238079071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.07351074367761612,
      "rewards/margins": 0.13679198920726776,
      "rewards/rejected": -0.21040038764476776,
      "step": 4600
    },
    {
      "epoch": 0.5954918297487567,
      "grad_norm": 1.6600973267590446,
      "learning_rate": 1.178255631879771e-06,
      "log_odds_chosen": 1.449853539466858,
      "log_odds_ratio": -0.4580078125,
      "logits/chosen": -1.2898437976837158,
      "logits/rejected": -1.120703101158142,
      "logps/chosen": -0.7720702886581421,
      "logps/rejected": -1.820703148841858,
      "loss": 0.9408,
      "nll_loss": 0.9009765386581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07707519829273224,
      "rewards/margins": 0.10495300590991974,
      "rewards/rejected": -0.18203124403953552,
      "step": 4610
    },
    {
      "epoch": 0.5967835690757605,
      "grad_norm": 1.8433355547136911,
      "learning_rate": 1.1769797726729992e-06,
      "log_odds_chosen": 1.3949463367462158,
      "log_odds_ratio": -0.40888673067092896,
      "logits/chosen": -1.4402344226837158,
      "logits/rejected": -1.2185547351837158,
      "logps/chosen": -0.705859363079071,
      "logps/rejected": -1.7117187976837158,
      "loss": 0.9033,
      "nll_loss": 0.8785156011581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07056884467601776,
      "rewards/margins": 0.10054321587085724,
      "rewards/rejected": -0.17124024033546448,
      "step": 4620
    },
    {
      "epoch": 0.5980753084027643,
      "grad_norm": 2.1071194219845824,
      "learning_rate": 1.1757080491544881e-06,
      "log_odds_chosen": 1.3976562023162842,
      "log_odds_ratio": -0.38671875,
      "logits/chosen": -1.275390625,
      "logits/rejected": -1.058984398841858,
      "logps/chosen": -0.7533203363418579,
      "logps/rejected": -1.750390648841858,
      "loss": 0.9568,
      "nll_loss": 0.961621105670929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07528076320886612,
      "rewards/margins": 0.09956512600183487,
      "rewards/rejected": -0.17487792670726776,
      "step": 4630
    },
    {
      "epoch": 0.5993670477297681,
      "grad_norm": 1.7164512041798885,
      "learning_rate": 1.1744404390294068e-06,
      "log_odds_chosen": 1.392431616783142,
      "log_odds_ratio": -0.4547363221645355,
      "logits/chosen": -1.377539038658142,
      "logits/rejected": -1.187890648841858,
      "logps/chosen": -0.775585949420929,
      "logps/rejected": -1.8341796398162842,
      "loss": 0.9576,
      "nll_loss": 0.8851562738418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07756347954273224,
      "rewards/margins": 0.10582885891199112,
      "rewards/rejected": -0.18354491889476776,
      "step": 4640
    },
    {
      "epoch": 0.6006587870567719,
      "grad_norm": 1.7598338518007444,
      "learning_rate": 1.1731769201708264e-06,
      "log_odds_chosen": 1.6369140148162842,
      "log_odds_ratio": -0.3788818418979645,
      "logits/chosen": -1.3781249523162842,
      "logits/rejected": -1.095312476158142,
      "logps/chosen": -0.7300781011581421,
      "logps/rejected": -1.9070312976837158,
      "loss": 0.9299,
      "nll_loss": 0.854687511920929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.072998046875,
      "rewards/margins": 0.11772765964269638,
      "rewards/rejected": -0.19052734971046448,
      "step": 4650
    },
    {
      "epoch": 0.6019505263837758,
      "grad_norm": 1.8652940546569594,
      "learning_rate": 1.1719174706180952e-06,
      "log_odds_chosen": 1.6630859375,
      "log_odds_ratio": -0.36518555879592896,
      "logits/chosen": -1.259374976158142,
      "logits/rejected": -1.073632836341858,
      "logps/chosen": -0.712695300579071,
      "logps/rejected": -1.9132812023162842,
      "loss": 0.9401,
      "nll_loss": 0.902539074420929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07132568210363388,
      "rewards/margins": 0.12020263820886612,
      "rewards/rejected": -0.1912841796875,
      "step": 4660
    },
    {
      "epoch": 0.6032422657107795,
      "grad_norm": 1.6835402672395174,
      "learning_rate": 1.1706620685752386e-06,
      "log_odds_chosen": 1.418359398841858,
      "log_odds_ratio": -0.3988281190395355,
      "logits/chosen": -1.347265601158142,
      "logits/rejected": -1.161718726158142,
      "logps/chosen": -0.8218749761581421,
      "logps/rejected": -1.883203148841858,
      "loss": 0.9281,
      "nll_loss": 0.9017578363418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.08214111626148224,
      "rewards/margins": 0.10606689751148224,
      "rewards/rejected": -0.18825682997703552,
      "step": 4670
    },
    {
      "epoch": 0.6045340050377834,
      "grad_norm": 1.6475924030731275,
      "learning_rate": 1.1694106924093723e-06,
      "log_odds_chosen": 1.669335961341858,
      "log_odds_ratio": -0.3777099549770355,
      "logits/chosen": -1.3615233898162842,
      "logits/rejected": -1.145898461341858,
      "logps/chosen": -0.7308593988418579,
      "logps/rejected": -1.9775390625,
      "loss": 0.9672,
      "nll_loss": 0.929492175579071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07308349758386612,
      "rewards/margins": 0.1246337890625,
      "rewards/rejected": -0.19775390625,
      "step": 4680
    },
    {
      "epoch": 0.6058257443647872,
      "grad_norm": 2.1634850586313457,
      "learning_rate": 1.1681633206491381e-06,
      "log_odds_chosen": 1.429296851158142,
      "log_odds_ratio": -0.4195312559604645,
      "logits/chosen": -1.3583984375,
      "logits/rejected": -1.16015625,
      "logps/chosen": -0.7787109613418579,
      "logps/rejected": -1.825781226158142,
      "loss": 0.9211,
      "nll_loss": 0.863085925579071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07784423977136612,
      "rewards/margins": 0.10484619438648224,
      "rewards/rejected": -0.18264159560203552,
      "step": 4690
    },
    {
      "epoch": 0.607117483691791,
      "grad_norm": 1.9414226751309138,
      "learning_rate": 1.1669199319831564e-06,
      "log_odds_chosen": 1.534277319908142,
      "log_odds_ratio": -0.38422852754592896,
      "logits/chosen": -1.3107421398162842,
      "logits/rejected": -1.0558593273162842,
      "logps/chosen": -0.830273449420929,
      "logps/rejected": -1.962499976158142,
      "loss": 0.929,
      "nll_loss": 0.931640625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.08302001655101776,
      "rewards/margins": 0.11324463039636612,
      "rewards/rejected": -0.19624023139476776,
      "step": 4700
    },
    {
      "epoch": 0.6084092230187949,
      "grad_norm": 1.9269438778221415,
      "learning_rate": 1.1656805052584958e-06,
      "log_odds_chosen": 1.4923827648162842,
      "log_odds_ratio": -0.398193359375,
      "logits/chosen": -1.2468750476837158,
      "logits/rejected": -1.097265601158142,
      "logps/chosen": -0.7474609613418579,
      "logps/rejected": -1.818359375,
      "loss": 0.9337,
      "nll_loss": 0.889453113079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07470703125,
      "rewards/margins": 0.10718993842601776,
      "rewards/rejected": -0.18181152641773224,
      "step": 4710
    },
    {
      "epoch": 0.6097009623457986,
      "grad_norm": 1.4827038793621574,
      "learning_rate": 1.164445019479164e-06,
      "log_odds_chosen": 1.3251464366912842,
      "log_odds_ratio": -0.44194334745407104,
      "logits/chosen": -1.3642578125,
      "logits/rejected": -1.1603515148162842,
      "logps/chosen": -0.7841796875,
      "logps/rejected": -1.741796851158142,
      "loss": 0.9373,
      "nll_loss": 0.8958984613418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07841797173023224,
      "rewards/margins": 0.0958709716796875,
      "rewards/rejected": -0.17421874403953552,
      "step": 4720
    },
    {
      "epoch": 0.6109927016728024,
      "grad_norm": 1.521194017145698,
      "learning_rate": 1.1632134538046105e-06,
      "log_odds_chosen": 1.294335961341858,
      "log_odds_ratio": -0.4267578125,
      "logits/chosen": -1.423242211341858,
      "logits/rejected": -1.2599608898162842,
      "logps/chosen": -0.6556640863418579,
      "logps/rejected": -1.562109351158142,
      "loss": 0.9216,
      "nll_loss": 0.846386730670929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06552734225988388,
      "rewards/margins": 0.09072265774011612,
      "rewards/rejected": -0.15622559189796448,
      "step": 4730
    },
    {
      "epoch": 0.6122844409998063,
      "grad_norm": 1.5106667448677706,
      "learning_rate": 1.1619857875482536e-06,
      "log_odds_chosen": 1.4130859375,
      "log_odds_ratio": -0.4093261659145355,
      "logits/chosen": -1.2595703601837158,
      "logits/rejected": -1.0859375,
      "logps/chosen": -0.748046875,
      "logps/rejected": -1.7863280773162842,
      "loss": 0.954,
      "nll_loss": 0.9212890863418579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.07484130561351776,
      "rewards/margins": 0.10380248725414276,
      "rewards/rejected": -0.17868652939796448,
      "step": 4740
    },
    {
      "epoch": 0.61357618032681,
      "grad_norm": 1.5678402091733594,
      "learning_rate": 1.1607620001760185e-06,
      "log_odds_chosen": 1.454003930091858,
      "log_odds_ratio": -0.4056152403354645,
      "logits/chosen": -1.4031250476837158,
      "logits/rejected": -1.168554663658142,
      "logps/chosen": -0.693359375,
      "logps/rejected": -1.738671898841858,
      "loss": 0.9266,
      "nll_loss": 0.8882812261581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06936035305261612,
      "rewards/margins": 0.10445556789636612,
      "rewards/rejected": -0.17375488579273224,
      "step": 4750
    },
    {
      "epoch": 0.6148679196538138,
      "grad_norm": 2.0045427333252657,
      "learning_rate": 1.1595420713048968e-06,
      "log_odds_chosen": 1.432031273841858,
      "log_odds_ratio": -0.4193359315395355,
      "logits/chosen": -1.325781226158142,
      "logits/rejected": -1.18359375,
      "logps/chosen": -0.7109375,
      "logps/rejected": -1.766015648841858,
      "loss": 0.9417,
      "nll_loss": 0.8802734613418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07110595703125,
      "rewards/margins": 0.10545654594898224,
      "rewards/rejected": -0.17666015028953552,
      "step": 4760
    },
    {
      "epoch": 0.6161596589808177,
      "grad_norm": 1.7544758887761251,
      "learning_rate": 1.1583259807015182e-06,
      "log_odds_chosen": 1.470117211341858,
      "log_odds_ratio": -0.39306640625,
      "logits/chosen": -1.330664038658142,
      "logits/rejected": -1.078515648841858,
      "logps/chosen": -0.6883789300918579,
      "logps/rejected": -1.7570312023162842,
      "loss": 0.9147,
      "nll_loss": 0.8861328363418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06884765625,
      "rewards/margins": 0.10677490383386612,
      "rewards/rejected": -0.17558594048023224,
      "step": 4770
    },
    {
      "epoch": 0.6174513983078215,
      "grad_norm": 2.061701028402448,
      "learning_rate": 1.1571137082807434e-06,
      "log_odds_chosen": 1.318212866783142,
      "log_odds_ratio": -0.446044921875,
      "logits/chosen": -1.3771483898162842,
      "logits/rejected": -1.1638672351837158,
      "logps/chosen": -0.768359363079071,
      "logps/rejected": -1.713281273841858,
      "loss": 0.9151,
      "nll_loss": 0.9048827886581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07685546576976776,
      "rewards/margins": 0.09467468410730362,
      "rewards/rejected": -0.17148438096046448,
      "step": 4780
    },
    {
      "epoch": 0.6187431376348252,
      "grad_norm": 2.40147734398139,
      "learning_rate": 1.155905234104269e-06,
      "log_odds_chosen": 1.7833983898162842,
      "log_odds_ratio": -0.3466796875,
      "logits/chosen": -1.274804711341858,
      "logits/rejected": -1.042382836341858,
      "logps/chosen": -0.700390636920929,
      "logps/rejected": -2.0082030296325684,
      "loss": 0.965,
      "nll_loss": 0.974609375,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.07001952826976776,
      "rewards/margins": 0.13078613579273224,
      "rewards/rejected": -0.20083007216453552,
      "step": 4790
    },
    {
      "epoch": 0.6200348769618291,
      "grad_norm": 1.725765911582742,
      "learning_rate": 1.1547005383792516e-06,
      "log_odds_chosen": 1.4757201671600342,
      "log_odds_ratio": -0.38720703125,
      "logits/chosen": -1.3132812976837158,
      "logits/rejected": -1.1369140148162842,
      "logps/chosen": -0.701171875,
      "logps/rejected": -1.736328125,
      "loss": 0.9109,
      "nll_loss": 0.807910144329071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07010497897863388,
      "rewards/margins": 0.10360107570886612,
      "rewards/rejected": -0.17365722358226776,
      "step": 4800
    },
    {
      "epoch": 0.6213266162888329,
      "grad_norm": 1.8623256582275152,
      "learning_rate": 1.1534996014569446e-06,
      "log_odds_chosen": 1.824609398841858,
      "log_odds_ratio": -0.34953612089157104,
      "logits/chosen": -1.308007836341858,
      "logits/rejected": -1.0634765625,
      "logps/chosen": -0.6854492425918579,
      "logps/rejected": -2.0171875953674316,
      "loss": 0.912,
      "nll_loss": 0.8739258050918579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06857910007238388,
      "rewards/margins": 0.13327637314796448,
      "rewards/rejected": -0.20195312798023224,
      "step": 4810
    },
    {
      "epoch": 0.6226183556158367,
      "grad_norm": 1.5125635922009868,
      "learning_rate": 1.1523024038313547e-06,
      "log_odds_chosen": 1.3976562023162842,
      "log_odds_ratio": -0.4427246153354645,
      "logits/chosen": -1.2775390148162842,
      "logits/rejected": -1.1337890625,
      "logps/chosen": -0.7425781488418579,
      "logps/rejected": -1.7410156726837158,
      "loss": 0.9233,
      "nll_loss": 0.8642578125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0743408203125,
      "rewards/margins": 0.09991455078125,
      "rewards/rejected": -0.17429199814796448,
      "step": 4820
    },
    {
      "epoch": 0.6239100949428406,
      "grad_norm": 2.9032916169658813,
      "learning_rate": 1.1511089261379083e-06,
      "log_odds_chosen": 1.6281249523162842,
      "log_odds_ratio": -0.3427734375,
      "logits/chosen": -1.2488281726837158,
      "logits/rejected": -1.0564453601837158,
      "logps/chosen": -0.7388671636581421,
      "logps/rejected": -1.913671851158142,
      "loss": 0.9456,
      "nll_loss": 0.945507824420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07386474311351776,
      "rewards/margins": 0.11735840141773224,
      "rewards/rejected": -0.19140625,
      "step": 4830
    },
    {
      "epoch": 0.6252018342698443,
      "grad_norm": 1.6858719081790596,
      "learning_rate": 1.149919149152138e-06,
      "log_odds_chosen": 1.564453125,
      "log_odds_ratio": -0.3506103456020355,
      "logits/chosen": -1.3107421398162842,
      "logits/rejected": -1.106054663658142,
      "logps/chosen": -0.7552734613418579,
      "logps/rejected": -1.869531273841858,
      "loss": 0.925,
      "nll_loss": 0.938281238079071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07559814304113388,
      "rewards/margins": 0.11140136420726776,
      "rewards/rejected": -0.18698731064796448,
      "step": 4840
    },
    {
      "epoch": 0.6264935735968482,
      "grad_norm": 1.6596285003405842,
      "learning_rate": 1.148733053788381e-06,
      "log_odds_chosen": 1.460351586341858,
      "log_odds_ratio": -0.4163574278354645,
      "logits/chosen": -1.324609398841858,
      "logits/rejected": -1.123437523841858,
      "logps/chosen": -0.741406261920929,
      "logps/rejected": -1.80859375,
      "loss": 0.9518,
      "nll_loss": 0.8597656488418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07415771484375,
      "rewards/margins": 0.10672607272863388,
      "rewards/rejected": -0.180908203125,
      "step": 4850
    },
    {
      "epoch": 0.627785312923852,
      "grad_norm": 1.5765557894306583,
      "learning_rate": 1.1475506210984938e-06,
      "log_odds_chosen": 1.466796875,
      "log_odds_ratio": -0.39924317598342896,
      "logits/chosen": -1.2951171398162842,
      "logits/rejected": -1.0642578601837158,
      "logps/chosen": -0.7203124761581421,
      "logps/rejected": -1.7648437023162842,
      "loss": 0.9128,
      "nll_loss": 0.860546886920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07208251953125,
      "rewards/margins": 0.10435791313648224,
      "rewards/rejected": -0.17636719346046448,
      "step": 4860
    },
    {
      "epoch": 0.6290770522508558,
      "grad_norm": 1.9706962212948955,
      "learning_rate": 1.1463718322705807e-06,
      "log_odds_chosen": 1.881250023841858,
      "log_odds_ratio": -0.31660157442092896,
      "logits/chosen": -1.3005859851837158,
      "logits/rejected": -1.0642578601837158,
      "logps/chosen": -0.6656249761581421,
      "logps/rejected": -2.049999952316284,
      "loss": 0.9037,
      "nll_loss": 0.836230456829071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06660155951976776,
      "rewards/margins": 0.13835449516773224,
      "rewards/rejected": -0.20498046278953552,
      "step": 4870
    },
    {
      "epoch": 0.6303687915778596,
      "grad_norm": 1.8196724461562912,
      "learning_rate": 1.1451966686277364e-06,
      "log_odds_chosen": 1.4851562976837158,
      "log_odds_ratio": -0.46043699979782104,
      "logits/chosen": -1.2919921875,
      "logits/rejected": -1.0734374523162842,
      "logps/chosen": -0.7691406011581421,
      "logps/rejected": -1.9207031726837158,
      "loss": 0.9505,
      "nll_loss": 0.948046863079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07692871242761612,
      "rewards/margins": 0.11520004272460938,
      "rewards/rejected": -0.19221191108226776,
      "step": 4880
    },
    {
      "epoch": 0.6316605309048634,
      "grad_norm": 2.042375933031292,
      "learning_rate": 1.1440251116268034e-06,
      "log_odds_chosen": 1.6767578125,
      "log_odds_ratio": -0.3396972715854645,
      "logits/chosen": -1.413671851158142,
      "logits/rejected": -1.1681640148162842,
      "logps/chosen": -0.684765636920929,
      "logps/rejected": -1.8796875476837158,
      "loss": 0.9386,
      "nll_loss": 0.8746093511581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06845702975988388,
      "rewards/margins": 0.11943359673023224,
      "rewards/rejected": -0.1878662109375,
      "step": 4890
    },
    {
      "epoch": 0.6329522702318672,
      "grad_norm": 2.0118674144529534,
      "learning_rate": 1.1428571428571428e-06,
      "log_odds_chosen": 1.6550781726837158,
      "log_odds_ratio": -0.40385740995407104,
      "logits/chosen": -1.259179711341858,
      "logits/rejected": -1.027734398841858,
      "logps/chosen": -0.7489258050918579,
      "logps/rejected": -1.966406226158142,
      "loss": 0.9226,
      "nll_loss": 0.922070324420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0748291015625,
      "rewards/margins": 0.121826171875,
      "rewards/rejected": -0.19672851264476776,
      "step": 4900
    },
    {
      "epoch": 0.6342440095588711,
      "grad_norm": 1.8757638806114496,
      "learning_rate": 1.14169274403942e-06,
      "log_odds_chosen": 1.790429711341858,
      "log_odds_ratio": -0.3604492247104645,
      "logits/chosen": -1.329687476158142,
      "logits/rejected": -1.092382788658142,
      "logps/chosen": -0.7071288824081421,
      "logps/rejected": -2.0230469703674316,
      "loss": 0.9436,
      "nll_loss": 0.9097656011581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07073974609375,
      "rewards/margins": 0.13170166313648224,
      "rewards/rejected": -0.20249024033546448,
      "step": 4910
    },
    {
      "epoch": 0.6355357488858748,
      "grad_norm": 1.644319967283968,
      "learning_rate": 1.140531897024402e-06,
      "log_odds_chosen": 1.878515601158142,
      "log_odds_ratio": -0.370849609375,
      "logits/chosen": -1.389257788658142,
      "logits/rejected": -1.138085961341858,
      "logps/chosen": -0.717578113079071,
      "logps/rejected": -2.132031202316284,
      "loss": 0.9503,
      "nll_loss": 0.9146484136581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07171630859375,
      "rewards/margins": 0.14151611924171448,
      "rewards/rejected": -0.21318359673023224,
      "step": 4920
    },
    {
      "epoch": 0.6368274882128786,
      "grad_norm": 1.7631865809061804,
      "learning_rate": 1.13937458379177e-06,
      "log_odds_chosen": 2.146289110183716,
      "log_odds_ratio": -0.3105835020542145,
      "logits/chosen": -1.375585913658142,
      "logits/rejected": -1.135156273841858,
      "logps/chosen": -0.671582043170929,
      "logps/rejected": -2.291796922683716,
      "loss": 0.9226,
      "nll_loss": 0.86328125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06723632663488388,
      "rewards/margins": 0.16206054389476776,
      "rewards/rejected": -0.22915038466453552,
      "step": 4930
    },
    {
      "epoch": 0.6381192275398825,
      "grad_norm": 2.091247414354018,
      "learning_rate": 1.1382207864489444e-06,
      "log_odds_chosen": 1.555517554283142,
      "log_odds_ratio": -0.42626953125,
      "logits/chosen": -1.416601538658142,
      "logits/rejected": -1.1474609375,
      "logps/chosen": -0.8333984613418579,
      "logps/rejected": -1.999609351158142,
      "loss": 0.9214,
      "nll_loss": 0.8431640863418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.08330078423023224,
      "rewards/margins": 0.11650390923023224,
      "rewards/rejected": -0.19992676377296448,
      "step": 4940
    },
    {
      "epoch": 0.6394109668668863,
      "grad_norm": 1.8288281790598933,
      "learning_rate": 1.1370704872299223e-06,
      "log_odds_chosen": 1.7179687023162842,
      "log_odds_ratio": -0.3648925721645355,
      "logits/chosen": -1.339257836341858,
      "logits/rejected": -1.0861327648162842,
      "logps/chosen": -0.7378906011581421,
      "logps/rejected": -2.022656202316284,
      "loss": 0.923,
      "nll_loss": 0.848437488079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07375488430261612,
      "rewards/margins": 0.12858887016773224,
      "rewards/rejected": -0.20249024033546448,
      "step": 4950
    },
    {
      "epoch": 0.64070270619389,
      "grad_norm": 1.636822763597907,
      "learning_rate": 1.1359236684941295e-06,
      "log_odds_chosen": 1.47509765625,
      "log_odds_ratio": -0.43115234375,
      "logits/chosen": -1.3093750476837158,
      "logits/rejected": -1.1365234851837158,
      "logps/chosen": -0.737597644329071,
      "logps/rejected": -1.814843773841858,
      "loss": 0.9082,
      "nll_loss": 0.940234363079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07376708835363388,
      "rewards/margins": 0.10769347846508026,
      "rewards/rejected": -0.18146972358226776,
      "step": 4960
    },
    {
      "epoch": 0.6419944455208939,
      "grad_norm": 2.025619268737137,
      "learning_rate": 1.1347803127252839e-06,
      "log_odds_chosen": 1.520898461341858,
      "log_odds_ratio": -0.3854003846645355,
      "logits/chosen": -1.2976562976837158,
      "logits/rejected": -1.050390601158142,
      "logps/chosen": -0.7300781011581421,
      "logps/rejected": -1.830078125,
      "loss": 0.9412,
      "nll_loss": 0.862597644329071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07301025092601776,
      "rewards/margins": 0.11009521782398224,
      "rewards/rejected": -0.18310546875,
      "step": 4970
    },
    {
      "epoch": 0.6432861848478977,
      "grad_norm": 1.9565179437503954,
      "learning_rate": 1.1336404025302715e-06,
      "log_odds_chosen": 1.4894530773162842,
      "log_odds_ratio": -0.3836425840854645,
      "logits/chosen": -1.321874976158142,
      "logits/rejected": -1.1414062976837158,
      "logps/chosen": -0.746874988079071,
      "logps/rejected": -1.8582031726837158,
      "loss": 0.9349,
      "nll_loss": 0.928906261920929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07467041164636612,
      "rewards/margins": 0.11126098781824112,
      "rewards/rejected": -0.18593749403953552,
      "step": 4980
    },
    {
      "epoch": 0.6445779241749015,
      "grad_norm": 1.909874702507349,
      "learning_rate": 1.1325039206380352e-06,
      "log_odds_chosen": 1.46337890625,
      "log_odds_ratio": -0.35566407442092896,
      "logits/chosen": -1.300390601158142,
      "logits/rejected": -1.0681641101837158,
      "logps/chosen": -0.712109386920929,
      "logps/rejected": -1.748632788658142,
      "loss": 0.9293,
      "nll_loss": 0.909375011920929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.07120361179113388,
      "rewards/margins": 0.10378418117761612,
      "rewards/rejected": -0.17495116591453552,
      "step": 4990
    },
    {
      "epoch": 0.6458696635019053,
      "grad_norm": 1.8122823557697962,
      "learning_rate": 1.131370849898476e-06,
      "log_odds_chosen": 1.9246094226837158,
      "log_odds_ratio": -0.34248048067092896,
      "logits/chosen": -1.3126952648162842,
      "logits/rejected": -1.139257788658142,
      "logps/chosen": -0.697460949420929,
      "logps/rejected": -2.133984327316284,
      "loss": 0.9455,
      "nll_loss": 0.842578113079071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06972656399011612,
      "rewards/margins": 0.14376220107078552,
      "rewards/rejected": -0.21333007514476776,
      "step": 5000
    },
    {
      "epoch": 0.6458696635019053,
      "eval_log_odds_chosen": 1.242581844329834,
      "eval_log_odds_ratio": -0.47447991371154785,
      "eval_logits/chosen": -1.2279024124145508,
      "eval_logits/rejected": -1.0133243799209595,
      "eval_logps/chosen": -0.8291401863098145,
      "eval_logps/rejected": -1.7743595838546753,
      "eval_loss": 1.2853127717971802,
      "eval_nll_loss": 1.246997594833374,
      "eval_rewards/accuracies": 0.7431685924530029,
      "eval_rewards/chosen": -0.0829220786690712,
      "eval_rewards/margins": 0.09452750533819199,
      "eval_rewards/rejected": -0.1774357259273529,
      "eval_runtime": 920.1418,
      "eval_samples_per_second": 59.778,
      "eval_steps_per_second": 0.935,
      "step": 5000
    }
  ],
  "logging_steps": 10,
  "max_steps": 7741,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 5000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}