{
  "best_metric": 1.188714861869812,
  "best_model_checkpoint": "models/llama3.2-3b-orpo-finegrained/checkpoint-5000",
  "epoch": 0.9999635926748461,
  "eval_steps": 5000,
  "global_step": 13733,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.000728146503076419,
      "grad_norm": 14.924340824624208,
      "learning_rate": 8e-07,
      "log_odds_chosen": 1.0834228992462158,
      "log_odds_ratio": -0.545361340045929,
      "logits/chosen": -1.100000023841858,
      "logits/rejected": -1.107031226158142,
      "logps/chosen": -2.315234422683716,
      "logps/rejected": -3.3148436546325684,
      "loss": 2.444,
      "nll_loss": 2.298046827316284,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.23154297471046448,
      "rewards/margins": 0.10004577785730362,
      "rewards/rejected": -0.33173829317092896,
      "step": 10
    },
    {
      "epoch": 0.001456293006152838,
      "grad_norm": 9.501986434443848,
      "learning_rate": 1.6e-06,
      "log_odds_chosen": 0.542285144329071,
      "log_odds_ratio": -0.7381836175918579,
      "logits/chosen": -1.164453148841858,
      "logits/rejected": -1.117773413658142,
      "logps/chosen": -2.229687452316284,
      "logps/rejected": -2.7249999046325684,
      "loss": 2.3392,
      "nll_loss": 2.376171827316284,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.22294922173023224,
      "rewards/margins": 0.04965515062212944,
      "rewards/rejected": -0.2725585997104645,
      "step": 20
    },
    {
      "epoch": 0.002184439509229257,
      "grad_norm": 3.957850921214621,
      "learning_rate": 2.4e-06,
      "log_odds_chosen": 0.2750610411167145,
      "log_odds_ratio": -0.762890636920929,
      "logits/chosen": -1.276953101158142,
      "logits/rejected": -1.2453124523162842,
      "logps/chosen": -1.824609398841858,
      "logps/rejected": -2.0972657203674316,
      "loss": 1.995,
      "nll_loss": 1.919921875,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.18251952528953552,
      "rewards/margins": 0.02733764611184597,
      "rewards/rejected": -0.21000976860523224,
      "step": 30
    },
    {
      "epoch": 0.002912586012305676,
      "grad_norm": 3.1498721961214073,
      "learning_rate": 3.2e-06,
      "log_odds_chosen": 0.37443846464157104,
      "log_odds_ratio": -0.695996105670929,
      "logits/chosen": -1.064062476158142,
      "logits/rejected": -1.00390625,
      "logps/chosen": -1.605859398841858,
      "logps/rejected": -1.9660155773162842,
      "loss": 1.8378,
      "nll_loss": 1.766015648841858,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.16059570014476776,
      "rewards/margins": 0.03613433986902237,
      "rewards/rejected": -0.19663086533546448,
      "step": 40
    },
    {
      "epoch": 0.003640732515382095,
      "grad_norm": 2.8961977700437127,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.45396727323532104,
      "log_odds_ratio": -0.658496081829071,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.8968750238418579,
      "logps/chosen": -1.48046875,
      "logps/rejected": -1.8855469226837158,
      "loss": 1.7232,
      "nll_loss": 1.7078125476837158,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.1480712890625,
      "rewards/margins": 0.04048766940832138,
      "rewards/rejected": -0.18852539360523224,
      "step": 50
    },
    {
      "epoch": 0.004368879018458514,
      "grad_norm": 2.604769874255877,
      "learning_rate": 4.8e-06,
      "log_odds_chosen": 0.37847900390625,
      "log_odds_ratio": -0.6636718511581421,
      "logits/chosen": -0.991015613079071,
      "logits/rejected": -0.9378906488418579,
      "logps/chosen": -1.3503906726837158,
      "logps/rejected": -1.666015625,
      "loss": 1.6685,
      "nll_loss": 1.569921851158142,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.13503417372703552,
      "rewards/margins": 0.03156738355755806,
      "rewards/rejected": -0.16657714545726776,
      "step": 60
    },
    {
      "epoch": 0.005097025521534933,
      "grad_norm": 2.547787098331564,
      "learning_rate": 5.6e-06,
      "log_odds_chosen": 0.501782238483429,
      "log_odds_ratio": -0.605273425579071,
      "logits/chosen": -1.0753905773162842,
      "logits/rejected": -1.015625,
      "logps/chosen": -1.255859375,
      "logps/rejected": -1.694921851158142,
      "loss": 1.6009,
      "nll_loss": 1.5011718273162842,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.12558594346046448,
      "rewards/margins": 0.04382934421300888,
      "rewards/rejected": -0.16943359375,
      "step": 70
    },
    {
      "epoch": 0.005825172024611352,
      "grad_norm": 2.4866096381563847,
      "learning_rate": 6.4e-06,
      "log_odds_chosen": 0.49003905057907104,
      "log_odds_ratio": -0.6048828363418579,
      "logits/chosen": -1.0732421875,
      "logits/rejected": -0.9947265386581421,
      "logps/chosen": -1.1552734375,
      "logps/rejected": -1.5490233898162842,
      "loss": 1.5947,
      "nll_loss": 1.41796875,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.11552734673023224,
      "rewards/margins": 0.039276123046875,
      "rewards/rejected": -0.15485839545726776,
      "step": 80
    },
    {
      "epoch": 0.006553318527687771,
      "grad_norm": 2.5896986661136934,
      "learning_rate": 7.2e-06,
      "log_odds_chosen": 0.41322022676467896,
      "log_odds_ratio": -0.633105456829071,
      "logits/chosen": -1.0517578125,
      "logits/rejected": -1.012304663658142,
      "logps/chosen": -1.205078125,
      "logps/rejected": -1.5320312976837158,
      "loss": 1.5336,
      "nll_loss": 1.4734375476837158,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12058105319738388,
      "rewards/margins": 0.03266448900103569,
      "rewards/rejected": -0.15310057997703552,
      "step": 90
    },
    {
      "epoch": 0.00728146503076419,
      "grad_norm": 2.520791586033232,
      "learning_rate": 8e-06,
      "log_odds_chosen": 0.500213623046875,
      "log_odds_ratio": -0.58251953125,
      "logits/chosen": -1.0818359851837158,
      "logits/rejected": -1.022070288658142,
      "logps/chosen": -1.125,
      "logps/rejected": -1.508398413658142,
      "loss": 1.517,
      "nll_loss": 1.423828125,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11247558891773224,
      "rewards/margins": 0.03836822509765625,
      "rewards/rejected": -0.15095214545726776,
      "step": 100
    },
    {
      "epoch": 0.008009611533840609,
      "grad_norm": 1.8027476277737309,
      "learning_rate": 7.627700713964738e-06,
      "log_odds_chosen": 0.45947265625,
      "log_odds_ratio": -0.6207031011581421,
      "logits/chosen": -0.997265636920929,
      "logits/rejected": -0.9388672113418579,
      "logps/chosen": -1.134179711341858,
      "logps/rejected": -1.5144531726837158,
      "loss": 1.4702,
      "nll_loss": 1.419531226158142,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1134033203125,
      "rewards/margins": 0.03805847093462944,
      "rewards/rejected": -0.151611328125,
      "step": 110
    },
    {
      "epoch": 0.008737758036917028,
      "grad_norm": 1.857753363468467,
      "learning_rate": 7.3029674334022146e-06,
      "log_odds_chosen": 0.46162110567092896,
      "log_odds_ratio": -0.612109363079071,
      "logits/chosen": -1.001562476158142,
      "logits/rejected": -0.933398425579071,
      "logps/chosen": -1.054101586341858,
      "logps/rejected": -1.426171898841858,
      "loss": 1.4526,
      "nll_loss": 1.361914038658142,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.10541991889476776,
      "rewards/margins": 0.03725586086511612,
      "rewards/rejected": -0.14265136420726776,
      "step": 120
    },
    {
      "epoch": 0.009465904539993446,
      "grad_norm": 1.617828052395909,
      "learning_rate": 7.016464154456233e-06,
      "log_odds_chosen": 0.529052734375,
      "log_odds_ratio": -0.559521496295929,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.9828125238418579,
      "logps/chosen": -1.009374976158142,
      "logps/rejected": -1.4015624523162842,
      "loss": 1.4068,
      "nll_loss": 1.36328125,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.10087890923023224,
      "rewards/margins": 0.03919372707605362,
      "rewards/rejected": -0.14016112685203552,
      "step": 130
    },
    {
      "epoch": 0.010194051043069866,
      "grad_norm": 1.4449249885781161,
      "learning_rate": 6.7612340378281325e-06,
      "log_odds_chosen": 0.6396728754043579,
      "log_odds_ratio": -0.6016601324081421,
      "logits/chosen": -1.080664038658142,
      "logits/rejected": -1.0224609375,
      "logps/chosen": -1.013085961341858,
      "logps/rejected": -1.5509765148162842,
      "loss": 1.3947,
      "nll_loss": 1.263671875,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.10129394382238388,
      "rewards/margins": 0.05373077467083931,
      "rewards/rejected": -0.15500488877296448,
      "step": 140
    },
    {
      "epoch": 0.010922197546146284,
      "grad_norm": 1.509210963153022,
      "learning_rate": 6.531972647421809e-06,
      "log_odds_chosen": 0.505322277545929,
      "log_odds_ratio": -0.5859375,
      "logits/chosen": -0.9404296875,
      "logits/rejected": -0.90234375,
      "logps/chosen": -1.0480468273162842,
      "logps/rejected": -1.4265625476837158,
      "loss": 1.3667,
      "nll_loss": 1.3279297351837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10483398288488388,
      "rewards/margins": 0.03777313232421875,
      "rewards/rejected": -0.14262695610523224,
      "step": 150
    },
    {
      "epoch": 0.011650344049222704,
      "grad_norm": 1.7720026658744639,
      "learning_rate": 6.3245553203367575e-06,
      "log_odds_chosen": 0.7771972417831421,
      "log_odds_ratio": -0.520800769329071,
      "logits/chosen": -0.983593761920929,
      "logits/rejected": -0.928906261920929,
      "logps/chosen": -0.968554675579071,
      "logps/rejected": -1.564453125,
      "loss": 1.3618,
      "nll_loss": 1.2130858898162842,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0968017578125,
      "rewards/margins": 0.05971679836511612,
      "rewards/rejected": -0.15654297173023224,
      "step": 160
    },
    {
      "epoch": 0.012378490552299122,
      "grad_norm": 1.6888274544149533,
      "learning_rate": 6.135719910778963e-06,
      "log_odds_chosen": 0.55780029296875,
      "log_odds_ratio": -0.603515625,
      "logits/chosen": -0.874804675579071,
      "logits/rejected": -0.8394531011581421,
      "logps/chosen": -1.0343749523162842,
      "logps/rejected": -1.4695312976837158,
      "loss": 1.3592,
      "nll_loss": 1.359765648841858,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.10349120944738388,
      "rewards/margins": 0.04346618801355362,
      "rewards/rejected": -0.14692382514476776,
      "step": 170
    },
    {
      "epoch": 0.013106637055375542,
      "grad_norm": 1.9132061638687,
      "learning_rate": 5.962847939999439e-06,
      "log_odds_chosen": 0.536724865436554,
      "log_odds_ratio": -0.616406261920929,
      "logits/chosen": -0.9156249761581421,
      "logits/rejected": -0.848828136920929,
      "logps/chosen": -1.0412108898162842,
      "logps/rejected": -1.448828101158142,
      "loss": 1.33,
      "nll_loss": 1.4132812023162842,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.10410156100988388,
      "rewards/margins": 0.04075469821691513,
      "rewards/rejected": -0.14492186903953552,
      "step": 180
    },
    {
      "epoch": 0.01383478355845196,
      "grad_norm": 1.6575069796743211,
      "learning_rate": 5.803810000880094e-06,
      "log_odds_chosen": 0.541333019733429,
      "log_odds_ratio": -0.5892578363418579,
      "logits/chosen": -0.914843738079071,
      "logits/rejected": -0.853710949420929,
      "logps/chosen": -0.9683593511581421,
      "logps/rejected": -1.3828125,
      "loss": 1.2763,
      "nll_loss": 1.2560546398162842,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.09685058891773224,
      "rewards/margins": 0.04147796705365181,
      "rewards/rejected": -0.1383056640625,
      "step": 190
    },
    {
      "epoch": 0.01456293006152838,
      "grad_norm": 1.6203662182799872,
      "learning_rate": 5.65685424949238e-06,
      "log_odds_chosen": 0.550671398639679,
      "log_odds_ratio": -0.6015625,
      "logits/chosen": -0.9144531488418579,
      "logits/rejected": -0.816210925579071,
      "logps/chosen": -0.9501953125,
      "logps/rejected": -1.389257788658142,
      "loss": 1.3354,
      "nll_loss": 1.3134765625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09499511867761612,
      "rewards/margins": 0.04401855543255806,
      "rewards/rejected": -0.13896484673023224,
      "step": 200
    },
    {
      "epoch": 0.0152910765646048,
      "grad_norm": 1.5962074750146626,
      "learning_rate": 5.5205244747388325e-06,
      "log_odds_chosen": 0.5894775390625,
      "log_odds_ratio": -0.5653320550918579,
      "logits/chosen": -0.8978515863418579,
      "logits/rejected": -0.8148437738418579,
      "logps/chosen": -0.931640625,
      "logps/rejected": -1.365625023841858,
      "loss": 1.2597,
      "nll_loss": 1.1833984851837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09321288764476776,
      "rewards/margins": 0.0435333251953125,
      "rewards/rejected": -0.13657227158546448,
      "step": 210
    },
    {
      "epoch": 0.016019223067681217,
      "grad_norm": 1.551764270448319,
      "learning_rate": 5.393598899705936e-06,
      "log_odds_chosen": 0.6410766839981079,
      "log_odds_ratio": -0.551464855670929,
      "logits/chosen": -0.8765624761581421,
      "logits/rejected": -0.8154296875,
      "logps/chosen": -0.9251953363418579,
      "logps/rejected": -1.4226562976837158,
      "loss": 1.2736,
      "nll_loss": 1.267187476158142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.09248046576976776,
      "rewards/margins": 0.04987030103802681,
      "rewards/rejected": -0.14248046278953552,
      "step": 220
    },
    {
      "epoch": 0.016747369570757637,
      "grad_norm": 1.7722325738548714,
      "learning_rate": 5.275043787166296e-06,
      "log_odds_chosen": 0.654980480670929,
      "log_odds_ratio": -0.5464843511581421,
      "logits/chosen": -0.9458984136581421,
      "logits/rejected": -0.8744140863418579,
      "logps/chosen": -0.904492199420929,
      "logps/rejected": -1.3845703601837158,
      "loss": 1.2292,
      "nll_loss": 1.145117163658142,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09033203125,
      "rewards/margins": 0.0479736328125,
      "rewards/rejected": -0.1383056640625,
      "step": 230
    },
    {
      "epoch": 0.017475516073834057,
      "grad_norm": 1.5661623534113884,
      "learning_rate": 5.163977794943223e-06,
      "log_odds_chosen": 0.707958996295929,
      "log_odds_ratio": -0.5506836175918579,
      "logits/chosen": -0.931835949420929,
      "logits/rejected": -0.8558593988418579,
      "logps/chosen": -0.916210949420929,
      "logps/rejected": -1.4636719226837158,
      "loss": 1.2352,
      "nll_loss": 1.1785156726837158,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.09162597358226776,
      "rewards/margins": 0.05474090576171875,
      "rewards/rejected": -0.14633789658546448,
      "step": 240
    },
    {
      "epoch": 0.018203662576910473,
      "grad_norm": 1.7274597512775318,
      "learning_rate": 5.059644256269407e-06,
      "log_odds_chosen": 0.6816650629043579,
      "log_odds_ratio": -0.567089855670929,
      "logits/chosen": -0.9320312738418579,
      "logits/rejected": -0.853710949420929,
      "logps/chosen": -0.8521484136581421,
      "logps/rejected": -1.3771483898162842,
      "loss": 1.2294,
      "nll_loss": 1.1886718273162842,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.08527831733226776,
      "rewards/margins": 0.05253143236041069,
      "rewards/rejected": -0.13771972060203552,
      "step": 250
    },
    {
      "epoch": 0.018931809079986893,
      "grad_norm": 7.311067524981038,
      "learning_rate": 4.961389383568338e-06,
      "log_odds_chosen": 0.4852294921875,
      "log_odds_ratio": -0.618359386920929,
      "logits/chosen": -0.8931640386581421,
      "logits/rejected": -0.8724609613418579,
      "logps/chosen": -0.873046875,
      "logps/rejected": -1.239648461341858,
      "loss": 1.243,
      "nll_loss": 1.2013671398162842,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.08732910454273224,
      "rewards/margins": 0.03660430759191513,
      "rewards/rejected": -0.12399902194738388,
      "step": 260
    },
    {
      "epoch": 0.019659955583063313,
      "grad_norm": 1.5967286666896203,
      "learning_rate": 4.8686449556014755e-06,
      "log_odds_chosen": 0.614990234375,
      "log_odds_ratio": -0.580371081829071,
      "logits/chosen": -0.9281250238418579,
      "logits/rejected": -0.854296863079071,
      "logps/chosen": -0.834765613079071,
      "logps/rejected": -1.306249976158142,
      "loss": 1.2239,
      "nll_loss": 1.158203125,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08347167819738388,
      "rewards/margins": 0.04712829738855362,
      "rewards/rejected": -0.13056640326976776,
      "step": 270
    },
    {
      "epoch": 0.020388102086139732,
      "grad_norm": 1.3620859355944803,
      "learning_rate": 4.780914437337574e-06,
      "log_odds_chosen": 0.7137451171875,
      "log_odds_ratio": -0.53955078125,
      "logits/chosen": -0.854687511920929,
      "logits/rejected": -0.8218749761581421,
      "logps/chosen": -0.792187511920929,
      "logps/rejected": -1.2917969226837158,
      "loss": 1.2133,
      "nll_loss": 1.2140624523162842,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0792236328125,
      "rewards/margins": 0.05008697509765625,
      "rewards/rejected": -0.12922362983226776,
      "step": 280
    },
    {
      "epoch": 0.02111624858921615,
      "grad_norm": 1.5653654891183468,
      "learning_rate": 4.697761756117627e-06,
      "log_odds_chosen": 0.6397460699081421,
      "log_odds_ratio": -0.564648449420929,
      "logits/chosen": -0.967968761920929,
      "logits/rejected": -0.921093761920929,
      "logps/chosen": -0.8109375238418579,
      "logps/rejected": -1.2634766101837158,
      "loss": 1.2222,
      "nll_loss": 1.2216796875,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0810546875,
      "rewards/margins": 0.04528503492474556,
      "rewards/rejected": -0.12629394233226776,
      "step": 290
    },
    {
      "epoch": 0.02184439509229257,
      "grad_norm": 1.4517479073848938,
      "learning_rate": 4.618802153517006e-06,
      "log_odds_chosen": 0.4509033262729645,
      "log_odds_ratio": -0.622851550579071,
      "logits/chosen": -0.9271484613418579,
      "logits/rejected": -0.881640613079071,
      "logps/chosen": -0.784375011920929,
      "logps/rejected": -1.097265601158142,
      "loss": 1.2147,
      "nll_loss": 1.162695288658142,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07844237983226776,
      "rewards/margins": 0.03138275071978569,
      "rewards/rejected": -0.10979004204273224,
      "step": 300
    },
    {
      "epoch": 0.02257254159536899,
      "grad_norm": 1.5184986157394886,
      "learning_rate": 4.543694673976518e-06,
      "log_odds_chosen": 0.789929211139679,
      "log_odds_ratio": -0.524218738079071,
      "logits/chosen": -0.954296886920929,
      "logits/rejected": -0.900585949420929,
      "logps/chosen": -0.780468761920929,
      "logps/rejected": -1.345703125,
      "loss": 1.191,
      "nll_loss": 1.178320288658142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.078125,
      "rewards/margins": 0.05646514892578125,
      "rewards/rejected": -0.13461914658546448,
      "step": 310
    },
    {
      "epoch": 0.023300688098445408,
      "grad_norm": 1.4159165186889227,
      "learning_rate": 4.472135954999579e-06,
      "log_odds_chosen": 0.585253894329071,
      "log_odds_ratio": -0.6026366949081421,
      "logits/chosen": -0.937304675579071,
      "logits/rejected": -0.87353515625,
      "logps/chosen": -0.825390636920929,
      "logps/rejected": -1.265039086341858,
      "loss": 1.2088,
      "nll_loss": 1.2177734375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08256836235523224,
      "rewards/margins": 0.043930817395448685,
      "rewards/rejected": -0.12651367485523224,
      "step": 320
    },
    {
      "epoch": 0.024028834601521828,
      "grad_norm": 1.3004630972387219,
      "learning_rate": 4.403855060505443e-06,
      "log_odds_chosen": 0.644482433795929,
      "log_odds_ratio": -0.5538085699081421,
      "logits/chosen": -0.9789062738418579,
      "logits/rejected": -0.931835949420929,
      "logps/chosen": -0.7730468511581421,
      "logps/rejected": -1.2265625,
      "loss": 1.2037,
      "nll_loss": 1.1396484375,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.077392578125,
      "rewards/margins": 0.04529418796300888,
      "rewards/rejected": -0.12260742485523224,
      "step": 330
    },
    {
      "epoch": 0.024756981104598244,
      "grad_norm": 1.3149112607847993,
      "learning_rate": 4.338609156373123e-06,
      "log_odds_chosen": 0.652636706829071,
      "log_odds_ratio": -0.5506836175918579,
      "logits/chosen": -0.9886718988418579,
      "logits/rejected": -0.9322265386581421,
      "logps/chosen": -0.769726574420929,
      "logps/rejected": -1.222070336341858,
      "loss": 1.206,
      "nll_loss": 1.1501953601837158,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07703857123851776,
      "rewards/margins": 0.04521484300494194,
      "rewards/rejected": -0.12226562201976776,
      "step": 340
    },
    {
      "epoch": 0.025485127607674664,
      "grad_norm": 1.5387878523139185,
      "learning_rate": 4.27617987059879e-06,
      "log_odds_chosen": 0.6639648675918579,
      "log_odds_ratio": -0.531933605670929,
      "logits/chosen": -0.9808593988418579,
      "logits/rejected": -0.940234363079071,
      "logps/chosen": -0.7919921875,
      "logps/rejected": -1.234960913658142,
      "loss": 1.1721,
      "nll_loss": 1.2236328125,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0792236328125,
      "rewards/margins": 0.044281005859375,
      "rewards/rejected": -0.12346191704273224,
      "step": 350
    },
    {
      "epoch": 0.026213274110751084,
      "grad_norm": 1.5632123406053722,
      "learning_rate": 4.216370213557839e-06,
      "log_odds_chosen": 0.5484374761581421,
      "log_odds_ratio": -0.6241210699081421,
      "logits/chosen": -0.9794921875,
      "logits/rejected": -0.9242187738418579,
      "logps/chosen": -0.8414062261581421,
      "logps/rejected": -1.216406226158142,
      "loss": 1.1985,
      "nll_loss": 1.181054711341858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08405761420726776,
      "rewards/margins": 0.037567138671875,
      "rewards/rejected": -0.12165527045726776,
      "step": 360
    },
    {
      "epoch": 0.026941420613827503,
      "grad_norm": 1.3539809998909873,
      "learning_rate": 4.15900195928029e-06,
      "log_odds_chosen": 0.6247314214706421,
      "log_odds_ratio": -0.570996105670929,
      "logits/chosen": -0.9634765386581421,
      "logits/rejected": -0.922656238079071,
      "logps/chosen": -0.7626953125,
      "logps/rejected": -1.187890648841858,
      "loss": 1.178,
      "nll_loss": 1.0378906726837158,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07622070610523224,
      "rewards/margins": 0.04259643703699112,
      "rewards/rejected": -0.11882324516773224,
      "step": 370
    },
    {
      "epoch": 0.02766956711690392,
      "grad_norm": 1.4817193256750378,
      "learning_rate": 4.103913408340617e-06,
      "log_odds_chosen": 0.675366222858429,
      "log_odds_ratio": -0.555859386920929,
      "logits/chosen": -1.0001952648162842,
      "logits/rejected": -0.951953113079071,
      "logps/chosen": -0.754687488079071,
      "logps/rejected": -1.2218749523162842,
      "loss": 1.1963,
      "nll_loss": 1.111328125,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07546386867761612,
      "rewards/margins": 0.04680938646197319,
      "rewards/rejected": -0.12226562201976776,
      "step": 380
    },
    {
      "epoch": 0.02839771361998034,
      "grad_norm": 1.5387865278625523,
      "learning_rate": 4.050957468334666e-06,
      "log_odds_chosen": 0.6034179925918579,
      "log_odds_ratio": -0.5555664300918579,
      "logits/chosen": -0.9828125238418579,
      "logits/rejected": -0.8999999761581421,
      "logps/chosen": -0.807421863079071,
      "logps/rejected": -1.2265625,
      "loss": 1.1811,
      "nll_loss": 1.120703101158142,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0806884765625,
      "rewards/margins": 0.0419769287109375,
      "rewards/rejected": -0.12272949516773224,
      "step": 390
    },
    {
      "epoch": 0.02912586012305676,
      "grad_norm": 1.4355835455366375,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.48689574003219604,
      "log_odds_ratio": -0.63525390625,
      "logits/chosen": -0.953320324420929,
      "logits/rejected": -0.93359375,
      "logps/chosen": -0.843945324420929,
      "logps/rejected": -1.197265625,
      "loss": 1.2055,
      "nll_loss": 1.080078125,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.08439941704273224,
      "rewards/margins": 0.03527374193072319,
      "rewards/rejected": -0.11967773735523224,
      "step": 400
    },
    {
      "epoch": 0.02985400662613318,
      "grad_norm": 1.5027831622710668,
      "learning_rate": 3.950918386598359e-06,
      "log_odds_chosen": 0.6747802495956421,
      "log_odds_ratio": -0.559374988079071,
      "logits/chosen": -0.9576171636581421,
      "logits/rejected": -0.8882812261581421,
      "logps/chosen": -0.75,
      "logps/rejected": -1.2404296398162842,
      "loss": 1.1956,
      "nll_loss": 1.090234398841858,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07502441108226776,
      "rewards/margins": 0.04903869703412056,
      "rewards/rejected": -0.12404785305261612,
      "step": 410
    },
    {
      "epoch": 0.0305821531292096,
      "grad_norm": 1.4280512968140957,
      "learning_rate": 3.903600291794132e-06,
      "log_odds_chosen": 0.668042004108429,
      "log_odds_ratio": -0.5658203363418579,
      "logits/chosen": -0.9326171875,
      "logits/rejected": -0.8863281011581421,
      "logps/chosen": -0.789257824420929,
      "logps/rejected": -1.259179711341858,
      "loss": 1.1769,
      "nll_loss": 1.068750023841858,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07901611179113388,
      "rewards/margins": 0.046973418444395065,
      "rewards/rejected": -0.12587890028953552,
      "step": 420
    },
    {
      "epoch": 0.031310299632286015,
      "grad_norm": 1.4730733041346917,
      "learning_rate": 3.857942577363297e-06,
      "log_odds_chosen": 0.850781261920929,
      "log_odds_ratio": -0.4913085997104645,
      "logits/chosen": -0.9478515386581421,
      "logits/rejected": -0.912304699420929,
      "logps/chosen": -0.7509765625,
      "logps/rejected": -1.326171875,
      "loss": 1.1831,
      "nll_loss": 1.0851562023162842,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.0750732421875,
      "rewards/margins": 0.05753784254193306,
      "rewards/rejected": -0.1326904296875,
      "step": 430
    },
    {
      "epoch": 0.032038446135362435,
      "grad_norm": 1.451709668646297,
      "learning_rate": 3.813850356982369e-06,
      "log_odds_chosen": 0.62457275390625,
      "log_odds_ratio": -0.5673828125,
      "logits/chosen": -0.9271484613418579,
      "logits/rejected": -0.8500000238418579,
      "logps/chosen": -0.721875011920929,
      "logps/rejected": -1.1496093273162842,
      "loss": 1.1566,
      "nll_loss": 1.0578124523162842,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07222900539636612,
      "rewards/margins": 0.04280243068933487,
      "rewards/rejected": -0.11495361477136612,
      "step": 440
    },
    {
      "epoch": 0.032766592638438855,
      "grad_norm": 1.5000831711435392,
      "learning_rate": 3.7712361663282537e-06,
      "log_odds_chosen": 0.7065674066543579,
      "log_odds_ratio": -0.599316418170929,
      "logits/chosen": -0.990234375,
      "logits/rejected": -0.9150390625,
      "logps/chosen": -0.8130859136581421,
      "logps/rejected": -1.350195288658142,
      "loss": 1.1826,
      "nll_loss": 1.15625,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08134765923023224,
      "rewards/margins": 0.05374603345990181,
      "rewards/rejected": -0.13510742783546448,
      "step": 450
    },
    {
      "epoch": 0.033494739141515274,
      "grad_norm": 3.8241943804343226,
      "learning_rate": 3.730019232961255e-06,
      "log_odds_chosen": 0.7632812261581421,
      "log_odds_ratio": -0.558398425579071,
      "logits/chosen": -0.9671875238418579,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.7974609136581421,
      "logps/rejected": -1.317968726158142,
      "loss": 1.1724,
      "nll_loss": 1.1091797351837158,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07976074516773224,
      "rewards/margins": 0.05216064304113388,
      "rewards/rejected": -0.13190917670726776,
      "step": 460
    },
    {
      "epoch": 0.034222885644591694,
      "grad_norm": 1.4037887232429922,
      "learning_rate": 3.6901248321155403e-06,
      "log_odds_chosen": 0.656494140625,
      "log_odds_ratio": -0.55126953125,
      "logits/chosen": -1.002343773841858,
      "logits/rejected": -0.9361327886581421,
      "logps/chosen": -0.755078136920929,
      "logps/rejected": -1.2111327648162842,
      "loss": 1.1963,
      "nll_loss": 1.075585961341858,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0755615234375,
      "rewards/margins": 0.04556732252240181,
      "rewards/rejected": -0.12116698920726776,
      "step": 470
    },
    {
      "epoch": 0.034951032147668114,
      "grad_norm": 14.473120499223526,
      "learning_rate": 3.6514837167011073e-06,
      "log_odds_chosen": 0.670764148235321,
      "log_odds_ratio": -0.55126953125,
      "logits/chosen": -0.983203113079071,
      "logits/rejected": -0.880664050579071,
      "logps/chosen": -0.782421886920929,
      "logps/rejected": -1.243554711341858,
      "loss": 1.1542,
      "nll_loss": 1.081445336341858,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0782470703125,
      "rewards/margins": 0.04622802883386612,
      "rewards/rejected": -0.1243896484375,
      "step": 480
    },
    {
      "epoch": 0.03567917865074453,
      "grad_norm": 1.4021631646997694,
      "learning_rate": 3.6140316116210052e-06,
      "log_odds_chosen": 0.6236938238143921,
      "log_odds_ratio": -0.5826171636581421,
      "logits/chosen": -0.927050769329071,
      "logits/rejected": -0.883593738079071,
      "logps/chosen": -0.801953136920929,
      "logps/rejected": -1.217382788658142,
      "loss": 1.1497,
      "nll_loss": 1.1140625476837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0802001953125,
      "rewards/margins": 0.04158172756433487,
      "rewards/rejected": -0.12180175632238388,
      "step": 490
    },
    {
      "epoch": 0.036407325153820946,
      "grad_norm": 1.4504280665517582,
      "learning_rate": 3.5777087639996634e-06,
      "log_odds_chosen": 0.7083984613418579,
      "log_odds_ratio": -0.5472656488418579,
      "logits/chosen": -0.979687511920929,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.7665039300918579,
      "logps/rejected": -1.2509765625,
      "loss": 1.1663,
      "nll_loss": 1.1181640625,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07667236030101776,
      "rewards/margins": 0.04839477688074112,
      "rewards/rejected": -0.12507323920726776,
      "step": 500
    },
    {
      "epoch": 0.037135471656897366,
      "grad_norm": 1.6116253479627094,
      "learning_rate": 3.5424595421603814e-06,
      "log_odds_chosen": 0.6789916753768921,
      "log_odds_ratio": -0.5692383050918579,
      "logits/chosen": -1.037500023841858,
      "logits/rejected": -0.960742175579071,
      "logps/chosen": -0.77734375,
      "logps/rejected": -1.2580077648162842,
      "loss": 1.1452,
      "nll_loss": 1.0730469226837158,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0777587890625,
      "rewards/margins": 0.04798584058880806,
      "rewards/rejected": -0.12578125298023224,
      "step": 510
    },
    {
      "epoch": 0.037863618159973786,
      "grad_norm": 1.539494298635483,
      "learning_rate": 3.5082320772281165e-06,
      "log_odds_chosen": 0.747753918170929,
      "log_odds_ratio": -0.528613269329071,
      "logits/chosen": -1.0373046398162842,
      "logits/rejected": -0.9320312738418579,
      "logps/chosen": -0.7099609375,
      "logps/rejected": -1.2082030773162842,
      "loss": 1.1636,
      "nll_loss": 1.0947265625,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07093505561351776,
      "rewards/margins": 0.04978637769818306,
      "rewards/rejected": -0.12082519382238388,
      "step": 520
    },
    {
      "epoch": 0.038591764663050206,
      "grad_norm": 1.4585989044057364,
      "learning_rate": 3.474977942104555e-06,
      "log_odds_chosen": 0.7982422113418579,
      "log_odds_ratio": -0.4876953065395355,
      "logits/chosen": -0.981640636920929,
      "logits/rejected": -0.9144531488418579,
      "logps/chosen": -0.758984386920929,
      "logps/rejected": -1.274999976158142,
      "loss": 1.1488,
      "nll_loss": 1.159765601158142,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07589111477136612,
      "rewards/margins": 0.05152587965130806,
      "rewards/rejected": -0.12739257514476776,
      "step": 530
    },
    {
      "epoch": 0.039319911166126625,
      "grad_norm": 1.4080621583665085,
      "learning_rate": 3.442651863295481e-06,
      "log_odds_chosen": 0.6578613519668579,
      "log_odds_ratio": -0.56298828125,
      "logits/chosen": -0.9917968511581421,
      "logits/rejected": -0.938281238079071,
      "logps/chosen": -0.765625,
      "logps/rejected": -1.209375023841858,
      "loss": 1.1478,
      "nll_loss": 1.089453101158142,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07663574069738388,
      "rewards/margins": 0.04428863525390625,
      "rewards/rejected": -0.12104491889476776,
      "step": 540
    },
    {
      "epoch": 0.040048057669203045,
      "grad_norm": 1.4487680890750678,
      "learning_rate": 3.4112114616897665e-06,
      "log_odds_chosen": 0.7410644292831421,
      "log_odds_ratio": -0.532910168170929,
      "logits/chosen": -0.981640636920929,
      "logits/rejected": -0.8919922113418579,
      "logps/chosen": -0.7835937738418579,
      "logps/rejected": -1.29296875,
      "loss": 1.1611,
      "nll_loss": 1.1779296398162842,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07835693657398224,
      "rewards/margins": 0.05080718919634819,
      "rewards/rejected": -0.12912598252296448,
      "step": 550
    },
    {
      "epoch": 0.040776204172279465,
      "grad_norm": 1.4460105953307048,
      "learning_rate": 3.3806170189140663e-06,
      "log_odds_chosen": 0.839404284954071,
      "log_odds_ratio": -0.518310546875,
      "logits/chosen": -0.978320300579071,
      "logits/rejected": -0.9076172113418579,
      "logps/chosen": -0.728222668170929,
      "logps/rejected": -1.279687523841858,
      "loss": 1.1673,
      "nll_loss": 1.0771484375,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07281494140625,
      "rewards/margins": 0.05517425388097763,
      "rewards/rejected": -0.1280517578125,
      "step": 560
    },
    {
      "epoch": 0.041504350675355885,
      "grad_norm": 1.3270439290769895,
      "learning_rate": 3.350831266333564e-06,
      "log_odds_chosen": 0.826708972454071,
      "log_odds_ratio": -0.509960949420929,
      "logits/chosen": -0.9378906488418579,
      "logits/rejected": -0.8890625238418579,
      "logps/chosen": -0.781445324420929,
      "logps/rejected": -1.3515625,
      "loss": 1.1297,
      "nll_loss": 1.1443359851837158,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07811279594898224,
      "rewards/margins": 0.05701599270105362,
      "rewards/rejected": -0.135009765625,
      "step": 570
    },
    {
      "epoch": 0.0422324971784323,
      "grad_norm": 1.4505105770320856,
      "learning_rate": 3.3218191941495984e-06,
      "log_odds_chosen": 0.88323974609375,
      "log_odds_ratio": -0.4817871153354645,
      "logits/chosen": -0.8863281011581421,
      "logits/rejected": -0.814648449420929,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -1.314062476158142,
      "loss": 1.1539,
      "nll_loss": 1.095703125,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07156982272863388,
      "rewards/margins": 0.05988464504480362,
      "rewards/rejected": -0.1314697265625,
      "step": 580
    },
    {
      "epoch": 0.04296064368150872,
      "grad_norm": 1.4158594488139011,
      "learning_rate": 3.293547878370473e-06,
      "log_odds_chosen": 0.7425292730331421,
      "log_odds_ratio": -0.558789074420929,
      "logits/chosen": -1.029296875,
      "logits/rejected": -0.9117187261581421,
      "logps/chosen": -0.7777343988418579,
      "logps/rejected": -1.300390601158142,
      "loss": 1.1698,
      "nll_loss": 1.0867187976837158,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07773437350988388,
      "rewards/margins": 0.05230712890625,
      "rewards/rejected": -0.13017578423023224,
      "step": 590
    },
    {
      "epoch": 0.04368879018458514,
      "grad_norm": 1.2635665666685174,
      "learning_rate": 3.2659863237109044e-06,
      "log_odds_chosen": 0.766796886920929,
      "log_odds_ratio": -0.525195300579071,
      "logits/chosen": -0.926953136920929,
      "logits/rejected": -0.8707031011581421,
      "logps/chosen": -0.747265636920929,
      "logps/rejected": -1.2433593273162842,
      "loss": 1.1475,
      "nll_loss": 1.129296898841858,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07471923530101776,
      "rewards/margins": 0.04962768405675888,
      "rewards/rejected": -0.12424316257238388,
      "step": 600
    },
    {
      "epoch": 0.04441693668766156,
      "grad_norm": 1.365323069745603,
      "learning_rate": 3.239105320715664e-06,
      "log_odds_chosen": 0.6534179449081421,
      "log_odds_ratio": -0.5713866949081421,
      "logits/chosen": -0.921875,
      "logits/rejected": -0.8384765386581421,
      "logps/chosen": -0.758984386920929,
      "logps/rejected": -1.2023437023162842,
      "loss": 1.13,
      "nll_loss": 1.08203125,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07584228366613388,
      "rewards/margins": 0.04430847242474556,
      "rewards/rejected": -0.12014160305261612,
      "step": 610
    },
    {
      "epoch": 0.04514508319073798,
      "grad_norm": 1.4386908687736795,
      "learning_rate": 3.2128773156099956e-06,
      "log_odds_chosen": 0.521655261516571,
      "log_odds_ratio": -0.615527331829071,
      "logits/chosen": -0.9248046875,
      "logits/rejected": -0.8724609613418579,
      "logps/chosen": -0.8402343988418579,
      "logps/rejected": -1.213281273841858,
      "loss": 1.1664,
      "nll_loss": 1.1300780773162842,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.08408202975988388,
      "rewards/margins": 0.03731994703412056,
      "rewards/rejected": -0.12141112983226776,
      "step": 620
    },
    {
      "epoch": 0.045873229693814396,
      "grad_norm": 1.4435488382072212,
      "learning_rate": 3.187276291558383e-06,
      "log_odds_chosen": 0.8658691644668579,
      "log_odds_ratio": -0.4942382872104645,
      "logits/chosen": -1.0087890625,
      "logits/rejected": -0.896289050579071,
      "logps/chosen": -0.7435547113418579,
      "logps/rejected": -1.3240234851837158,
      "loss": 1.1026,
      "nll_loss": 1.0828125476837158,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07431640475988388,
      "rewards/margins": 0.05805664137005806,
      "rewards/rejected": -0.13242188096046448,
      "step": 630
    },
    {
      "epoch": 0.046601376196890816,
      "grad_norm": 1.482787682541737,
      "learning_rate": 3.1622776601683788e-06,
      "log_odds_chosen": 0.8796631097793579,
      "log_odds_ratio": -0.49311524629592896,
      "logits/chosen": -0.9281250238418579,
      "logits/rejected": -0.874218761920929,
      "logps/chosen": -0.7124999761581421,
      "logps/rejected": -1.279687523841858,
      "loss": 1.1436,
      "nll_loss": 1.1337890625,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07125244289636612,
      "rewards/margins": 0.05678100511431694,
      "rewards/rejected": -0.12810058891773224,
      "step": 640
    },
    {
      "epoch": 0.047329522699967236,
      "grad_norm": 1.5473980483876462,
      "learning_rate": 3.1378581622109444e-06,
      "log_odds_chosen": 0.9545654058456421,
      "log_odds_ratio": -0.45380860567092896,
      "logits/chosen": -1.016992211341858,
      "logits/rejected": -0.9339843988418579,
      "logps/chosen": -0.7142578363418579,
      "logps/rejected": -1.357031226158142,
      "loss": 1.1105,
      "nll_loss": 1.082617163658142,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07135009765625,
      "rewards/margins": 0.06436462700366974,
      "rewards/rejected": -0.13576659560203552,
      "step": 650
    },
    {
      "epoch": 0.048057669203043656,
      "grad_norm": 1.4079702343392544,
      "learning_rate": 3.113995776646092e-06,
      "log_odds_chosen": 0.83203125,
      "log_odds_ratio": -0.50146484375,
      "logits/chosen": -0.981640636920929,
      "logits/rejected": -0.913867175579071,
      "logps/chosen": -0.7357422113418579,
      "logps/rejected": -1.3035156726837158,
      "loss": 1.1481,
      "nll_loss": 1.113671898841858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07359619438648224,
      "rewards/margins": 0.056671142578125,
      "rewards/rejected": -0.13026122748851776,
      "step": 660
    },
    {
      "epoch": 0.04878581570612007,
      "grad_norm": 1.4702021710737754,
      "learning_rate": 3.090669637145023e-06,
      "log_odds_chosen": 0.72796630859375,
      "log_odds_ratio": -0.530957043170929,
      "logits/chosen": -0.9609375,
      "logits/rejected": -0.880078136920929,
      "logps/chosen": -0.712890625,
      "logps/rejected": -1.212499976158142,
      "loss": 1.128,
      "nll_loss": 1.1613280773162842,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07121582329273224,
      "rewards/margins": 0.04995117336511612,
      "rewards/rejected": -0.12127685546875,
      "step": 670
    },
    {
      "epoch": 0.04951396220919649,
      "grad_norm": 1.559008568221807,
      "learning_rate": 3.0678599553894814e-06,
      "log_odds_chosen": 1.0158202648162842,
      "log_odds_ratio": -0.4886230528354645,
      "logits/chosen": -0.9683593511581421,
      "logits/rejected": -0.875,
      "logps/chosen": -0.748828113079071,
      "logps/rejected": -1.473046898841858,
      "loss": 1.1259,
      "nll_loss": 1.086523413658142,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07485351711511612,
      "rewards/margins": 0.0723876953125,
      "rewards/rejected": -0.14731445908546448,
      "step": 680
    },
    {
      "epoch": 0.05024210871227291,
      "grad_norm": 1.7932886969400792,
      "learning_rate": 3.0455479505075235e-06,
      "log_odds_chosen": 0.76513671875,
      "log_odds_ratio": -0.5536133050918579,
      "logits/chosen": -0.9830077886581421,
      "logits/rejected": -0.9150390625,
      "logps/chosen": -0.7515624761581421,
      "logps/rejected": -1.309179663658142,
      "loss": 1.1313,
      "nll_loss": 1.025976538658142,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07515869289636612,
      "rewards/margins": 0.055755615234375,
      "rewards/rejected": -0.13076171278953552,
      "step": 690
    },
    {
      "epoch": 0.05097025521534933,
      "grad_norm": 1.3812739282921287,
      "learning_rate": 3.0237157840738173e-06,
      "log_odds_chosen": 0.7027343511581421,
      "log_odds_ratio": -0.5433593988418579,
      "logits/chosen": -0.9306640625,
      "logits/rejected": -0.872265636920929,
      "logps/chosen": -0.7744140625,
      "logps/rejected": -1.233007788658142,
      "loss": 1.1238,
      "nll_loss": 1.1443359851837158,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07749023288488388,
      "rewards/margins": 0.04592742770910263,
      "rewards/rejected": -0.123291015625,
      "step": 700
    },
    {
      "epoch": 0.05169840171842575,
      "grad_norm": 1.4638094361730472,
      "learning_rate": 3.002346500163206e-06,
      "log_odds_chosen": 0.8546386957168579,
      "log_odds_ratio": -0.4844726622104645,
      "logits/chosen": -0.9876953363418579,
      "logits/rejected": -0.936718761920929,
      "logps/chosen": -0.70361328125,
      "logps/rejected": -1.2419922351837158,
      "loss": 1.1139,
      "nll_loss": 1.0587890148162842,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07028808444738388,
      "rewards/margins": 0.05384521558880806,
      "rewards/rejected": -0.12429199367761612,
      "step": 710
    },
    {
      "epoch": 0.05242654822150217,
      "grad_norm": 1.5176926052780488,
      "learning_rate": 2.9814239699997195e-06,
      "log_odds_chosen": 0.8565429449081421,
      "log_odds_ratio": -0.5035644769668579,
      "logits/chosen": -1.0478515625,
      "logits/rejected": -0.961132824420929,
      "logps/chosen": -0.734375,
      "logps/rejected": -1.3244140148162842,
      "loss": 1.1277,
      "nll_loss": 1.076562523841858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07342529296875,
      "rewards/margins": 0.05902709811925888,
      "rewards/rejected": -0.13239745795726776,
      "step": 720
    },
    {
      "epoch": 0.05315469472457859,
      "grad_norm": 1.4510014022690885,
      "learning_rate": 2.9609328407904207e-06,
      "log_odds_chosen": 0.6505126953125,
      "log_odds_ratio": -0.551562488079071,
      "logits/chosen": -1.008203148841858,
      "logits/rejected": -0.9556640386581421,
      "logps/chosen": -0.7339843511581421,
      "logps/rejected": -1.146875023841858,
      "loss": 1.1182,
      "nll_loss": 1.1130859851837158,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07340087741613388,
      "rewards/margins": 0.041185759007930756,
      "rewards/rejected": -0.11457519233226776,
      "step": 730
    },
    {
      "epoch": 0.05388284122765501,
      "grad_norm": 1.5201286658330744,
      "learning_rate": 2.940858488375231e-06,
      "log_odds_chosen": 0.6685546636581421,
      "log_odds_ratio": -0.549267590045929,
      "logits/chosen": -1.053320288658142,
      "logits/rejected": -0.9599609375,
      "logps/chosen": -0.7748047113418579,
      "logps/rejected": -1.248632788658142,
      "loss": 1.1478,
      "nll_loss": 1.1228516101837158,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07745361328125,
      "rewards/margins": 0.04729461669921875,
      "rewards/rejected": -0.12482909858226776,
      "step": 740
    },
    {
      "epoch": 0.054610987730731426,
      "grad_norm": 1.5232617658745549,
      "learning_rate": 2.9211869733608857e-06,
      "log_odds_chosen": 0.8666747808456421,
      "log_odds_ratio": -0.5086914300918579,
      "logits/chosen": -1.0291016101837158,
      "logits/rejected": -0.9716796875,
      "logps/chosen": -0.7669922113418579,
      "logps/rejected": -1.3732421398162842,
      "loss": 1.1273,
      "nll_loss": 1.015234351158142,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07661132514476776,
      "rewards/margins": 0.0606536865234375,
      "rewards/rejected": -0.13735350966453552,
      "step": 750
    },
    {
      "epoch": 0.05533913423380784,
      "grad_norm": 1.554150267019514,
      "learning_rate": 2.901905000440047e-06,
      "log_odds_chosen": 1.002893090248108,
      "log_odds_ratio": -0.4734863340854645,
      "logits/chosen": -1.052148461341858,
      "logits/rejected": -0.989062488079071,
      "logps/chosen": -0.774707019329071,
      "logps/rejected": -1.4660155773162842,
      "loss": 1.1179,
      "nll_loss": 1.0564453601837158,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07745361328125,
      "rewards/margins": 0.06914367526769638,
      "rewards/rejected": -0.1466064453125,
      "step": 760
    },
    {
      "epoch": 0.05606728073688426,
      "grad_norm": 1.6235967684077466,
      "learning_rate": 2.8829998806257885e-06,
      "log_odds_chosen": 0.967846691608429,
      "log_odds_ratio": -0.502246081829071,
      "logits/chosen": -1.0460937023162842,
      "logits/rejected": -0.9306640625,
      "logps/chosen": -0.719921886920929,
      "logps/rejected": -1.4093749523162842,
      "loss": 1.108,
      "nll_loss": 0.981640636920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07194824516773224,
      "rewards/margins": 0.0689239501953125,
      "rewards/rejected": -0.14094237983226776,
      "step": 770
    },
    {
      "epoch": 0.05679542723996068,
      "grad_norm": 1.6284312497330824,
      "learning_rate": 2.8644594961577314e-06,
      "log_odds_chosen": 0.686022937297821,
      "log_odds_ratio": -0.58935546875,
      "logits/chosen": -1.0148437023162842,
      "logits/rejected": -0.9302734136581421,
      "logps/chosen": -0.8052734136581421,
      "logps/rejected": -1.2841796875,
      "loss": 1.105,
      "nll_loss": 1.0955078601837158,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.08050537109375,
      "rewards/margins": 0.04795227199792862,
      "rewards/rejected": -0.12849120795726776,
      "step": 780
    },
    {
      "epoch": 0.0575235737430371,
      "grad_norm": 1.4199339043529113,
      "learning_rate": 2.84627226785928e-06,
      "log_odds_chosen": 0.8512207269668579,
      "log_odds_ratio": -0.481201171875,
      "logits/chosen": -0.987109363079071,
      "logits/rejected": -0.8896484375,
      "logps/chosen": -0.673144519329071,
      "logps/rejected": -1.225195288658142,
      "loss": 1.1308,
      "nll_loss": 1.0949218273162842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06732177734375,
      "rewards/margins": 0.055206298828125,
      "rewards/rejected": -0.12246093899011612,
      "step": 790
    },
    {
      "epoch": 0.05825172024611352,
      "grad_norm": 1.3919340886975955,
      "learning_rate": 2.82842712474619e-06,
      "log_odds_chosen": 0.902099609375,
      "log_odds_ratio": -0.4999023377895355,
      "logits/chosen": -0.9976562261581421,
      "logits/rejected": -0.9033203125,
      "logps/chosen": -0.7085937261581421,
      "logps/rejected": -1.313867211341858,
      "loss": 1.0997,
      "nll_loss": 1.0578124523162842,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07080078125,
      "rewards/margins": 0.06048583984375,
      "rewards/rejected": -0.13129882514476776,
      "step": 800
    },
    {
      "epoch": 0.05897986674918994,
      "grad_norm": 1.4695419553082982,
      "learning_rate": 2.810913475705226e-06,
      "log_odds_chosen": 1.1593749523162842,
      "log_odds_ratio": -0.4581054747104645,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.944531261920929,
      "logps/chosen": -0.7826172113418579,
      "logps/rejected": -1.597265601158142,
      "loss": 1.0938,
      "nll_loss": 1.076757788658142,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0782470703125,
      "rewards/margins": 0.08135986328125,
      "rewards/rejected": -0.15969237685203552,
      "step": 810
    },
    {
      "epoch": 0.05970801325226636,
      "grad_norm": 1.4805729889982453,
      "learning_rate": 2.7937211830783128e-06,
      "log_odds_chosen": 0.931445300579071,
      "log_odds_ratio": -0.5008789300918579,
      "logits/chosen": -1.0496094226837158,
      "logits/rejected": -0.9468749761581421,
      "logps/chosen": -0.7357422113418579,
      "logps/rejected": -1.3576171398162842,
      "loss": 1.1064,
      "nll_loss": 1.0029296875,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.0736083984375,
      "rewards/margins": 0.06226501613855362,
      "rewards/rejected": -0.13579101860523224,
      "step": 820
    },
    {
      "epoch": 0.06043615975534278,
      "grad_norm": 1.4434030394502348,
      "learning_rate": 2.776840538002493e-06,
      "log_odds_chosen": 0.8809570074081421,
      "log_odds_ratio": -0.4793945252895355,
      "logits/chosen": -1.033789038658142,
      "logits/rejected": -0.9521484375,
      "logps/chosen": -0.69775390625,
      "logps/rejected": -1.298437476158142,
      "loss": 1.1303,
      "nll_loss": 1.061914086341858,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06975097954273224,
      "rewards/margins": 0.06011962890625,
      "rewards/rejected": -0.12985840439796448,
      "step": 830
    },
    {
      "epoch": 0.0611643062584192,
      "grad_norm": 1.445210902579111,
      "learning_rate": 2.7602622373694163e-06,
      "log_odds_chosen": 1.0266602039337158,
      "log_odds_ratio": -0.48124998807907104,
      "logits/chosen": -1.096093773841858,
      "logits/rejected": -1.048242211341858,
      "logps/chosen": -0.740234375,
      "logps/rejected": -1.460351586341858,
      "loss": 1.1085,
      "nll_loss": 1.003515601158142,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07404784858226776,
      "rewards/margins": 0.07198486477136612,
      "rewards/rejected": -0.14599609375,
      "step": 840
    },
    {
      "epoch": 0.06189245276149561,
      "grad_norm": 1.477719898018172,
      "learning_rate": 2.743977362280141e-06,
      "log_odds_chosen": 0.818920910358429,
      "log_odds_ratio": -0.54248046875,
      "logits/chosen": -1.083593726158142,
      "logits/rejected": -0.999804675579071,
      "logps/chosen": -0.7671874761581421,
      "logps/rejected": -1.345117211341858,
      "loss": 1.0925,
      "nll_loss": 1.115820288658142,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07666015625,
      "rewards/margins": 0.05780944973230362,
      "rewards/rejected": -0.134521484375,
      "step": 850
    },
    {
      "epoch": 0.06262059926457203,
      "grad_norm": 1.5664238020213266,
      "learning_rate": 2.7279773578818937e-06,
      "log_odds_chosen": 0.76129150390625,
      "log_odds_ratio": -0.548632800579071,
      "logits/chosen": -1.045312523841858,
      "logits/rejected": -0.977734386920929,
      "logps/chosen": -0.740429699420929,
      "logps/rejected": -1.2587890625,
      "loss": 1.1179,
      "nll_loss": 1.1189453601837158,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07407226413488388,
      "rewards/margins": 0.05186767503619194,
      "rewards/rejected": -0.1259765625,
      "step": 860
    },
    {
      "epoch": 0.06334874576764846,
      "grad_norm": 1.632884354692461,
      "learning_rate": 2.7122540144832417e-06,
      "log_odds_chosen": 0.7477782964706421,
      "log_odds_ratio": -0.574267566204071,
      "logits/chosen": -1.031640648841858,
      "logits/rejected": -0.9537109136581421,
      "logps/chosen": -0.740429699420929,
      "logps/rejected": -1.2312500476837158,
      "loss": 1.1306,
      "nll_loss": 1.1638672351837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0740966796875,
      "rewards/margins": 0.04911651462316513,
      "rewards/rejected": -0.12309570610523224,
      "step": 870
    },
    {
      "epoch": 0.06407689227072487,
      "grad_norm": 1.5482195059977542,
      "learning_rate": 2.696799449852968e-06,
      "log_odds_chosen": 0.804443359375,
      "log_odds_ratio": -0.525634765625,
      "logits/chosen": -1.04296875,
      "logits/rejected": -0.9759765863418579,
      "logps/chosen": -0.711718738079071,
      "logps/rejected": -1.258398413658142,
      "loss": 1.0972,
      "nll_loss": 1.0595703125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0711669921875,
      "rewards/margins": 0.054766081273555756,
      "rewards/rejected": -0.12595215439796448,
      "step": 880
    },
    {
      "epoch": 0.06480503877380128,
      "grad_norm": 3.5555074291227964,
      "learning_rate": 2.6816060926159636e-06,
      "log_odds_chosen": 0.9256347417831421,
      "log_odds_ratio": -0.5201171636581421,
      "logits/chosen": -1.0642578601837158,
      "logits/rejected": -0.98046875,
      "logps/chosen": -0.7430664300918579,
      "logps/rejected": -1.408789038658142,
      "loss": 1.1157,
      "nll_loss": 0.9947265386581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07421875,
      "rewards/margins": 0.066558837890625,
      "rewards/rejected": -0.14091797173023224,
      "step": 890
    },
    {
      "epoch": 0.06553318527687771,
      "grad_norm": 1.4174933600999189,
      "learning_rate": 2.6666666666666664e-06,
      "log_odds_chosen": 0.6136230230331421,
      "log_odds_ratio": -0.5966796875,
      "logits/chosen": -1.002539038658142,
      "logits/rejected": -0.934765636920929,
      "logps/chosen": -0.752148449420929,
      "logps/rejected": -1.1818358898162842,
      "loss": 1.093,
      "nll_loss": 1.0529296398162842,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07524414360523224,
      "rewards/margins": 0.04295806959271431,
      "rewards/rejected": -0.11813964694738388,
      "step": 900
    },
    {
      "epoch": 0.06626133177995412,
      "grad_norm": 1.4890553701732194,
      "learning_rate": 2.6519741765271837e-06,
      "log_odds_chosen": 0.9593750238418579,
      "log_odds_ratio": -0.4740234315395355,
      "logits/chosen": -1.014257788658142,
      "logits/rejected": -0.920117199420929,
      "logps/chosen": -0.744921863079071,
      "logps/rejected": -1.415624976158142,
      "loss": 1.1001,
      "nll_loss": 1.103515625,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07451172173023224,
      "rewards/margins": 0.06707458198070526,
      "rewards/rejected": -0.14152832329273224,
      "step": 910
    },
    {
      "epoch": 0.06698947828303055,
      "grad_norm": 1.3975980730325914,
      "learning_rate": 2.637521893583148e-06,
      "log_odds_chosen": 0.848095715045929,
      "log_odds_ratio": -0.515429675579071,
      "logits/chosen": -0.9839843511581421,
      "logits/rejected": -0.9105468988418579,
      "logps/chosen": -0.764843761920929,
      "logps/rejected": -1.356835961341858,
      "loss": 1.1127,
      "nll_loss": 1.0896484851837158,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07646484673023224,
      "rewards/margins": 0.05929107591509819,
      "rewards/rejected": -0.13579101860523224,
      "step": 920
    },
    {
      "epoch": 0.06771762478610696,
      "grad_norm": 1.435023633429663,
      "learning_rate": 2.6233033431358115e-06,
      "log_odds_chosen": 0.843212902545929,
      "log_odds_ratio": -0.5230468511581421,
      "logits/chosen": -0.9898437261581421,
      "logits/rejected": -0.931445300579071,
      "logps/chosen": -0.745898425579071,
      "logps/rejected": -1.337499976158142,
      "loss": 1.0889,
      "nll_loss": 1.1142578125,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07463379204273224,
      "rewards/margins": 0.05906982347369194,
      "rewards/rejected": -0.13371582329273224,
      "step": 930
    },
    {
      "epoch": 0.06844577128918339,
      "grad_norm": 1.4935717264406958,
      "learning_rate": 2.6093122922137685e-06,
      "log_odds_chosen": 0.94183349609375,
      "log_odds_ratio": -0.5414062738418579,
      "logits/chosen": -0.9859374761581421,
      "logits/rejected": -0.926953136920929,
      "logps/chosen": -0.775683581829071,
      "logps/rejected": -1.462304711341858,
      "loss": 1.0815,
      "nll_loss": 1.078515648841858,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07757568359375,
      "rewards/margins": 0.06874694675207138,
      "rewards/rejected": -0.14619140326976776,
      "step": 940
    },
    {
      "epoch": 0.0691739177922598,
      "grad_norm": 1.510518929242062,
      "learning_rate": 2.5955427380922006e-06,
      "log_odds_chosen": 0.7543700933456421,
      "log_odds_ratio": -0.552539050579071,
      "logits/chosen": -1.027929663658142,
      "logits/rejected": -0.9564453363418579,
      "logps/chosen": -0.719433605670929,
      "logps/rejected": -1.2312500476837158,
      "loss": 1.1039,
      "nll_loss": 1.088281273841858,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07185058295726776,
      "rewards/margins": 0.05126190185546875,
      "rewards/rejected": -0.12324218451976776,
      "step": 950
    },
    {
      "epoch": 0.06990206429533623,
      "grad_norm": 1.4306650324913062,
      "learning_rate": 2.5819888974716113e-06,
      "log_odds_chosen": 0.940722644329071,
      "log_odds_ratio": -0.4866699278354645,
      "logits/chosen": -0.9966796636581421,
      "logits/rejected": -0.9052734375,
      "logps/chosen": -0.7251952886581421,
      "logps/rejected": -1.332421898841858,
      "loss": 1.092,
      "nll_loss": 1.0734374523162842,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07254638522863388,
      "rewards/margins": 0.06078185886144638,
      "rewards/rejected": -0.13332518935203552,
      "step": 960
    },
    {
      "epoch": 0.07063021079841264,
      "grad_norm": 2.329925696452697,
      "learning_rate": 2.5686451962717425e-06,
      "log_odds_chosen": 1.128076195716858,
      "log_odds_ratio": -0.46479493379592896,
      "logits/chosen": -1.0232422351837158,
      "logits/rejected": -0.9603515863418579,
      "logps/chosen": -0.7115234136581421,
      "logps/rejected": -1.5144531726837158,
      "loss": 1.0808,
      "nll_loss": 1.017578125,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07120361179113388,
      "rewards/margins": 0.08016662299633026,
      "rewards/rejected": -0.1514892578125,
      "step": 970
    },
    {
      "epoch": 0.07135835730148905,
      "grad_norm": 1.6432086166632125,
      "learning_rate": 2.5555062599997596e-06,
      "log_odds_chosen": 0.9437500238418579,
      "log_odds_ratio": -0.5213867425918579,
      "logits/chosen": -1.048437476158142,
      "logits/rejected": -0.9541015625,
      "logps/chosen": -0.7256835699081421,
      "logps/rejected": -1.3894531726837158,
      "loss": 1.1013,
      "nll_loss": 1.0392577648162842,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07248535007238388,
      "rewards/margins": 0.0663604736328125,
      "rewards/rejected": -0.13894042372703552,
      "step": 980
    },
    {
      "epoch": 0.07208650380456548,
      "grad_norm": 1.5872670734097316,
      "learning_rate": 2.5425669046549126e-06,
      "log_odds_chosen": 0.921679675579071,
      "log_odds_ratio": -0.47236329317092896,
      "logits/chosen": -0.975781261920929,
      "logits/rejected": -0.9267578125,
      "logps/chosen": -0.7533203363418579,
      "logps/rejected": -1.401757836341858,
      "loss": 1.1028,
      "nll_loss": 1.0740234851837158,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07534179836511612,
      "rewards/margins": 0.06479492038488388,
      "rewards/rejected": -0.1402587890625,
      "step": 990
    },
    {
      "epoch": 0.07281465030764189,
      "grad_norm": 1.4221976989012124,
      "learning_rate": 2.5298221281347034e-06,
      "log_odds_chosen": 1.090576171875,
      "log_odds_ratio": -0.48486328125,
      "logits/chosen": -1.0205078125,
      "logits/rejected": -0.9134765863418579,
      "logps/chosen": -0.702832043170929,
      "logps/rejected": -1.4716796875,
      "loss": 1.0746,
      "nll_loss": 0.968945324420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07027588039636612,
      "rewards/margins": 0.076873779296875,
      "rewards/rejected": -0.147216796875,
      "step": 1000
    },
    {
      "epoch": 0.07354279681071832,
      "grad_norm": 1.383077192382236,
      "learning_rate": 2.5172671021102103e-06,
      "log_odds_chosen": 0.972949206829071,
      "log_odds_ratio": -0.515820324420929,
      "logits/chosen": -1.012109398841858,
      "logits/rejected": -0.896679699420929,
      "logps/chosen": -0.741406261920929,
      "logps/rejected": -1.4412109851837158,
      "loss": 1.0932,
      "nll_loss": 0.9925781488418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07416991889476776,
      "rewards/margins": 0.07006530463695526,
      "rewards/rejected": -0.14414063096046448,
      "step": 1010
    },
    {
      "epoch": 0.07427094331379473,
      "grad_norm": 2.223136668653118,
      "learning_rate": 2.504897164340598e-06,
      "log_odds_chosen": 0.84228515625,
      "log_odds_ratio": -0.5210937261581421,
      "logits/chosen": -0.9996093511581421,
      "logits/rejected": -0.8960937261581421,
      "logps/chosen": -0.723437488079071,
      "logps/rejected": -1.326171875,
      "loss": 1.0865,
      "nll_loss": 0.9447265863418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07232666015625,
      "rewards/margins": 0.06026763841509819,
      "rewards/rejected": -0.13264159858226776,
      "step": 1020
    },
    {
      "epoch": 0.07499908981687116,
      "grad_norm": 1.529298401359949,
      "learning_rate": 2.492707811399023e-06,
      "log_odds_chosen": 0.994677722454071,
      "log_odds_ratio": -0.47016602754592896,
      "logits/chosen": -0.970898449420929,
      "logits/rejected": -0.8958984613418579,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -1.388085961341858,
      "loss": 1.104,
      "nll_loss": 1.0750000476837158,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07166747748851776,
      "rewards/margins": 0.06716613471508026,
      "rewards/rejected": -0.13876953721046448,
      "step": 1030
    },
    {
      "epoch": 0.07572723631994757,
      "grad_norm": 1.9274793719149392,
      "learning_rate": 2.480694691784169e-06,
      "log_odds_chosen": 1.168554663658142,
      "log_odds_ratio": -0.451904296875,
      "logits/chosen": -0.960156261920929,
      "logits/rejected": -0.9037109613418579,
      "logps/chosen": -0.660351574420929,
      "logps/rejected": -1.4646484851837158,
      "loss": 1.0873,
      "nll_loss": 0.9681640863418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06596679985523224,
      "rewards/margins": 0.0804443359375,
      "rewards/rejected": -0.14638671278953552,
      "step": 1040
    },
    {
      "epoch": 0.076455382823024,
      "grad_norm": 1.4681245416457267,
      "learning_rate": 2.4688535993934706e-06,
      "log_odds_chosen": 0.8275512456893921,
      "log_odds_ratio": -0.522216796875,
      "logits/chosen": -0.9496093988418579,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.767578125,
      "logps/rejected": -1.328710913658142,
      "loss": 1.0935,
      "nll_loss": 1.037109375,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07677002251148224,
      "rewards/margins": 0.05603943020105362,
      "rewards/rejected": -0.13288573920726776,
      "step": 1050
    },
    {
      "epoch": 0.07718352932610041,
      "grad_norm": 1.368834243715235,
      "learning_rate": 2.457180467335805e-06,
      "log_odds_chosen": 1.0066406726837158,
      "log_odds_ratio": -0.4501953125,
      "logits/chosen": -0.9873046875,
      "logits/rejected": -0.9017578363418579,
      "logps/chosen": -0.73828125,
      "logps/rejected": -1.4298827648162842,
      "loss": 1.1135,
      "nll_loss": 1.015625,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07381591945886612,
      "rewards/margins": 0.06925048679113388,
      "rewards/rejected": -0.1431884765625,
      "step": 1060
    },
    {
      "epoch": 0.07791167582917682,
      "grad_norm": 1.4152081073930696,
      "learning_rate": 2.4456713620629725e-06,
      "log_odds_chosen": 0.772229015827179,
      "log_odds_ratio": -0.541455090045929,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.925585925579071,
      "logps/chosen": -0.760546863079071,
      "logps/rejected": -1.306054711341858,
      "loss": 1.0868,
      "nll_loss": 1.053320288658142,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07602538913488388,
      "rewards/margins": 0.05467681959271431,
      "rewards/rejected": -0.1307373046875,
      "step": 1070
    },
    {
      "epoch": 0.07863982233225325,
      "grad_norm": 1.9822301530309727,
      "learning_rate": 2.4343224778007378e-06,
      "log_odds_chosen": 0.9384521245956421,
      "log_odds_ratio": -0.4916015565395355,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.934765636920929,
      "logps/chosen": -0.746386706829071,
      "logps/rejected": -1.4005858898162842,
      "loss": 1.0838,
      "nll_loss": 1.064062476158142,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0745849609375,
      "rewards/margins": 0.06540985405445099,
      "rewards/rejected": -0.13999024033546448,
      "step": 1080
    },
    {
      "epoch": 0.07936796883532966,
      "grad_norm": 2.202526241983929,
      "learning_rate": 2.4231301312615306e-06,
      "log_odds_chosen": 1.0329101085662842,
      "log_odds_ratio": -0.49482423067092896,
      "logits/chosen": -0.975781261920929,
      "logits/rejected": -0.9292968511581421,
      "logps/chosen": -0.716992199420929,
      "logps/rejected": -1.4386718273162842,
      "loss": 1.0802,
      "nll_loss": 1.048437476158142,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07170410454273224,
      "rewards/margins": 0.07221069186925888,
      "rewards/rejected": -0.14394530653953552,
      "step": 1090
    },
    {
      "epoch": 0.08009611533840609,
      "grad_norm": 1.6027988332078156,
      "learning_rate": 2.412090756622109e-06,
      "log_odds_chosen": 1.067169189453125,
      "log_odds_ratio": -0.48115235567092896,
      "logits/chosen": -1.029687523841858,
      "logits/rejected": -0.953906238079071,
      "logps/chosen": -0.7548828125,
      "logps/rejected": -1.5302734375,
      "loss": 1.0948,
      "nll_loss": 1.0304687023162842,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07551269233226776,
      "rewards/margins": 0.07766570895910263,
      "rewards/rejected": -0.15302734076976776,
      "step": 1100
    },
    {
      "epoch": 0.0808242618414825,
      "grad_norm": 1.643132733255971,
      "learning_rate": 2.401200900750657e-06,
      "log_odds_chosen": 0.902056872844696,
      "log_odds_ratio": -0.533007800579071,
      "logits/chosen": -1.0412108898162842,
      "logits/rejected": -0.946093738079071,
      "logps/chosen": -0.7572265863418579,
      "logps/rejected": -1.4031250476837158,
      "loss": 1.0726,
      "nll_loss": 1.0583984851837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07576904445886612,
      "rewards/margins": 0.06465759128332138,
      "rewards/rejected": -0.14042969048023224,
      "step": 1110
    },
    {
      "epoch": 0.08155240834455893,
      "grad_norm": 1.9226517362029898,
      "learning_rate": 2.390457218668787e-06,
      "log_odds_chosen": 1.058691382408142,
      "log_odds_ratio": -0.4754882752895355,
      "logits/chosen": -1.033593773841858,
      "logits/rejected": -0.941601574420929,
      "logps/chosen": -0.7318359613418579,
      "logps/rejected": -1.4845702648162842,
      "loss": 1.0736,
      "nll_loss": 1.0519530773162842,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07321777194738388,
      "rewards/margins": 0.07510223239660263,
      "rewards/rejected": -0.1484375,
      "step": 1120
    },
    {
      "epoch": 0.08228055484763534,
      "grad_norm": 1.5707523722744505,
      "learning_rate": 2.379856469234918e-06,
      "log_odds_chosen": 0.9898315668106079,
      "log_odds_ratio": -0.515429675579071,
      "logits/chosen": -0.990039050579071,
      "logits/rejected": -0.9574218988418579,
      "logps/chosen": -0.7496093511581421,
      "logps/rejected": -1.4255859851837158,
      "loss": 1.0878,
      "nll_loss": 1.101171851158142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07503662258386612,
      "rewards/margins": 0.06759033352136612,
      "rewards/rejected": -0.1427001953125,
      "step": 1130
    },
    {
      "epoch": 0.08300870135071177,
      "grad_norm": 1.5151634227882143,
      "learning_rate": 2.369395511036369e-06,
      "log_odds_chosen": 0.921337902545929,
      "log_odds_ratio": -0.5189453363418579,
      "logits/chosen": -1.0085937976837158,
      "logits/rejected": -0.949414074420929,
      "logps/chosen": -0.712695300579071,
      "logps/rejected": -1.3650391101837158,
      "loss": 1.0905,
      "nll_loss": 0.979296863079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.0712890625,
      "rewards/margins": 0.06532363593578339,
      "rewards/rejected": -0.1365966796875,
      "step": 1140
    },
    {
      "epoch": 0.08373684785378818,
      "grad_norm": 1.6354502026488225,
      "learning_rate": 2.359071298478354e-06,
      "log_odds_chosen": 1.1275146007537842,
      "log_odds_ratio": -0.452392578125,
      "logits/chosen": -1.0578124523162842,
      "logits/rejected": -0.9759765863418579,
      "logps/chosen": -0.7294921875,
      "logps/rejected": -1.506445288658142,
      "loss": 1.0588,
      "nll_loss": 1.0207030773162842,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07301025092601776,
      "rewards/margins": 0.07759933173656464,
      "rewards/rejected": -0.15061035752296448,
      "step": 1150
    },
    {
      "epoch": 0.0844649943568646,
      "grad_norm": 1.6252368308796055,
      "learning_rate": 2.3488808780588137e-06,
      "log_odds_chosen": 0.969042956829071,
      "log_odds_ratio": -0.4974609315395355,
      "logits/chosen": -1.017968773841858,
      "logits/rejected": -0.9468749761581421,
      "logps/chosen": -0.702929675579071,
      "logps/rejected": -1.384765625,
      "loss": 1.0991,
      "nll_loss": 0.9876953363418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07027588039636612,
      "rewards/margins": 0.06812133640050888,
      "rewards/rejected": -0.13837890326976776,
      "step": 1160
    },
    {
      "epoch": 0.08519314085994102,
      "grad_norm": 1.5239459289762518,
      "learning_rate": 2.3388213848187446e-06,
      "log_odds_chosen": 0.9083496332168579,
      "log_odds_ratio": -0.519726574420929,
      "logits/chosen": -1.035742163658142,
      "logits/rejected": -0.9693359136581421,
      "logps/chosen": -0.776562511920929,
      "logps/rejected": -1.4111328125,
      "loss": 1.0821,
      "nll_loss": 1.0128905773162842,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07764892280101776,
      "rewards/margins": 0.06345214694738388,
      "rewards/rejected": -0.14116211235523224,
      "step": 1170
    },
    {
      "epoch": 0.08592128736301743,
      "grad_norm": 1.565448796675523,
      "learning_rate": 2.328890038958328e-06,
      "log_odds_chosen": 1.08013916015625,
      "log_odds_ratio": -0.508056640625,
      "logits/chosen": -1.022851586341858,
      "logits/rejected": -0.943164050579071,
      "logps/chosen": -0.7431640625,
      "logps/rejected": -1.554101586341858,
      "loss": 1.0871,
      "nll_loss": 0.978515625,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07426758110523224,
      "rewards/margins": 0.08112182468175888,
      "rewards/rejected": -0.15544433891773224,
      "step": 1180
    },
    {
      "epoch": 0.08664943386609386,
      "grad_norm": 1.7610646634625353,
      "learning_rate": 2.3190841426097937e-06,
      "log_odds_chosen": 0.895751953125,
      "log_odds_ratio": -0.509570300579071,
      "logits/chosen": -1.0007812976837158,
      "logits/rejected": -0.921679675579071,
      "logps/chosen": -0.731640636920929,
      "logps/rejected": -1.3388671875,
      "loss": 1.0859,
      "nll_loss": 1.0378906726837158,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07314453274011612,
      "rewards/margins": 0.06079406663775444,
      "rewards/rejected": -0.13400879502296448,
      "step": 1190
    },
    {
      "epoch": 0.08737758036917027,
      "grad_norm": 1.4186266041359494,
      "learning_rate": 2.309401076758503e-06,
      "log_odds_chosen": 0.97607421875,
      "log_odds_ratio": -0.501660168170929,
      "logits/chosen": -1.032617211341858,
      "logits/rejected": -0.9296875,
      "logps/chosen": -0.736523449420929,
      "logps/rejected": -1.4294922351837158,
      "loss": 1.0894,
      "nll_loss": 1.083398461341858,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07362060248851776,
      "rewards/margins": 0.06939087063074112,
      "rewards/rejected": -0.14301757514476776,
      "step": 1200
    },
    {
      "epoch": 0.0881057268722467,
      "grad_norm": 1.562127406514597,
      "learning_rate": 2.299838298304276e-06,
      "log_odds_chosen": 0.9479004144668579,
      "log_odds_ratio": -0.4913085997104645,
      "logits/chosen": -1.005468726158142,
      "logits/rejected": -0.9271484613418579,
      "logps/chosen": -0.751171886920929,
      "logps/rejected": -1.411718726158142,
      "loss": 1.055,
      "nll_loss": 1.0447266101837158,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0750732421875,
      "rewards/margins": 0.06611251831054688,
      "rewards/rejected": -0.1412353515625,
      "step": 1210
    },
    {
      "epoch": 0.08883387337532311,
      "grad_norm": 1.5505309130527312,
      "learning_rate": 2.2903933372554728e-06,
      "log_odds_chosen": 1.0265624523162842,
      "log_odds_ratio": -0.45332032442092896,
      "logits/chosen": -1.0087890625,
      "logits/rejected": -0.912304699420929,
      "logps/chosen": -0.716992199420929,
      "logps/rejected": -1.415429711341858,
      "loss": 1.0463,
      "nll_loss": 0.968554675579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07176513969898224,
      "rewards/margins": 0.06972046196460724,
      "rewards/rejected": -0.14157715439796448,
      "step": 1220
    },
    {
      "epoch": 0.08956201987839954,
      "grad_norm": 1.5296659589041084,
      "learning_rate": 2.281063794048804e-06,
      "log_odds_chosen": 1.3583984375,
      "log_odds_ratio": -0.43110352754592896,
      "logits/chosen": -1.014062523841858,
      "logits/rejected": -0.926953136920929,
      "logps/chosen": -0.730664074420929,
      "logps/rejected": -1.696874976158142,
      "loss": 1.0678,
      "nll_loss": 1.011328101158142,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07313232123851776,
      "rewards/margins": 0.09641113132238388,
      "rewards/rejected": -0.16960449516773224,
      "step": 1230
    },
    {
      "epoch": 0.09029016638147595,
      "grad_norm": 1.4841860037908619,
      "learning_rate": 2.271847336988259e-06,
      "log_odds_chosen": 1.1979491710662842,
      "log_odds_ratio": -0.4358886778354645,
      "logits/chosen": -1.092187523841858,
      "logits/rejected": -0.991992175579071,
      "logps/chosen": -0.695507824420929,
      "logps/rejected": -1.5234375,
      "loss": 1.0795,
      "nll_loss": 0.979687511920929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06956787407398224,
      "rewards/margins": 0.08283539116382599,
      "rewards/rejected": -0.15234375,
      "step": 1240
    },
    {
      "epoch": 0.09101831288455237,
      "grad_norm": 1.6108566103762665,
      "learning_rate": 2.262741699796952e-06,
      "log_odds_chosen": 1.051782250404358,
      "log_odds_ratio": -0.49951171875,
      "logits/chosen": -1.0070312023162842,
      "logits/rejected": -0.9400390386581421,
      "logps/chosen": -0.782031238079071,
      "logps/rejected": -1.535546898841858,
      "loss": 1.0888,
      "nll_loss": 1.052734375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07827148586511612,
      "rewards/margins": 0.07554473727941513,
      "rewards/rejected": -0.15375976264476776,
      "step": 1250
    },
    {
      "epoch": 0.09174645938762879,
      "grad_norm": 1.4972463881173497,
      "learning_rate": 2.253744679276044e-06,
      "log_odds_chosen": 1.225622534751892,
      "log_odds_ratio": -0.4591308534145355,
      "logits/chosen": -1.000585913658142,
      "logits/rejected": -0.938671886920929,
      "logps/chosen": -0.698046863079071,
      "logps/rejected": -1.5841796398162842,
      "loss": 1.0734,
      "nll_loss": 0.998046875,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06973876804113388,
      "rewards/margins": 0.08869476616382599,
      "rewards/rejected": -0.158447265625,
      "step": 1260
    },
    {
      "epoch": 0.0924746058907052,
      "grad_norm": 1.8348572484361985,
      "learning_rate": 2.244854133065255e-06,
      "log_odds_chosen": 1.0773437023162842,
      "log_odds_ratio": -0.5150390863418579,
      "logits/chosen": -0.9906250238418579,
      "logits/rejected": -0.9466797113418579,
      "logps/chosen": -0.741406261920929,
      "logps/rejected": -1.539648413658142,
      "loss": 1.0932,
      "nll_loss": 1.0251953601837158,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07416991889476776,
      "rewards/margins": 0.07987823337316513,
      "rewards/rejected": -0.15402832627296448,
      "step": 1270
    },
    {
      "epoch": 0.09320275239378163,
      "grad_norm": 1.6133142568483645,
      "learning_rate": 2.2360679774997895e-06,
      "log_odds_chosen": 0.969921886920929,
      "log_odds_ratio": -0.4927734434604645,
      "logits/chosen": -0.973437488079071,
      "logits/rejected": -0.9039062261581421,
      "logps/chosen": -0.7376953363418579,
      "logps/rejected": -1.412500023841858,
      "loss": 1.0842,
      "nll_loss": 1.0671875476837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07375488430261612,
      "rewards/margins": 0.06759033352136612,
      "rewards/rejected": -0.14133301377296448,
      "step": 1280
    },
    {
      "epoch": 0.09393089889685804,
      "grad_norm": 1.6775542759003017,
      "learning_rate": 2.2273841855588183e-06,
      "log_odds_chosen": 1.2883789539337158,
      "log_odds_ratio": -0.42138671875,
      "logits/chosen": -1.088281273841858,
      "logits/rejected": -0.993945300579071,
      "logps/chosen": -0.7162109613418579,
      "logps/rejected": -1.618749976158142,
      "loss": 1.0621,
      "nll_loss": 1.0001952648162842,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07164306938648224,
      "rewards/margins": 0.09030761569738388,
      "rewards/rejected": -0.16184082627296448,
      "step": 1290
    },
    {
      "epoch": 0.09465904539993447,
      "grad_norm": 1.507207195221332,
      "learning_rate": 2.2188007849009167e-06,
      "log_odds_chosen": 1.1524779796600342,
      "log_odds_ratio": -0.46198731660842896,
      "logits/chosen": -1.001953125,
      "logits/rejected": -0.9205077886581421,
      "logps/chosen": -0.708203136920929,
      "logps/rejected": -1.552148461341858,
      "loss": 1.0799,
      "nll_loss": 1.0417969226837158,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07082519680261612,
      "rewards/margins": 0.0844573974609375,
      "rewards/rejected": -0.15507812798023224,
      "step": 1300
    },
    {
      "epoch": 0.09538719190301088,
      "grad_norm": 1.8051144128159562,
      "learning_rate": 2.2103158559821502e-06,
      "log_odds_chosen": 0.9556640386581421,
      "log_odds_ratio": -0.5223633050918579,
      "logits/chosen": -1.033593773841858,
      "logits/rejected": -0.9564453363418579,
      "logps/chosen": -0.7701171636581421,
      "logps/rejected": -1.4884765148162842,
      "loss": 1.0495,
      "nll_loss": 1.0460937023162842,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07700195163488388,
      "rewards/margins": 0.07175903022289276,
      "rewards/rejected": -0.14890137314796448,
      "step": 1310
    },
    {
      "epoch": 0.09611533840608731,
      "grad_norm": 1.616030602353496,
      "learning_rate": 2.2019275302527213e-06,
      "log_odds_chosen": 1.0687987804412842,
      "log_odds_ratio": -0.46826171875,
      "logits/chosen": -0.9800781011581421,
      "logits/rejected": -0.9302734136581421,
      "logps/chosen": -0.7474609613418579,
      "logps/rejected": -1.5080077648162842,
      "loss": 1.057,
      "nll_loss": 1.0576171875,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07470703125,
      "rewards/margins": 0.076141357421875,
      "rewards/rejected": -0.15083007514476776,
      "step": 1320
    },
    {
      "epoch": 0.09684348490916372,
      "grad_norm": 1.5231051481214068,
      "learning_rate": 2.193633988428327e-06,
      "log_odds_chosen": 1.095849633216858,
      "log_odds_ratio": -0.46333009004592896,
      "logits/chosen": -1.0291016101837158,
      "logits/rejected": -0.9437500238418579,
      "logps/chosen": -0.680859386920929,
      "logps/rejected": -1.4445312023162842,
      "loss": 1.0492,
      "nll_loss": 1.014062523841858,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06807861477136612,
      "rewards/margins": 0.07635498046875,
      "rewards/rejected": -0.14453125,
      "step": 1330
    },
    {
      "epoch": 0.09757163141224014,
      "grad_norm": 1.531604982147399,
      "learning_rate": 2.185433458832612e-06,
      "log_odds_chosen": 1.081030249595642,
      "log_odds_ratio": -0.4999023377895355,
      "logits/chosen": -0.990039050579071,
      "logits/rejected": -0.921679675579071,
      "logps/chosen": -0.762890636920929,
      "logps/rejected": -1.5333983898162842,
      "loss": 1.0893,
      "nll_loss": 1.128515601158142,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0762939453125,
      "rewards/margins": 0.07698974758386612,
      "rewards/rejected": -0.15327148139476776,
      "step": 1340
    },
    {
      "epoch": 0.09829977791531656,
      "grad_norm": 1.7209422087310642,
      "learning_rate": 2.177324215807269e-06,
      "log_odds_chosen": 0.7672363519668579,
      "log_odds_ratio": -0.552441418170929,
      "logits/chosen": -1.064843773841858,
      "logits/rejected": -0.991406261920929,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -1.2277343273162842,
      "loss": 1.0525,
      "nll_loss": 0.936328113079071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07163085788488388,
      "rewards/margins": 0.05113830417394638,
      "rewards/rejected": -0.12287597358226776,
      "step": 1350
    },
    {
      "epoch": 0.09902792441839298,
      "grad_norm": 1.4920770020081804,
      "learning_rate": 2.1693045781865616e-06,
      "log_odds_chosen": 1.0016601085662842,
      "log_odds_ratio": -0.48388671875,
      "logits/chosen": -1.0632812976837158,
      "logits/rejected": -0.975390613079071,
      "logps/chosen": -0.706835925579071,
      "logps/rejected": -1.3830077648162842,
      "loss": 1.0659,
      "nll_loss": 1.058007836341858,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07062987983226776,
      "rewards/margins": 0.06766357272863388,
      "rewards/rejected": -0.1383056640625,
      "step": 1360
    },
    {
      "epoch": 0.0997560709214694,
      "grad_norm": 1.8918684639580574,
      "learning_rate": 2.1613729078331965e-06,
      "log_odds_chosen": 1.3237793445587158,
      "log_odds_ratio": -0.4039062559604645,
      "logits/chosen": -1.0458984375,
      "logits/rejected": -0.994335949420929,
      "logps/chosen": -0.695507824420929,
      "logps/rejected": -1.6281249523162842,
      "loss": 1.0803,
      "nll_loss": 1.0031249523162842,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06960449367761612,
      "rewards/margins": 0.09326782077550888,
      "rewards/rejected": -0.16286620497703552,
      "step": 1370
    },
    {
      "epoch": 0.10048421742454582,
      "grad_norm": 1.5676314373127447,
      "learning_rate": 2.1535276082326617e-06,
      "log_odds_chosen": 1.0940430164337158,
      "log_odds_ratio": -0.49433594942092896,
      "logits/chosen": -1.052734375,
      "logits/rejected": -0.975781261920929,
      "logps/chosen": -0.670703113079071,
      "logps/rejected": -1.4083983898162842,
      "loss": 1.0669,
      "nll_loss": 0.9701172113418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06704101711511612,
      "rewards/margins": 0.07375182956457138,
      "rewards/rejected": -0.1407470703125,
      "step": 1380
    },
    {
      "epoch": 0.10121236392762224,
      "grad_norm": 1.6226410989304125,
      "learning_rate": 2.14576712314328e-06,
      "log_odds_chosen": 1.211523413658142,
      "log_odds_ratio": -0.46953123807907104,
      "logits/chosen": -0.9800781011581421,
      "logits/rejected": -0.9013671875,
      "logps/chosen": -0.6998046636581421,
      "logps/rejected": -1.5490233898162842,
      "loss": 1.0453,
      "nll_loss": 1.0148437023162842,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06998290866613388,
      "rewards/margins": 0.08496703952550888,
      "rewards/rejected": -0.15493163466453552,
      "step": 1390
    },
    {
      "epoch": 0.10194051043069866,
      "grad_norm": 1.5153257120800114,
      "learning_rate": 2.138089935299395e-06,
      "log_odds_chosen": 1.2509765625,
      "log_odds_ratio": -0.4617675840854645,
      "logits/chosen": -1.0714843273162842,
      "logits/rejected": -0.9873046875,
      "logps/chosen": -0.7191406488418579,
      "logps/rejected": -1.635156273841858,
      "loss": 1.0624,
      "nll_loss": 0.972851574420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07192382961511612,
      "rewards/margins": 0.09164123237133026,
      "rewards/rejected": -0.16357421875,
      "step": 1400
    },
    {
      "epoch": 0.10266865693377508,
      "grad_norm": 1.547222073839455,
      "learning_rate": 2.1304945651652297e-06,
      "log_odds_chosen": 1.2150390148162842,
      "log_odds_ratio": -0.45502930879592896,
      "logits/chosen": -1.034765601158142,
      "logits/rejected": -0.954296886920929,
      "logps/chosen": -0.689746081829071,
      "logps/rejected": -1.5789062976837158,
      "loss": 1.0458,
      "nll_loss": 1.0642578601837158,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06899414211511612,
      "rewards/margins": 0.08889465034008026,
      "rewards/rejected": -0.157958984375,
      "step": 1410
    },
    {
      "epoch": 0.1033968034368515,
      "grad_norm": 1.5809340493267345,
      "learning_rate": 2.122979569737101e-06,
      "log_odds_chosen": 1.0255858898162842,
      "log_odds_ratio": -0.46245115995407104,
      "logits/chosen": -1.041601538658142,
      "logits/rejected": -0.9517577886581421,
      "logps/chosen": -0.7085937261581421,
      "logps/rejected": -1.4210937023162842,
      "loss": 1.0833,
      "nll_loss": 1.022851586341858,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07087402045726776,
      "rewards/margins": 0.07132568210363388,
      "rewards/rejected": -0.14211425185203552,
      "step": 1420
    },
    {
      "epoch": 0.10412494993992791,
      "grad_norm": 1.5964919924770689,
      "learning_rate": 2.11554354139178e-06,
      "log_odds_chosen": 0.9452148675918579,
      "log_odds_ratio": -0.533496081829071,
      "logits/chosen": -1.1056640148162842,
      "logits/rejected": -1.008203148841858,
      "logps/chosen": -0.765625,
      "logps/rejected": -1.4462890625,
      "loss": 1.0598,
      "nll_loss": 1.0818359851837158,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07664795219898224,
      "rewards/margins": 0.06786040961742401,
      "rewards/rejected": -0.14448241889476776,
      "step": 1430
    },
    {
      "epoch": 0.10485309644300433,
      "grad_norm": 1.5760409793442225,
      "learning_rate": 2.1081851067789196e-06,
      "log_odds_chosen": 1.1474609375,
      "log_odds_ratio": -0.48115235567092896,
      "logits/chosen": -1.045507788658142,
      "logits/rejected": -0.950390636920929,
      "logps/chosen": -0.7152343988418579,
      "logps/rejected": -1.537500023841858,
      "loss": 1.0492,
      "nll_loss": 0.9341796636581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07148437201976776,
      "rewards/margins": 0.08223877102136612,
      "rewards/rejected": -0.15383300185203552,
      "step": 1440
    },
    {
      "epoch": 0.10558124294608075,
      "grad_norm": 1.551482511303002,
      "learning_rate": 2.1009029257555606e-06,
      "log_odds_chosen": 1.193457007408142,
      "log_odds_ratio": -0.4422851502895355,
      "logits/chosen": -1.006250023841858,
      "logits/rejected": -0.892773449420929,
      "logps/chosen": -0.697070300579071,
      "logps/rejected": -1.5128905773162842,
      "loss": 1.0492,
      "nll_loss": 0.9736328125,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06971435248851776,
      "rewards/margins": 0.08153076469898224,
      "rewards/rejected": -0.1512451171875,
      "step": 1450
    },
    {
      "epoch": 0.10630938944915717,
      "grad_norm": 1.6128891531028628,
      "learning_rate": 2.0936956903608545e-06,
      "log_odds_chosen": 1.230859398841858,
      "log_odds_ratio": -0.455078125,
      "logits/chosen": -1.019921898841858,
      "logits/rejected": -0.9375,
      "logps/chosen": -0.7007812261581421,
      "logps/rejected": -1.599609375,
      "loss": 1.0249,
      "nll_loss": 0.930859386920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07009277492761612,
      "rewards/margins": 0.089935302734375,
      "rewards/rejected": -0.15998534858226776,
      "step": 1460
    },
    {
      "epoch": 0.10703753595223359,
      "grad_norm": 1.8501092889250195,
      "learning_rate": 2.0865621238292046e-06,
      "log_odds_chosen": 0.9693359136581421,
      "log_odds_ratio": -0.5165039300918579,
      "logits/chosen": -0.9957031011581421,
      "logits/rejected": -0.9037109613418579,
      "logps/chosen": -0.751953125,
      "logps/rejected": -1.457617163658142,
      "loss": 1.0762,
      "nll_loss": 1.015234351158142,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07513427734375,
      "rewards/margins": 0.070587158203125,
      "rewards/rejected": -0.14589843153953552,
      "step": 1470
    },
    {
      "epoch": 0.10776568245531001,
      "grad_norm": 1.535500647707011,
      "learning_rate": 2.079500979640145e-06,
      "log_odds_chosen": 1.0024902820587158,
      "log_odds_ratio": -0.4825195372104645,
      "logits/chosen": -1.027929663658142,
      "logits/rejected": -0.9234374761581421,
      "logps/chosen": -0.7740234136581421,
      "logps/rejected": -1.4939453601837158,
      "loss": 1.0627,
      "nll_loss": 0.988476574420929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07738037407398224,
      "rewards/margins": 0.07195739448070526,
      "rewards/rejected": -0.14926758408546448,
      "step": 1480
    },
    {
      "epoch": 0.10849382895838643,
      "grad_norm": 1.5811838534620308,
      "learning_rate": 2.072511040603359e-06,
      "log_odds_chosen": 1.189453125,
      "log_odds_ratio": -0.4715820252895355,
      "logits/chosen": -1.0185546875,
      "logits/rejected": -0.9228515625,
      "logps/chosen": -0.730273425579071,
      "logps/rejected": -1.60546875,
      "loss": 1.0477,
      "nll_loss": 0.983203113079071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07296142727136612,
      "rewards/margins": 0.08760376274585724,
      "rewards/rejected": -0.16049805283546448,
      "step": 1490
    },
    {
      "epoch": 0.10922197546146285,
      "grad_norm": 1.6359205790422995,
      "learning_rate": 2.065591117977289e-06,
      "log_odds_chosen": 1.2340209484100342,
      "log_odds_ratio": -0.4440673887729645,
      "logits/chosen": -0.988085925579071,
      "logits/rejected": -0.898242175579071,
      "logps/chosen": -0.695996105670929,
      "logps/rejected": -1.567968726158142,
      "loss": 1.0545,
      "nll_loss": 1.014257788658142,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06960449367761612,
      "rewards/margins": 0.08713836967945099,
      "rewards/rejected": -0.15676268935203552,
      "step": 1500
    },
    {
      "epoch": 0.10995012196453927,
      "grad_norm": 1.6210576908156278,
      "learning_rate": 2.058740050619915e-06,
      "log_odds_chosen": 1.2014648914337158,
      "log_odds_ratio": -0.40815430879592896,
      "logits/chosen": -1.038476586341858,
      "logits/rejected": -0.9828125238418579,
      "logps/chosen": -0.640625,
      "logps/rejected": -1.4357421398162842,
      "loss": 1.045,
      "nll_loss": 0.970507800579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06407471001148224,
      "rewards/margins": 0.07945556938648224,
      "rewards/rejected": -0.1435546875,
      "step": 1510
    },
    {
      "epoch": 0.11067826846761568,
      "grad_norm": 1.5094454583682124,
      "learning_rate": 2.0519567041703083e-06,
      "log_odds_chosen": 0.9427490234375,
      "log_odds_ratio": -0.49150389432907104,
      "logits/chosen": -1.041406273841858,
      "logits/rejected": -0.9345703125,
      "logps/chosen": -0.7076171636581421,
      "logps/rejected": -1.310937523841858,
      "loss": 1.0642,
      "nll_loss": 0.9994140863418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07076416164636612,
      "rewards/margins": 0.060192108154296875,
      "rewards/rejected": -0.1309814453125,
      "step": 1520
    },
    {
      "epoch": 0.1114064149706921,
      "grad_norm": 1.5746253179960445,
      "learning_rate": 2.0452399702596544e-06,
      "log_odds_chosen": 1.3107421398162842,
      "log_odds_ratio": -0.45263671875,
      "logits/chosen": -1.0205078125,
      "logits/rejected": -0.9203125238418579,
      "logps/chosen": -0.7134765386581421,
      "logps/rejected": -1.6863281726837158,
      "loss": 1.0531,
      "nll_loss": 1.031640648841858,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07138671725988388,
      "rewards/margins": 0.09716796875,
      "rewards/rejected": -0.16853027045726776,
      "step": 1530
    },
    {
      "epoch": 0.11213456147376852,
      "grad_norm": 1.6428800506670196,
      "learning_rate": 2.0385887657505017e-06,
      "log_odds_chosen": 0.9951171875,
      "log_odds_ratio": -0.5116211175918579,
      "logits/chosen": -0.987109363079071,
      "logits/rejected": -0.91015625,
      "logps/chosen": -0.701367199420929,
      "logps/rejected": -1.375,
      "loss": 1.049,
      "nll_loss": 0.955078125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07011719048023224,
      "rewards/margins": 0.06721191108226776,
      "rewards/rejected": -0.13728027045726776,
      "step": 1540
    },
    {
      "epoch": 0.11286270797684494,
      "grad_norm": 1.7743280649885278,
      "learning_rate": 2.032002032003048e-06,
      "log_odds_chosen": 1.0622069835662842,
      "log_odds_ratio": -0.4854980409145355,
      "logits/chosen": -1.019140601158142,
      "logits/rejected": -0.9365234375,
      "logps/chosen": -0.7261718511581421,
      "logps/rejected": -1.482031226158142,
      "loss": 1.0583,
      "nll_loss": 1.0095703601837158,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0726318359375,
      "rewards/margins": 0.075592041015625,
      "rewards/rejected": -0.14816895127296448,
      "step": 1550
    },
    {
      "epoch": 0.11359085447992136,
      "grad_norm": 1.6231324220953163,
      "learning_rate": 2.025478734167333e-06,
      "log_odds_chosen": 1.053613305091858,
      "log_odds_ratio": -0.4950195252895355,
      "logits/chosen": -0.993359386920929,
      "logits/rejected": -0.906054675579071,
      "logps/chosen": -0.745898425579071,
      "logps/rejected": -1.5128905773162842,
      "loss": 1.0373,
      "nll_loss": 1.0216796398162842,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07468261569738388,
      "rewards/margins": 0.07673492282629013,
      "rewards/rejected": -0.15141601860523224,
      "step": 1560
    },
    {
      "epoch": 0.11431900098299778,
      "grad_norm": 1.448829911711827,
      "learning_rate": 2.0190178605002747e-06,
      "log_odds_chosen": 1.182714819908142,
      "log_odds_ratio": -0.4615234434604645,
      "logits/chosen": -1.013281226158142,
      "logits/rejected": -0.9166015386581421,
      "logps/chosen": -0.7081054449081421,
      "logps/rejected": -1.539648413658142,
      "loss": 1.0537,
      "nll_loss": 0.97265625,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07088623195886612,
      "rewards/margins": 0.08308105170726776,
      "rewards/rejected": -0.15396729111671448,
      "step": 1570
    },
    {
      "epoch": 0.1150471474860742,
      "grad_norm": 1.8672995427915349,
      "learning_rate": 2.0126184217065104e-06,
      "log_odds_chosen": 1.3938477039337158,
      "log_odds_ratio": -0.428466796875,
      "logits/chosen": -0.990429699420929,
      "logits/rejected": -0.891796886920929,
      "logps/chosen": -0.706250011920929,
      "logps/rejected": -1.7068359851837158,
      "loss": 1.0567,
      "nll_loss": 0.9996093511581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07064209133386612,
      "rewards/margins": 0.09992065280675888,
      "rewards/rejected": -0.17062988877296448,
      "step": 1580
    },
    {
      "epoch": 0.11577529398915062,
      "grad_norm": 2.0174321197396736,
      "learning_rate": 2.0062794503020765e-06,
      "log_odds_chosen": 1.16357421875,
      "log_odds_ratio": -0.4569335877895355,
      "logits/chosen": -1.005468726158142,
      "logits/rejected": -0.896289050579071,
      "logps/chosen": -0.689453125,
      "logps/rejected": -1.5466797351837158,
      "loss": 1.0499,
      "nll_loss": 1.036523461341858,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06893310695886612,
      "rewards/margins": 0.08577270805835724,
      "rewards/rejected": -0.1546630859375,
      "step": 1590
    },
    {
      "epoch": 0.11650344049222704,
      "grad_norm": 1.5577324609982013,
      "learning_rate": 2e-06,
      "log_odds_chosen": 1.169335961341858,
      "log_odds_ratio": -0.4536376893520355,
      "logits/chosen": -1.0144531726837158,
      "logits/rejected": -0.9261718988418579,
      "logps/chosen": -0.7152343988418579,
      "logps/rejected": -1.5710937976837158,
      "loss": 1.0785,
      "nll_loss": 1.0417969226837158,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07155761867761612,
      "rewards/margins": 0.0855712890625,
      "rewards/rejected": -0.15708008408546448,
      "step": 1600
    },
    {
      "epoch": 0.11723158699530345,
      "grad_norm": 1.6379848934154986,
      "learning_rate": 1.993779145116907e-06,
      "log_odds_chosen": 1.309179663658142,
      "log_odds_ratio": -0.42583006620407104,
      "logits/chosen": -0.9957031011581421,
      "logits/rejected": -0.8857421875,
      "logps/chosen": -0.7095702886581421,
      "logps/rejected": -1.640625,
      "loss": 1.0324,
      "nll_loss": 0.9931640625,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07093505561351776,
      "rewards/margins": 0.09320678561925888,
      "rewards/rejected": -0.1641845703125,
      "step": 1610
    },
    {
      "epoch": 0.11795973349837988,
      "grad_norm": 1.6026185532394994,
      "learning_rate": 1.987615979999813e-06,
      "log_odds_chosen": 1.145898461341858,
      "log_odds_ratio": -0.45966798067092896,
      "logits/chosen": -1.014062523841858,
      "logits/rejected": -0.9117187261581421,
      "logps/chosen": -0.7115234136581421,
      "logps/rejected": -1.527929663658142,
      "loss": 1.0287,
      "nll_loss": 0.9453125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07115478813648224,
      "rewards/margins": 0.08154602348804474,
      "rewards/rejected": -0.15263672173023224,
      "step": 1620
    },
    {
      "epoch": 0.11868788000145629,
      "grad_norm": 1.7969036712533162,
      "learning_rate": 1.9815096184722797e-06,
      "log_odds_chosen": 1.2506835460662842,
      "log_odds_ratio": -0.4867187440395355,
      "logits/chosen": -0.955078125,
      "logits/rejected": -0.877734363079071,
      "logps/chosen": -0.7259765863418579,
      "logps/rejected": -1.6574218273162842,
      "loss": 1.0561,
      "nll_loss": 0.9789062738418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07258300483226776,
      "rewards/margins": 0.09311523288488388,
      "rewards/rejected": -0.16574707627296448,
      "step": 1630
    },
    {
      "epoch": 0.11941602650453272,
      "grad_norm": 1.5222598488761336,
      "learning_rate": 1.9754591932991793e-06,
      "log_odds_chosen": 1.148339867591858,
      "log_odds_ratio": -0.455322265625,
      "logits/chosen": -0.9375,
      "logits/rejected": -0.8656250238418579,
      "logps/chosen": -0.7069336175918579,
      "logps/rejected": -1.485937476158142,
      "loss": 1.019,
      "nll_loss": 0.9462890625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07071533054113388,
      "rewards/margins": 0.077911376953125,
      "rewards/rejected": -0.14858397841453552,
      "step": 1640
    },
    {
      "epoch": 0.12014417300760913,
      "grad_norm": 1.909481026394746,
      "learning_rate": 1.9694638556693235e-06,
      "log_odds_chosen": 1.142968773841858,
      "log_odds_ratio": -0.4237304627895355,
      "logits/chosen": -0.983203113079071,
      "logits/rejected": -0.9037109613418579,
      "logps/chosen": -0.636914074420929,
      "logps/rejected": -1.407812476158142,
      "loss": 1.043,
      "nll_loss": 0.990429699420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06368408352136612,
      "rewards/margins": 0.07710571587085724,
      "rewards/rejected": -0.14082030951976776,
      "step": 1650
    },
    {
      "epoch": 0.12087231951068556,
      "grad_norm": 1.544957152191981,
      "learning_rate": 1.963522774695264e-06,
      "log_odds_chosen": 1.455664038658142,
      "log_odds_ratio": -0.4110351502895355,
      "logits/chosen": -1.022070288658142,
      "logits/rejected": -0.94140625,
      "logps/chosen": -0.67919921875,
      "logps/rejected": -1.7333984375,
      "loss": 1.0146,
      "nll_loss": 0.9146484136581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06793212890625,
      "rewards/margins": 0.10528564453125,
      "rewards/rejected": -0.17329101264476776,
      "step": 1660
    },
    {
      "epoch": 0.12160046601376197,
      "grad_norm": 1.691361663087259,
      "learning_rate": 1.9576351369295853e-06,
      "log_odds_chosen": 1.160546898841858,
      "log_odds_ratio": -0.43828123807907104,
      "logits/chosen": -1.0314452648162842,
      "logits/rejected": -0.931835949420929,
      "logps/chosen": -0.6924804449081421,
      "logps/rejected": -1.48046875,
      "loss": 1.0242,
      "nll_loss": 1.005859375,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06927490234375,
      "rewards/margins": 0.0787353515625,
      "rewards/rejected": -0.1480712890625,
      "step": 1670
    },
    {
      "epoch": 0.1223286125168384,
      "grad_norm": 1.4603663949519994,
      "learning_rate": 1.951800145897066e-06,
      "log_odds_chosen": 1.5099608898162842,
      "log_odds_ratio": -0.412841796875,
      "logits/chosen": -1.0283203125,
      "logits/rejected": -0.9009765386581421,
      "logps/chosen": -0.7222656011581421,
      "logps/rejected": -1.8156249523162842,
      "loss": 1.0419,
      "nll_loss": 0.966601550579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07222900539636612,
      "rewards/margins": 0.109375,
      "rewards/rejected": -0.18154296278953552,
      "step": 1680
    },
    {
      "epoch": 0.12305675901991481,
      "grad_norm": 1.683093672443576,
      "learning_rate": 1.9460170216420797e-06,
      "log_odds_chosen": 1.371826171875,
      "log_odds_ratio": -0.4059081971645355,
      "logits/chosen": -1.0158202648162842,
      "logits/rejected": -0.9189453125,
      "logps/chosen": -0.6650390625,
      "logps/rejected": -1.6408202648162842,
      "loss": 1.0129,
      "nll_loss": 0.955273449420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06650390475988388,
      "rewards/margins": 0.09738769382238388,
      "rewards/rejected": -0.16386719048023224,
      "step": 1690
    },
    {
      "epoch": 0.12378490552299122,
      "grad_norm": 1.61091241435004,
      "learning_rate": 1.9402850002906637e-06,
      "log_odds_chosen": 1.0524780750274658,
      "log_odds_ratio": -0.5194336175918579,
      "logits/chosen": -0.971875011920929,
      "logits/rejected": -0.901562511920929,
      "logps/chosen": -0.6923828125,
      "logps/rejected": -1.435156226158142,
      "loss": 1.0037,
      "nll_loss": 0.9681640863418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06923828274011612,
      "rewards/margins": 0.07434539496898651,
      "rewards/rejected": -0.14357909560203552,
      "step": 1700
    },
    {
      "epoch": 0.12451305202606765,
      "grad_norm": 1.7458372462703642,
      "learning_rate": 1.9346033336266974e-06,
      "log_odds_chosen": 0.9312499761581421,
      "log_odds_ratio": -0.4717773497104645,
      "logits/chosen": -1.0,
      "logits/rejected": -0.903124988079071,
      "logps/chosen": -0.678027331829071,
      "logps/rejected": -1.297265648841858,
      "loss": 1.0423,
      "nll_loss": 0.9349609613418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06783447414636612,
      "rewards/margins": 0.06181640550494194,
      "rewards/rejected": -0.12971191108226776,
      "step": 1710
    },
    {
      "epoch": 0.12524119852914406,
      "grad_norm": 1.5806300632378767,
      "learning_rate": 1.9289712886816486e-06,
      "log_odds_chosen": 1.1687500476837158,
      "log_odds_ratio": -0.4324707090854645,
      "logits/chosen": -1.0304687023162842,
      "logits/rejected": -0.908007800579071,
      "logps/chosen": -0.736523449420929,
      "logps/rejected": -1.581640601158142,
      "loss": 1.0356,
      "nll_loss": 1.010156273841858,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07365722954273224,
      "rewards/margins": 0.08444824069738388,
      "rewards/rejected": -0.15815429389476776,
      "step": 1720
    },
    {
      "epoch": 0.1259693450322205,
      "grad_norm": 1.7187327408762862,
      "learning_rate": 1.92338814733738e-06,
      "log_odds_chosen": 1.0644042491912842,
      "log_odds_ratio": -0.46147459745407104,
      "logits/chosen": -1.001953125,
      "logits/rejected": -0.921679675579071,
      "logps/chosen": -0.7040039300918579,
      "logps/rejected": -1.406835913658142,
      "loss": 1.0049,
      "nll_loss": 0.957226574420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07036133110523224,
      "rewards/margins": 0.0702667236328125,
      "rewards/rejected": -0.14067383110523224,
      "step": 1730
    },
    {
      "epoch": 0.1266974915352969,
      "grad_norm": 1.3964795279251796,
      "learning_rate": 1.9178532059415367e-06,
      "log_odds_chosen": 0.9427734613418579,
      "log_odds_ratio": -0.508349597454071,
      "logits/chosen": -1.0361328125,
      "logits/rejected": -0.932812511920929,
      "logps/chosen": -0.7269531488418579,
      "logps/rejected": -1.367773413658142,
      "loss": 1.0304,
      "nll_loss": 0.917773425579071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07272949069738388,
      "rewards/margins": 0.06398315727710724,
      "rewards/rejected": -0.13666991889476776,
      "step": 1740
    },
    {
      "epoch": 0.1274256380383733,
      "grad_norm": 1.5914585831593873,
      "learning_rate": 1.9123657749350298e-06,
      "log_odds_chosen": 1.140905737876892,
      "log_odds_ratio": -0.4703125059604645,
      "logits/chosen": -0.998828113079071,
      "logits/rejected": -0.9322265386581421,
      "logps/chosen": -0.7279297113418579,
      "logps/rejected": -1.536523461341858,
      "loss": 1.029,
      "nll_loss": 1.011132836341858,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07285156100988388,
      "rewards/margins": 0.08081360161304474,
      "rewards/rejected": -0.15358886122703552,
      "step": 1750
    },
    {
      "epoch": 0.12815378454144974,
      "grad_norm": 1.4478596476264443,
      "learning_rate": 1.9069251784911844e-06,
      "log_odds_chosen": 1.255761742591858,
      "log_odds_ratio": -0.43095701932907104,
      "logits/chosen": -0.9482421875,
      "logits/rejected": -0.830273449420929,
      "logps/chosen": -0.68359375,
      "logps/rejected": -1.570703148841858,
      "loss": 1.0189,
      "nll_loss": 1.0222656726837158,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06833495944738388,
      "rewards/margins": 0.08867187798023224,
      "rewards/rejected": -0.15700682997703552,
      "step": 1760
    },
    {
      "epoch": 0.12888193104452617,
      "grad_norm": 1.9255632739901059,
      "learning_rate": 1.9015307541661132e-06,
      "log_odds_chosen": 1.0532715320587158,
      "log_odds_ratio": -0.5038086175918579,
      "logits/chosen": -0.9976562261581421,
      "logits/rejected": -0.918749988079071,
      "logps/chosen": -0.7562500238418579,
      "logps/rejected": -1.5353515148162842,
      "loss": 1.0406,
      "nll_loss": 0.999804675579071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07567138969898224,
      "rewards/margins": 0.07784118503332138,
      "rewards/rejected": -0.15336914360523224,
      "step": 1770
    },
    {
      "epoch": 0.12961007754760256,
      "grad_norm": 1.629459324851094,
      "learning_rate": 1.8961818525599089e-06,
      "log_odds_chosen": 1.028100609779358,
      "log_odds_ratio": -0.486328125,
      "logits/chosen": -0.981640636920929,
      "logits/rejected": -0.9017578363418579,
      "logps/chosen": -0.726757824420929,
      "logps/rejected": -1.451171875,
      "loss": 1.0198,
      "nll_loss": 0.9769531488418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07271728664636612,
      "rewards/margins": 0.07234954833984375,
      "rewards/rejected": -0.1451416015625,
      "step": 1780
    },
    {
      "epoch": 0.130338224050679,
      "grad_norm": 1.6428303502174528,
      "learning_rate": 1.890877836988262e-06,
      "log_odds_chosen": 1.18310546875,
      "log_odds_ratio": -0.4529785215854645,
      "logits/chosen": -1.0041015148162842,
      "logits/rejected": -0.88671875,
      "logps/chosen": -0.7494140863418579,
      "logps/rejected": -1.598046898841858,
      "loss": 1.0324,
      "nll_loss": 1.023828148841858,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07490234076976776,
      "rewards/margins": 0.08489990234375,
      "rewards/rejected": -0.15993651747703552,
      "step": 1790
    },
    {
      "epoch": 0.13106637055375542,
      "grad_norm": 1.6414899541327086,
      "learning_rate": 1.8856180831641269e-06,
      "log_odds_chosen": 1.225976586341858,
      "log_odds_ratio": -0.4285644590854645,
      "logits/chosen": -0.9429687261581421,
      "logits/rejected": -0.863476574420929,
      "logps/chosen": -0.70166015625,
      "logps/rejected": -1.544531226158142,
      "loss": 1.0124,
      "nll_loss": 0.995898425579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07015381008386612,
      "rewards/margins": 0.08428344875574112,
      "rewards/rejected": -0.1544189453125,
      "step": 1800
    },
    {
      "epoch": 0.13179451705683184,
      "grad_norm": 2.1200877644282374,
      "learning_rate": 1.8804019788890737e-06,
      "log_odds_chosen": 1.094824194908142,
      "log_odds_ratio": -0.48066407442092896,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.8623046875,
      "logps/chosen": -0.735058605670929,
      "logps/rejected": -1.474218726158142,
      "loss": 1.0347,
      "nll_loss": 0.993945300579071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07359619438648224,
      "rewards/margins": 0.07393188774585724,
      "rewards/rejected": -0.14743652939796448,
      "step": 1810
    },
    {
      "epoch": 0.13252266355990824,
      "grad_norm": 1.5932154131897984,
      "learning_rate": 1.8752289237539816e-06,
      "log_odds_chosen": 1.0927734375,
      "log_odds_ratio": -0.4952148497104645,
      "logits/chosen": -0.951953113079071,
      "logits/rejected": -0.9048827886581421,
      "logps/chosen": -0.7237304449081421,
      "logps/rejected": -1.471093773841858,
      "loss": 1.0375,
      "nll_loss": 1.0300781726837158,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07235107570886612,
      "rewards/margins": 0.07469825446605682,
      "rewards/rejected": -0.14699706435203552,
      "step": 1820
    },
    {
      "epoch": 0.13325081006298467,
      "grad_norm": 1.6108589071788282,
      "learning_rate": 1.8700983288487376e-06,
      "log_odds_chosen": 0.8958984613418579,
      "log_odds_ratio": -0.540234386920929,
      "logits/chosen": -0.9791015386581421,
      "logits/rejected": -0.9212890863418579,
      "logps/chosen": -0.7074218988418579,
      "logps/rejected": -1.3308594226837158,
      "loss": 1.0417,
      "nll_loss": 0.991992175579071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07075195014476776,
      "rewards/margins": 0.06231689453125,
      "rewards/rejected": -0.13303223252296448,
      "step": 1830
    },
    {
      "epoch": 0.1339789565660611,
      "grad_norm": 1.7213276876637547,
      "learning_rate": 1.8650096164806275e-06,
      "log_odds_chosen": 1.28369140625,
      "log_odds_ratio": -0.42241209745407104,
      "logits/chosen": -0.9330078363418579,
      "logits/rejected": -0.8394531011581421,
      "logps/chosen": -0.673144519329071,
      "logps/rejected": -1.552734375,
      "loss": 1.0317,
      "nll_loss": 0.96484375,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06730957329273224,
      "rewards/margins": 0.08797607570886612,
      "rewards/rejected": -0.15532226860523224,
      "step": 1840
    },
    {
      "epoch": 0.13470710306913752,
      "grad_norm": 1.6130667842465651,
      "learning_rate": 1.8599622199011084e-06,
      "log_odds_chosen": 0.9668945074081421,
      "log_odds_ratio": -0.4765625,
      "logits/chosen": -0.931835949420929,
      "logits/rejected": -0.8548828363418579,
      "logps/chosen": -0.694140613079071,
      "logps/rejected": -1.3259766101837158,
      "loss": 1.0477,
      "nll_loss": 0.984375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06939697265625,
      "rewards/margins": 0.06313476711511612,
      "rewards/rejected": -0.1324462890625,
      "step": 1850
    },
    {
      "epoch": 0.13543524957221392,
      "grad_norm": 1.6609879194448833,
      "learning_rate": 1.854955583040673e-06,
      "log_odds_chosen": 1.163476586341858,
      "log_odds_ratio": -0.46025389432907104,
      "logits/chosen": -0.952343761920929,
      "logits/rejected": -0.837695300579071,
      "logps/chosen": -0.6958984136581421,
      "logps/rejected": -1.5177733898162842,
      "loss": 1.0413,
      "nll_loss": 0.9771484136581421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06960449367761612,
      "rewards/margins": 0.08206786960363388,
      "rewards/rejected": -0.1517333984375,
      "step": 1860
    },
    {
      "epoch": 0.13616339607529035,
      "grad_norm": 1.7462902977892767,
      "learning_rate": 1.849989160251521e-06,
      "log_odds_chosen": 1.3381836414337158,
      "log_odds_ratio": -0.429443359375,
      "logits/chosen": -0.941210925579071,
      "logits/rejected": -0.8511718511581421,
      "logps/chosen": -0.693652331829071,
      "logps/rejected": -1.626953125,
      "loss": 1.0398,
      "nll_loss": 0.9765625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06939697265625,
      "rewards/margins": 0.09348144382238388,
      "rewards/rejected": -0.16276855766773224,
      "step": 1870
    },
    {
      "epoch": 0.13689154257836678,
      "grad_norm": 1.8769820126274868,
      "learning_rate": 1.8450624160577701e-06,
      "log_odds_chosen": 1.0939452648162842,
      "log_odds_ratio": -0.47309571504592896,
      "logits/chosen": -0.9486328363418579,
      "logits/rejected": -0.8666015863418579,
      "logps/chosen": -0.7012695074081421,
      "logps/rejected": -1.4523437023162842,
      "loss": 1.0159,
      "nll_loss": 0.974609375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07015381008386612,
      "rewards/margins": 0.075103759765625,
      "rewards/rejected": -0.14521484076976776,
      "step": 1880
    },
    {
      "epoch": 0.13761968908144318,
      "grad_norm": 1.9544043745370518,
      "learning_rate": 1.8401748249129445e-06,
      "log_odds_chosen": 1.1393554210662842,
      "log_odds_ratio": -0.4766601622104645,
      "logits/chosen": -1.0300781726837158,
      "logits/rejected": -0.9410156011581421,
      "logps/chosen": -0.6929687261581421,
      "logps/rejected": -1.5060546398162842,
      "loss": 1.0281,
      "nll_loss": 0.872265636920929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06923828274011612,
      "rewards/margins": 0.081298828125,
      "rewards/rejected": -0.15065917372703552,
      "step": 1890
    },
    {
      "epoch": 0.1383478355845196,
      "grad_norm": 2.4515750208575118,
      "learning_rate": 1.8353258709644938e-06,
      "log_odds_chosen": 1.2607421875,
      "log_odds_ratio": -0.4656738340854645,
      "logits/chosen": -0.977734386920929,
      "logits/rejected": -0.846484363079071,
      "logps/chosen": -0.734570324420929,
      "logps/rejected": -1.689843773841858,
      "loss": 1.0297,
      "nll_loss": 0.941601574420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07338867336511612,
      "rewards/margins": 0.09562377631664276,
      "rewards/rejected": -0.16896972060203552,
      "step": 1900
    },
    {
      "epoch": 0.13907598208759603,
      "grad_norm": 1.6034938189999806,
      "learning_rate": 1.830515047825102e-06,
      "log_odds_chosen": 1.1419188976287842,
      "log_odds_ratio": -0.41582030057907104,
      "logits/chosen": -0.9720703363418579,
      "logits/rejected": -0.904101550579071,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -1.480078101158142,
      "loss": 1.0239,
      "nll_loss": 0.9410156011581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07072754204273224,
      "rewards/margins": 0.07727813720703125,
      "rewards/rejected": -0.14799804985523224,
      "step": 1910
    },
    {
      "epoch": 0.13980412859067246,
      "grad_norm": 2.8715977067876706,
      "learning_rate": 1.8257418583505536e-06,
      "log_odds_chosen": 1.2405273914337158,
      "log_odds_ratio": -0.431640625,
      "logits/chosen": -0.972851574420929,
      "logits/rejected": -0.873242199420929,
      "logps/chosen": -0.658984363079071,
      "logps/rejected": -1.533593773841858,
      "loss": 1.0203,
      "nll_loss": 0.930859386920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06586913764476776,
      "rewards/margins": 0.08740539848804474,
      "rewards/rejected": -0.15327148139476776,
      "step": 1920
    },
    {
      "epoch": 0.14053227509374885,
      "grad_norm": 1.6102585892198653,
      "learning_rate": 1.8210058144239416e-06,
      "log_odds_chosen": 1.2312500476837158,
      "log_odds_ratio": -0.4256347715854645,
      "logits/chosen": -0.97265625,
      "logits/rejected": -0.9007812738418579,
      "logps/chosen": -0.7142578363418579,
      "logps/rejected": -1.5652344226837158,
      "loss": 1.0311,
      "nll_loss": 1.0158202648162842,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07138671725988388,
      "rewards/margins": 0.08516235649585724,
      "rewards/rejected": -0.15656737983226776,
      "step": 1930
    },
    {
      "epoch": 0.14126042159682528,
      "grad_norm": 2.4298585873916694,
      "learning_rate": 1.816306436745999e-06,
      "log_odds_chosen": 1.2435791492462158,
      "log_odds_ratio": -0.43940430879592896,
      "logits/chosen": -1.019921898841858,
      "logits/rejected": -0.9267578125,
      "logps/chosen": -0.675000011920929,
      "logps/rejected": -1.5244140625,
      "loss": 1.0506,
      "nll_loss": 0.9794921875,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06746826320886612,
      "rewards/margins": 0.08503799140453339,
      "rewards/rejected": -0.15249022841453552,
      "step": 1940
    },
    {
      "epoch": 0.1419885680999017,
      "grad_norm": 1.9332931824159094,
      "learning_rate": 1.8116432546313529e-06,
      "log_odds_chosen": 1.0442383289337158,
      "log_odds_ratio": -0.5205078125,
      "logits/chosen": -0.969921886920929,
      "logits/rejected": -0.8919922113418579,
      "logps/chosen": -0.704785168170929,
      "logps/rejected": -1.465234398841858,
      "loss": 1.0173,
      "nll_loss": 0.973437488079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07054443657398224,
      "rewards/margins": 0.07599182426929474,
      "rewards/rejected": -0.14654541015625,
      "step": 1950
    },
    {
      "epoch": 0.1427167146029781,
      "grad_norm": 1.6274816620966024,
      "learning_rate": 1.8070158058105026e-06,
      "log_odds_chosen": 1.350683569908142,
      "log_odds_ratio": -0.4427246153354645,
      "logits/chosen": -1.038671851158142,
      "logits/rejected": -0.9292968511581421,
      "logps/chosen": -0.7430664300918579,
      "logps/rejected": -1.698828101158142,
      "loss": 1.0312,
      "nll_loss": 1.0558593273162842,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07437743991613388,
      "rewards/margins": 0.09544067084789276,
      "rewards/rejected": -0.16987304389476776,
      "step": 1960
    },
    {
      "epoch": 0.14344486110605453,
      "grad_norm": 1.7299797054804598,
      "learning_rate": 1.8024236362373315e-06,
      "log_odds_chosen": 1.3367187976837158,
      "log_odds_ratio": -0.441650390625,
      "logits/chosen": -1.071874976158142,
      "logits/rejected": -0.971484363079071,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -1.6267578601837158,
      "loss": 0.9921,
      "nll_loss": 0.890429675579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07069091498851776,
      "rewards/margins": 0.09212646633386612,
      "rewards/rejected": -0.16262206435203552,
      "step": 1970
    },
    {
      "epoch": 0.14417300760913096,
      "grad_norm": 1.69885124566666,
      "learning_rate": 1.7978662999019787e-06,
      "log_odds_chosen": 1.156640648841858,
      "log_odds_ratio": -0.42802733182907104,
      "logits/chosen": -0.9697265625,
      "logits/rejected": -0.8677734136581421,
      "logps/chosen": -0.68310546875,
      "logps/rejected": -1.4865233898162842,
      "loss": 1.0165,
      "nll_loss": 0.932421863079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06826172024011612,
      "rewards/margins": 0.08034668117761612,
      "rewards/rejected": -0.14858397841453552,
      "step": 1980
    },
    {
      "epoch": 0.1449011541122074,
      "grad_norm": 1.800291145787404,
      "learning_rate": 1.793343358648881e-06,
      "log_odds_chosen": 1.2516601085662842,
      "log_odds_ratio": -0.4446777403354645,
      "logits/chosen": -0.9820312261581421,
      "logits/rejected": -0.87109375,
      "logps/chosen": -0.716601550579071,
      "logps/rejected": -1.611328125,
      "loss": 1.0027,
      "nll_loss": 1.0251953601837158,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07163085788488388,
      "rewards/margins": 0.089508056640625,
      "rewards/rejected": -0.16120605170726776,
      "step": 1990
    },
    {
      "epoch": 0.14562930061528379,
      "grad_norm": 1.550129809567034,
      "learning_rate": 1.7888543819998317e-06,
      "log_odds_chosen": 1.2411620616912842,
      "log_odds_ratio": -0.45341795682907104,
      "logits/chosen": -1.0099608898162842,
      "logits/rejected": -0.8951171636581421,
      "logps/chosen": -0.6849609613418579,
      "logps/rejected": -1.540429711341858,
      "loss": 1.0294,
      "nll_loss": 0.9212890863418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06846924126148224,
      "rewards/margins": 0.08555908501148224,
      "rewards/rejected": -0.15400390326976776,
      "step": 2000
    },
    {
      "epoch": 0.1463574471183602,
      "grad_norm": 1.6381828358597794,
      "learning_rate": 1.7843989469818819e-06,
      "log_odds_chosen": 1.3162109851837158,
      "log_odds_ratio": -0.3995117247104645,
      "logits/chosen": -0.950390636920929,
      "logits/rejected": -0.845898449420929,
      "logps/chosen": -0.6778320074081421,
      "logps/rejected": -1.581640601158142,
      "loss": 1.0136,
      "nll_loss": 0.9765625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06779785454273224,
      "rewards/margins": 0.09033203125,
      "rewards/rejected": -0.15805664658546448,
      "step": 2010
    },
    {
      "epoch": 0.14708559362143664,
      "grad_norm": 1.746531136512054,
      "learning_rate": 1.779976637959939e-06,
      "log_odds_chosen": 1.3447265625,
      "log_odds_ratio": -0.384033203125,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.675585925579071,
      "logps/rejected": -1.6085937023162842,
      "loss": 1.0072,
      "nll_loss": 0.931640625,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06757812201976776,
      "rewards/margins": 0.09316406399011612,
      "rewards/rejected": -0.16081543266773224,
      "step": 2020
    },
    {
      "epoch": 0.14781374012451307,
      "grad_norm": 1.8047805973762499,
      "learning_rate": 1.7755870464739012e-06,
      "log_odds_chosen": 0.8568359613418579,
      "log_odds_ratio": -0.551464855670929,
      "logits/chosen": -0.98046875,
      "logits/rejected": -0.9039062261581421,
      "logps/chosen": -0.7124999761581421,
      "logps/rejected": -1.273828148841858,
      "loss": 1.0163,
      "nll_loss": 1.0380859375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07120361179113388,
      "rewards/margins": 0.05621948093175888,
      "rewards/rejected": -0.12734374403953552,
      "step": 2030
    },
    {
      "epoch": 0.14854188662758946,
      "grad_norm": 1.749005326222433,
      "learning_rate": 1.7712297710801907e-06,
      "log_odds_chosen": 1.0390746593475342,
      "log_odds_ratio": -0.4970703125,
      "logits/chosen": -1.0095703601837158,
      "logits/rejected": -0.9261718988418579,
      "logps/chosen": -0.7406250238418579,
      "logps/rejected": -1.4939453601837158,
      "loss": 1.0249,
      "nll_loss": 1.000585913658142,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07404784858226776,
      "rewards/margins": 0.075225830078125,
      "rewards/rejected": -0.14921875298023224,
      "step": 2040
    },
    {
      "epoch": 0.1492700331306659,
      "grad_norm": 1.7189210971520612,
      "learning_rate": 1.7669044171975444e-06,
      "log_odds_chosen": 1.1669921875,
      "log_odds_ratio": -0.459716796875,
      "logits/chosen": -1.033593773841858,
      "logits/rejected": -0.9287109375,
      "logps/chosen": -0.713085949420929,
      "logps/rejected": -1.533593773841858,
      "loss": 1.0038,
      "nll_loss": 0.9828125238418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0712890625,
      "rewards/margins": 0.08214111626148224,
      "rewards/rejected": -0.15346679091453552,
      "step": 2050
    },
    {
      "epoch": 0.14999817963374232,
      "grad_norm": 1.71758005597113,
      "learning_rate": 1.7626105969569268e-06,
      "log_odds_chosen": 1.257568359375,
      "log_odds_ratio": -0.41083985567092896,
      "logits/chosen": -1.020898461341858,
      "logits/rejected": -0.8990234136581421,
      "logps/chosen": -0.661328136920929,
      "logps/rejected": -1.5107421875,
      "loss": 1.0069,
      "nll_loss": 0.979296863079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06613769382238388,
      "rewards/margins": 0.08494873344898224,
      "rewards/rejected": -0.1510009765625,
      "step": 2060
    },
    {
      "epoch": 0.15072632613681872,
      "grad_norm": 1.7130965267383629,
      "learning_rate": 1.758347929055432e-06,
      "log_odds_chosen": 1.084570288658142,
      "log_odds_ratio": -0.48554688692092896,
      "logits/chosen": -1.0646483898162842,
      "logits/rejected": -0.9613281488418579,
      "logps/chosen": -0.718945324420929,
      "logps/rejected": -1.4775390625,
      "loss": 1.0217,
      "nll_loss": 0.9417968988418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07188721001148224,
      "rewards/margins": 0.07598266750574112,
      "rewards/rejected": -0.14785155653953552,
      "step": 2070
    },
    {
      "epoch": 0.15145447263989514,
      "grad_norm": 1.8344799461563874,
      "learning_rate": 1.7541160386140582e-06,
      "log_odds_chosen": 1.0607421398162842,
      "log_odds_ratio": -0.4691406190395355,
      "logits/chosen": -1.025390625,
      "logits/rejected": -0.9564453363418579,
      "logps/chosen": -0.7255859375,
      "logps/rejected": -1.472265601158142,
      "loss": 1.0015,
      "nll_loss": 0.9560546875,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07259521633386612,
      "rewards/margins": 0.07469482719898224,
      "rewards/rejected": -0.14731445908546448,
      "step": 2080
    },
    {
      "epoch": 0.15218261914297157,
      "grad_norm": 1.539438641009839,
      "learning_rate": 1.7499145570392284e-06,
      "log_odds_chosen": 1.2386963367462158,
      "log_odds_ratio": -0.44999998807907104,
      "logits/chosen": -1.041406273841858,
      "logits/rejected": -0.9437500238418579,
      "logps/chosen": -0.722460925579071,
      "logps/rejected": -1.609765648841858,
      "loss": 1.0223,
      "nll_loss": 0.9937499761581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.072265625,
      "rewards/margins": 0.08864593505859375,
      "rewards/rejected": -0.16083984076976776,
      "step": 2090
    },
    {
      "epoch": 0.152910765646048,
      "grad_norm": 2.0451787667388235,
      "learning_rate": 1.745743121887939e-06,
      "log_odds_chosen": 1.3621094226837158,
      "log_odds_ratio": -0.43037110567092896,
      "logits/chosen": -1.0498046875,
      "logits/rejected": -0.9457031488418579,
      "logps/chosen": -0.6919921636581421,
      "logps/rejected": -1.65625,
      "loss": 1.0299,
      "nll_loss": 0.997265636920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06912841647863388,
      "rewards/margins": 0.09648437798023224,
      "rewards/rejected": -0.16562500596046448,
      "step": 2100
    },
    {
      "epoch": 0.1536389121491244,
      "grad_norm": 1.5874290564085507,
      "learning_rate": 1.7416013767364324e-06,
      "log_odds_chosen": 1.393457055091858,
      "log_odds_ratio": -0.41093748807907104,
      "logits/chosen": -0.992382824420929,
      "logits/rejected": -0.921093761920929,
      "logps/chosen": -0.6705077886581421,
      "logps/rejected": -1.6492187976837158,
      "loss": 1.0175,
      "nll_loss": 0.943164050579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06704101711511612,
      "rewards/margins": 0.097869873046875,
      "rewards/rejected": -0.16496582329273224,
      "step": 2110
    },
    {
      "epoch": 0.15436705865220082,
      "grad_norm": 1.6631761854334697,
      "learning_rate": 1.7374889710522776e-06,
      "log_odds_chosen": 0.9184204339981079,
      "log_odds_ratio": -0.533984363079071,
      "logits/chosen": -0.9873046875,
      "logits/rejected": -0.900585949420929,
      "logps/chosen": -0.724804699420929,
      "logps/rejected": -1.4033203125,
      "loss": 0.9923,
      "nll_loss": 1.0046875476837158,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.072509765625,
      "rewards/margins": 0.06780395656824112,
      "rewards/rejected": -0.140380859375,
      "step": 2120
    },
    {
      "epoch": 0.15509520515527725,
      "grad_norm": 1.6789811101008383,
      "learning_rate": 1.7334055600697579e-06,
      "log_odds_chosen": 1.1096680164337158,
      "log_odds_ratio": -0.47880858182907104,
      "logits/chosen": -0.9761718511581421,
      "logits/rejected": -0.9087890386581421,
      "logps/chosen": -0.715039074420929,
      "logps/rejected": -1.5193359851837158,
      "loss": 0.9945,
      "nll_loss": 0.962695300579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07147216796875,
      "rewards/margins": 0.08041992038488388,
      "rewards/rejected": -0.15190429985523224,
      "step": 2130
    },
    {
      "epoch": 0.15582335165835365,
      "grad_norm": 1.6094773818234376,
      "learning_rate": 1.7293508046684678e-06,
      "log_odds_chosen": 1.352929711341858,
      "log_odds_ratio": -0.443115234375,
      "logits/chosen": -1.0148437023162842,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.72265625,
      "logps/rejected": -1.721289038658142,
      "loss": 1.0064,
      "nll_loss": 0.9701172113418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07231445610523224,
      "rewards/margins": 0.09984131157398224,
      "rewards/rejected": -0.17219237983226776,
      "step": 2140
    },
    {
      "epoch": 0.15655149816143007,
      "grad_norm": 1.5949788466112023,
      "learning_rate": 1.7253243712550145e-06,
      "log_odds_chosen": 1.190820336341858,
      "log_odds_ratio": -0.44501954317092896,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.929492175579071,
      "logps/chosen": -0.670703113079071,
      "logps/rejected": -1.494531273841858,
      "loss": 0.9975,
      "nll_loss": 0.910937488079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06711425632238388,
      "rewards/margins": 0.08246765285730362,
      "rewards/rejected": -0.1495361328125,
      "step": 2150
    },
    {
      "epoch": 0.1572796446645065,
      "grad_norm": 1.6678535049326102,
      "learning_rate": 1.7213259316477406e-06,
      "log_odds_chosen": 1.051123023033142,
      "log_odds_ratio": -0.45195311307907104,
      "logits/chosen": -1.0007812976837158,
      "logits/rejected": -0.9521484375,
      "logps/chosen": -0.668652355670929,
      "logps/rejected": -1.368554711341858,
      "loss": 0.9923,
      "nll_loss": 0.932812511920929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06688232719898224,
      "rewards/margins": 0.069915771484375,
      "rewards/rejected": -0.13691405951976776,
      "step": 2160
    },
    {
      "epoch": 0.15800779116758293,
      "grad_norm": 1.9545158765252604,
      "learning_rate": 1.7173551629643674e-06,
      "log_odds_chosen": 1.1601073741912842,
      "log_odds_ratio": -0.44755858182907104,
      "logits/chosen": -1.0255858898162842,
      "logits/rejected": -0.9501953125,
      "logps/chosen": -0.719531238079071,
      "logps/rejected": -1.5519530773162842,
      "loss": 1.0346,
      "nll_loss": 1.0048828125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07194824516773224,
      "rewards/margins": 0.0833740234375,
      "rewards/rejected": -0.15529784560203552,
      "step": 2170
    },
    {
      "epoch": 0.15873593767065933,
      "grad_norm": 1.979180439485981,
      "learning_rate": 1.713411747512477e-06,
      "log_odds_chosen": 1.4584228992462158,
      "log_odds_ratio": -0.40791016817092896,
      "logits/chosen": -0.9716796875,
      "logits/rejected": -0.8960937261581421,
      "logps/chosen": -0.7134765386581421,
      "logps/rejected": -1.748437523841858,
      "loss": 0.9882,
      "nll_loss": 0.94482421875,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0714111328125,
      "rewards/margins": 0.10352478176355362,
      "rewards/rejected": -0.17496338486671448,
      "step": 2180
    },
    {
      "epoch": 0.15946408417373575,
      "grad_norm": 1.5273491940559047,
      "learning_rate": 1.709495372682753e-06,
      "log_odds_chosen": 1.4318358898162842,
      "log_odds_ratio": -0.38471680879592896,
      "logits/chosen": -1.0595703125,
      "logits/rejected": -0.946093738079071,
      "logps/chosen": -0.740429699420929,
      "logps/rejected": -1.803125023841858,
      "loss": 1.005,
      "nll_loss": 0.9310547113418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07410888373851776,
      "rewards/margins": 0.10616455227136612,
      "rewards/rejected": -0.18010254204273224,
      "step": 2190
    },
    {
      "epoch": 0.16019223067681218,
      "grad_norm": 1.711683710044129,
      "learning_rate": 1.7056057308448832e-06,
      "log_odds_chosen": 1.303808569908142,
      "log_odds_ratio": -0.42353516817092896,
      "logits/chosen": -0.9878906011581421,
      "logits/rejected": -0.911914050579071,
      "logps/chosen": -0.651171863079071,
      "logps/rejected": -1.580078125,
      "loss": 0.9733,
      "nll_loss": 0.8873046636581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.065185546875,
      "rewards/margins": 0.09279175102710724,
      "rewards/rejected": -0.15798339247703552,
      "step": 2200
    },
    {
      "epoch": 0.1609203771798886,
      "grad_norm": 1.9284877838544199,
      "learning_rate": 1.701742519246068e-06,
      "log_odds_chosen": 1.0916016101837158,
      "log_odds_ratio": -0.469482421875,
      "logits/chosen": -1.009179711341858,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.714648425579071,
      "logps/rejected": -1.480078101158142,
      "loss": 0.9963,
      "nll_loss": 0.9859374761581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07148437201976776,
      "rewards/margins": 0.07654418796300888,
      "rewards/rejected": -0.14804688096046448,
      "step": 2210
    },
    {
      "epoch": 0.161648523682965,
      "grad_norm": 1.6677132302523026,
      "learning_rate": 1.6979054399120355e-06,
      "log_odds_chosen": 1.203222632408142,
      "log_odds_ratio": -0.412353515625,
      "logits/chosen": -0.998242199420929,
      "logits/rejected": -0.8988281488418579,
      "logps/chosen": -0.6796875,
      "logps/rejected": -1.4855468273162842,
      "loss": 1.0306,
      "nll_loss": 0.9644531011581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.0679931640625,
      "rewards/margins": 0.08062133938074112,
      "rewards/rejected": -0.14865723252296448,
      "step": 2220
    },
    {
      "epoch": 0.16237667018604143,
      "grad_norm": 1.5347502390545833,
      "learning_rate": 1.6940941995505069e-06,
      "log_odds_chosen": 1.1356933116912842,
      "log_odds_ratio": -0.4703125059604645,
      "logits/chosen": -0.9642578363418579,
      "logits/rejected": -0.8759765625,
      "logps/chosen": -0.7196289300918579,
      "logps/rejected": -1.5158202648162842,
      "loss": 0.9893,
      "nll_loss": 0.92138671875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07192382961511612,
      "rewards/margins": 0.07964172214269638,
      "rewards/rejected": -0.15166015923023224,
      "step": 2230
    },
    {
      "epoch": 0.16310481668911786,
      "grad_norm": 1.6299746914295434,
      "learning_rate": 1.6903085094570331e-06,
      "log_odds_chosen": 1.1393554210662842,
      "log_odds_ratio": -0.4344726502895355,
      "logits/chosen": -0.966796875,
      "logits/rejected": -0.855273425579071,
      "logps/chosen": -0.6673828363418579,
      "logps/rejected": -1.4609375,
      "loss": 0.9922,
      "nll_loss": 0.98828125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06676025688648224,
      "rewards/margins": 0.07943115383386612,
      "rewards/rejected": -0.14603272080421448,
      "step": 2240
    },
    {
      "epoch": 0.16383296319219426,
      "grad_norm": 1.7449231003566283,
      "learning_rate": 1.6865480854231356e-06,
      "log_odds_chosen": 1.25592041015625,
      "log_odds_ratio": -0.4454589784145355,
      "logits/chosen": -0.9683593511581421,
      "logits/rejected": -0.861132800579071,
      "logps/chosen": -0.6722656488418579,
      "logps/rejected": -1.549414038658142,
      "loss": 0.9795,
      "nll_loss": 0.9642578363418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06724853813648224,
      "rewards/margins": 0.08760986477136612,
      "rewards/rejected": -0.15495605766773224,
      "step": 2250
    },
    {
      "epoch": 0.16456110969527069,
      "grad_norm": 1.7002753047029264,
      "learning_rate": 1.682812647646685e-06,
      "log_odds_chosen": 1.313378930091858,
      "log_odds_ratio": -0.4354003965854645,
      "logits/chosen": -0.91796875,
      "logits/rejected": -0.8310546875,
      "logps/chosen": -0.67626953125,
      "logps/rejected": -1.6375000476837158,
      "loss": 1.004,
      "nll_loss": 0.909960925579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.067626953125,
      "rewards/margins": 0.09614257514476776,
      "rewards/rejected": -0.16376952826976776,
      "step": 2260
    },
    {
      "epoch": 0.1652892561983471,
      "grad_norm": 2.0232683933890776,
      "learning_rate": 1.6791019206444541e-06,
      "log_odds_chosen": 1.073339819908142,
      "log_odds_ratio": -0.4795898497104645,
      "logits/chosen": -0.9906250238418579,
      "logits/rejected": -0.891406238079071,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -1.478124976158142,
      "loss": 0.9939,
      "nll_loss": 0.927539050579071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07056884467601776,
      "rewards/margins": 0.07724609225988388,
      "rewards/rejected": -0.14790038764476776,
      "step": 2270
    },
    {
      "epoch": 0.16601740270142354,
      "grad_norm": 1.9798155265204258,
      "learning_rate": 1.675415633166782e-06,
      "log_odds_chosen": 1.2253906726837158,
      "log_odds_ratio": -0.45561522245407104,
      "logits/chosen": -0.9566406011581421,
      "logits/rejected": -0.844531238079071,
      "logps/chosen": -0.6962890625,
      "logps/rejected": -1.544335961341858,
      "loss": 1.0055,
      "nll_loss": 0.9769531488418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06960449367761612,
      "rewards/margins": 0.08475951850414276,
      "rewards/rejected": -0.15424804389476776,
      "step": 2280
    },
    {
      "epoch": 0.16674554920449994,
      "grad_norm": 1.7011885763722043,
      "learning_rate": 1.6717535181142914e-06,
      "log_odds_chosen": 1.075585961341858,
      "log_odds_ratio": -0.46967774629592896,
      "logits/chosen": -0.994335949420929,
      "logits/rejected": -0.884765625,
      "logps/chosen": -0.696093738079071,
      "logps/rejected": -1.4347655773162842,
      "loss": 0.9983,
      "nll_loss": 0.9501953125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06960449367761612,
      "rewards/margins": 0.07398681342601776,
      "rewards/rejected": -0.14365234971046448,
      "step": 2290
    },
    {
      "epoch": 0.16747369570757636,
      "grad_norm": 1.7367949626065802,
      "learning_rate": 1.668115312456598e-06,
      "log_odds_chosen": 1.261816382408142,
      "log_odds_ratio": -0.4483398497104645,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.841503918170929,
      "logps/chosen": -0.728222668170929,
      "logps/rejected": -1.638085961341858,
      "loss": 1.0066,
      "nll_loss": 1.013085961341858,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07285156100988388,
      "rewards/margins": 0.09094543755054474,
      "rewards/rejected": -0.163818359375,
      "step": 2300
    },
    {
      "epoch": 0.1682018422106528,
      "grad_norm": 1.7625586947575886,
      "learning_rate": 1.6645007571529578e-06,
      "log_odds_chosen": 1.155371069908142,
      "log_odds_ratio": -0.4588867127895355,
      "logits/chosen": -0.9574218988418579,
      "logits/rejected": -0.8167968988418579,
      "logps/chosen": -0.685839831829071,
      "logps/rejected": -1.5128905773162842,
      "loss": 0.9724,
      "nll_loss": 0.880664050579071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06859131157398224,
      "rewards/margins": 0.08280029147863388,
      "rewards/rejected": -0.15139159560203552,
      "step": 2310
    },
    {
      "epoch": 0.1689299887137292,
      "grad_norm": 1.8747264036819244,
      "learning_rate": 1.6609095970747992e-06,
      "log_odds_chosen": 1.260351538658142,
      "log_odds_ratio": -0.42768555879592896,
      "logits/chosen": -0.9390624761581421,
      "logits/rejected": -0.8550781011581421,
      "logps/chosen": -0.658984363079071,
      "logps/rejected": -1.5261719226837158,
      "loss": 0.9614,
      "nll_loss": 0.926562488079071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06585693359375,
      "rewards/margins": 0.08670654147863388,
      "rewards/rejected": -0.15256348252296448,
      "step": 2320
    },
    {
      "epoch": 0.16965813521680562,
      "grad_norm": 1.7836679926663312,
      "learning_rate": 1.6573415809300833e-06,
      "log_odds_chosen": 1.0992310047149658,
      "log_odds_ratio": -0.4937500059604645,
      "logits/chosen": -0.962109386920929,
      "logits/rejected": -0.906054675579071,
      "logps/chosen": -0.730761706829071,
      "logps/rejected": -1.507421851158142,
      "loss": 0.9864,
      "nll_loss": 0.9111328125,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07304687798023224,
      "rewards/margins": 0.07770691066980362,
      "rewards/rejected": -0.15083007514476776,
      "step": 2330
    },
    {
      "epoch": 0.17038628171988204,
      "grad_norm": 1.6412883442094568,
      "learning_rate": 1.6537964611894462e-06,
      "log_odds_chosen": 1.260644555091858,
      "log_odds_ratio": -0.4375976622104645,
      "logits/chosen": -0.8960937261581421,
      "logits/rejected": -0.819140613079071,
      "logps/chosen": -0.711132824420929,
      "logps/rejected": -1.640625,
      "loss": 0.997,
      "nll_loss": 0.9283202886581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07108154147863388,
      "rewards/margins": 0.09306640923023224,
      "rewards/rejected": -0.16408690810203552,
      "step": 2340
    },
    {
      "epoch": 0.17111442822295847,
      "grad_norm": 1.6539924885076915,
      "learning_rate": 1.6502739940140692e-06,
      "log_odds_chosen": 1.266992211341858,
      "log_odds_ratio": -0.3993164002895355,
      "logits/chosen": -0.9697265625,
      "logits/rejected": -0.8560546636581421,
      "logps/chosen": -0.697265625,
      "logps/rejected": -1.6082031726837158,
      "loss": 1.0253,
      "nll_loss": 0.9759765863418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06977538764476776,
      "rewards/margins": 0.09099121391773224,
      "rewards/rejected": -0.16081543266773224,
      "step": 2350
    },
    {
      "epoch": 0.17184257472603487,
      "grad_norm": 1.8805234680157779,
      "learning_rate": 1.6467739391852364e-06,
      "log_odds_chosen": 1.14990234375,
      "log_odds_ratio": -0.4625488221645355,
      "logits/chosen": -0.9556640386581421,
      "logits/rejected": -0.851367175579071,
      "logps/chosen": -0.6856445074081421,
      "logps/rejected": -1.494531273841858,
      "loss": 0.9757,
      "nll_loss": 0.9189453125,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06861571967601776,
      "rewards/margins": 0.08083496242761612,
      "rewards/rejected": -0.14934082329273224,
      "step": 2360
    },
    {
      "epoch": 0.1725707212291113,
      "grad_norm": 1.851740048111942,
      "learning_rate": 1.6432960600355221e-06,
      "log_odds_chosen": 1.459619164466858,
      "log_odds_ratio": -0.42827147245407104,
      "logits/chosen": -0.9800781011581421,
      "logits/rejected": -0.907031238079071,
      "logps/chosen": -0.687695324420929,
      "logps/rejected": -1.7345702648162842,
      "loss": 0.9805,
      "nll_loss": 0.9203125238418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06875000149011612,
      "rewards/margins": 0.10480346530675888,
      "rewards/rejected": -0.17348632216453552,
      "step": 2370
    },
    {
      "epoch": 0.17329886773218772,
      "grad_norm": 1.8693265724537342,
      "learning_rate": 1.6398401233815756e-06,
      "log_odds_chosen": 1.18212890625,
      "log_odds_ratio": -0.4681152403354645,
      "logits/chosen": -0.966601550579071,
      "logits/rejected": -0.8603515625,
      "logps/chosen": -0.722460925579071,
      "logps/rejected": -1.5583984851837158,
      "loss": 0.9945,
      "nll_loss": 0.9283202886581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07237549126148224,
      "rewards/margins": 0.08355712890625,
      "rewards/rejected": -0.15590819716453552,
      "step": 2380
    },
    {
      "epoch": 0.17402701423526412,
      "grad_norm": 1.659878915220236,
      "learning_rate": 1.6364058994584524e-06,
      "log_odds_chosen": 1.001953125,
      "log_odds_ratio": -0.4740234315395355,
      "logits/chosen": -0.9496093988418579,
      "logits/rejected": -0.834765613079071,
      "logps/chosen": -0.667187511920929,
      "logps/rejected": -1.339257836341858,
      "loss": 0.9863,
      "nll_loss": 0.8802734613418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0667724609375,
      "rewards/margins": 0.06713561713695526,
      "rewards/rejected": -0.13383789360523224,
      "step": 2390
    },
    {
      "epoch": 0.17475516073834055,
      "grad_norm": 2.316586642853329,
      "learning_rate": 1.6329931618554522e-06,
      "log_odds_chosen": 1.373632788658142,
      "log_odds_ratio": -0.377685546875,
      "logits/chosen": -0.94921875,
      "logits/rejected": -0.844921886920929,
      "logps/chosen": -0.6634765863418579,
      "logps/rejected": -1.600976586341858,
      "loss": 0.9967,
      "nll_loss": 0.9170898199081421,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06630859524011612,
      "rewards/margins": 0.09375,
      "rewards/rejected": -0.16008301079273224,
      "step": 2400
    },
    {
      "epoch": 0.17548330724141697,
      "grad_norm": 2.4590357818443254,
      "learning_rate": 1.6296016874534209e-06,
      "log_odds_chosen": 1.5207030773162842,
      "log_odds_ratio": -0.39570313692092896,
      "logits/chosen": -0.94921875,
      "logits/rejected": -0.8392578363418579,
      "logps/chosen": -0.642285168170929,
      "logps/rejected": -1.730078101158142,
      "loss": 0.9686,
      "nll_loss": 0.908203125,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06423339992761612,
      "rewards/margins": 0.10895995795726776,
      "rewards/rejected": -0.17316894233226776,
      "step": 2410
    },
    {
      "epoch": 0.1762114537444934,
      "grad_norm": 1.9392199805473873,
      "learning_rate": 1.6262312563634835e-06,
      "log_odds_chosen": 1.333740234375,
      "log_odds_ratio": -0.40507811307907104,
      "logits/chosen": -0.9957031011581421,
      "logits/rejected": -0.882031261920929,
      "logps/chosen": -0.631640613079071,
      "logps/rejected": -1.5558593273162842,
      "loss": 0.9756,
      "nll_loss": 0.9183593988418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06317138671875,
      "rewards/margins": 0.09233398735523224,
      "rewards/rejected": -0.15561524033546448,
      "step": 2420
    },
    {
      "epoch": 0.1769396002475698,
      "grad_norm": 1.7401538261420497,
      "learning_rate": 1.6228816518671587e-06,
      "log_odds_chosen": 1.1311156749725342,
      "log_odds_ratio": -0.458740234375,
      "logits/chosen": -0.966601550579071,
      "logits/rejected": -0.879687488079071,
      "logps/chosen": -0.699999988079071,
      "logps/rejected": -1.5105469226837158,
      "loss": 0.9956,
      "nll_loss": 0.96875,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07001952826976776,
      "rewards/margins": 0.08120880275964737,
      "rewards/rejected": -0.15114745497703552,
      "step": 2430
    },
    {
      "epoch": 0.17766774675064623,
      "grad_norm": 1.7387675863635101,
      "learning_rate": 1.619552660357832e-06,
      "log_odds_chosen": 1.4942626953125,
      "log_odds_ratio": -0.3973144590854645,
      "logits/chosen": -1.0046875476837158,
      "logits/rejected": -0.9009765386581421,
      "logps/chosen": -0.67626953125,
      "logps/rejected": -1.777734398841858,
      "loss": 0.9984,
      "nll_loss": 0.9429687261581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06761474907398224,
      "rewards/margins": 0.11007995903491974,
      "rewards/rejected": -0.177734375,
      "step": 2440
    },
    {
      "epoch": 0.17839589325372265,
      "grad_norm": 1.7524662122322663,
      "learning_rate": 1.616244071283537e-06,
      "log_odds_chosen": 1.662500023841858,
      "log_odds_ratio": -0.3309082090854645,
      "logits/chosen": -0.9994140863418579,
      "logits/rejected": -0.8785156011581421,
      "logps/chosen": -0.705859363079071,
      "logps/rejected": -1.9113280773162842,
      "loss": 0.9807,
      "nll_loss": 0.910351574420929,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.07054443657398224,
      "rewards/margins": 0.12045898288488388,
      "rewards/rejected": -0.19106444716453552,
      "step": 2450
    },
    {
      "epoch": 0.17912403975679908,
      "grad_norm": 1.936513413974107,
      "learning_rate": 1.6129556770910235e-06,
      "log_odds_chosen": 1.3793456554412842,
      "log_odds_ratio": -0.40385740995407104,
      "logits/chosen": -0.949999988079071,
      "logits/rejected": -0.8304687738418579,
      "logps/chosen": -0.66650390625,
      "logps/rejected": -1.62890625,
      "loss": 0.9961,
      "nll_loss": 0.9654296636581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06671142578125,
      "rewards/margins": 0.09617309272289276,
      "rewards/rejected": -0.16293945908546448,
      "step": 2460
    },
    {
      "epoch": 0.17985218625987548,
      "grad_norm": 1.8747143255272356,
      "learning_rate": 1.6096872731710673e-06,
      "log_odds_chosen": 1.072167992591858,
      "log_odds_ratio": -0.47880858182907104,
      "logits/chosen": -0.949414074420929,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.700976550579071,
      "logps/rejected": -1.4314453601837158,
      "loss": 0.9992,
      "nll_loss": 0.919140636920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07009277492761612,
      "rewards/margins": 0.07298889011144638,
      "rewards/rejected": -0.14304199814796448,
      "step": 2470
    },
    {
      "epoch": 0.1805803327629519,
      "grad_norm": 1.7652420176242012,
      "learning_rate": 1.6064386578049978e-06,
      "log_odds_chosen": 1.355810523033142,
      "log_odds_ratio": -0.43608397245407104,
      "logits/chosen": -1.005859375,
      "logits/rejected": -0.892773449420929,
      "logps/chosen": -0.751953125,
      "logps/rejected": -1.7556641101837158,
      "loss": 0.9679,
      "nll_loss": 0.9833984375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07515869289636612,
      "rewards/margins": 0.10038451850414276,
      "rewards/rejected": -0.17556151747703552,
      "step": 2480
    },
    {
      "epoch": 0.18130847926602833,
      "grad_norm": 1.6342061802531707,
      "learning_rate": 1.6032096321124046e-06,
      "log_odds_chosen": 1.2822997570037842,
      "log_odds_ratio": -0.4527343809604645,
      "logits/chosen": -0.9771484136581421,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.759570300579071,
      "logps/rejected": -1.660546898841858,
      "loss": 0.9824,
      "nll_loss": 0.9931640625,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.075927734375,
      "rewards/margins": 0.09000472724437714,
      "rewards/rejected": -0.16591796278953552,
      "step": 2490
    },
    {
      "epoch": 0.18203662576910473,
      "grad_norm": 2.2198920472661974,
      "learning_rate": 1.6e-06,
      "log_odds_chosen": 1.123779296875,
      "log_odds_ratio": -0.45781248807907104,
      "logits/chosen": -1.0037109851837158,
      "logits/rejected": -0.903124988079071,
      "logps/chosen": -0.6566406488418579,
      "logps/rejected": -1.4304687976837158,
      "loss": 0.9895,
      "nll_loss": 0.930859386920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06564941257238388,
      "rewards/margins": 0.07747497409582138,
      "rewards/rejected": -0.14309081435203552,
      "step": 2500
    },
    {
      "epoch": 0.18276477227218116,
      "grad_norm": 1.5938048746443403,
      "learning_rate": 1.5968095681115984e-06,
      "log_odds_chosen": 1.220800757408142,
      "log_odds_ratio": -0.4927734434604645,
      "logits/chosen": -0.9662109613418579,
      "logits/rejected": -0.876171886920929,
      "logps/chosen": -0.7642577886581421,
      "logps/rejected": -1.6121094226837158,
      "loss": 0.9914,
      "nll_loss": 0.977734386920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07648925483226776,
      "rewards/margins": 0.08474121242761612,
      "rewards/rejected": -0.16120605170726776,
      "step": 2510
    },
    {
      "epoch": 0.18349291877525759,
      "grad_norm": 1.757444473374284,
      "learning_rate": 1.5936381457791914e-06,
      "log_odds_chosen": 1.196313500404358,
      "log_odds_ratio": -0.4432128965854645,
      "logits/chosen": -1.001953125,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.687792956829071,
      "logps/rejected": -1.500390648841858,
      "loss": 0.981,
      "nll_loss": 0.935351550579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06877441704273224,
      "rewards/margins": 0.08116760104894638,
      "rewards/rejected": -0.14992675185203552,
      "step": 2520
    },
    {
      "epoch": 0.184221065278334,
      "grad_norm": 2.1627537795712213,
      "learning_rate": 1.590485544975088e-06,
      "log_odds_chosen": 1.232421875,
      "log_odds_ratio": -0.4478515684604645,
      "logits/chosen": -1.016210913658142,
      "logits/rejected": -0.9039062261581421,
      "logps/chosen": -0.7220703363418579,
      "logps/rejected": -1.6316406726837158,
      "loss": 0.9771,
      "nll_loss": 0.918164074420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07222900539636612,
      "rewards/margins": 0.09097900241613388,
      "rewards/rejected": -0.1632080078125,
      "step": 2530
    },
    {
      "epoch": 0.1849492117814104,
      "grad_norm": 1.8098305840340616,
      "learning_rate": 1.5873515802650901e-06,
      "log_odds_chosen": 1.4041016101837158,
      "log_odds_ratio": -0.40693360567092896,
      "logits/chosen": -1.017187476158142,
      "logits/rejected": -0.894726574420929,
      "logps/chosen": -0.7105468511581421,
      "logps/rejected": -1.710546851158142,
      "loss": 0.9553,
      "nll_loss": 0.916015625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07106933742761612,
      "rewards/margins": 0.10007324069738388,
      "rewards/rejected": -0.17119140923023224,
      "step": 2540
    },
    {
      "epoch": 0.18567735828448684,
      "grad_norm": 1.6365423388715024,
      "learning_rate": 1.584236068762679e-06,
      "log_odds_chosen": 1.2229125499725342,
      "log_odds_ratio": -0.44731444120407104,
      "logits/chosen": -1.0154297351837158,
      "logits/rejected": -0.91796875,
      "logps/chosen": -0.6957031488418579,
      "logps/rejected": -1.574804663658142,
      "loss": 0.9939,
      "nll_loss": 0.923632800579071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.069580078125,
      "rewards/margins": 0.08797454833984375,
      "rewards/rejected": -0.15749511122703552,
      "step": 2550
    },
    {
      "epoch": 0.18640550478756326,
      "grad_norm": 1.844757640743835,
      "learning_rate": 1.5811388300841894e-06,
      "log_odds_chosen": 1.3916015625,
      "log_odds_ratio": -0.4144287109375,
      "logits/chosen": -0.9664062261581421,
      "logits/rejected": -0.900585949420929,
      "logps/chosen": -0.641406238079071,
      "logps/rejected": -1.586523413658142,
      "loss": 0.9988,
      "nll_loss": 0.874218761920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06416015326976776,
      "rewards/margins": 0.0944976806640625,
      "rewards/rejected": -0.15854492783546448,
      "step": 2560
    },
    {
      "epoch": 0.18713365129063966,
      "grad_norm": 1.7225735430035751,
      "learning_rate": 1.5780596863049431e-06,
      "log_odds_chosen": 1.188574194908142,
      "log_odds_ratio": -0.44023436307907104,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.893750011920929,
      "logps/chosen": -0.6748046875,
      "logps/rejected": -1.447851538658142,
      "loss": 0.989,
      "nll_loss": 1.0080077648162842,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06746826320886612,
      "rewards/margins": 0.07734374701976776,
      "rewards/rejected": -0.14472655951976776,
      "step": 2570
    },
    {
      "epoch": 0.1878617977937161,
      "grad_norm": 3.8651739541641152,
      "learning_rate": 1.5749984619163156e-06,
      "log_odds_chosen": 1.2480957508087158,
      "log_odds_ratio": -0.4471191465854645,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.9212890863418579,
      "logps/chosen": -0.6947265863418579,
      "logps/rejected": -1.547265648841858,
      "loss": 0.9871,
      "nll_loss": 0.920703113079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06944580376148224,
      "rewards/margins": 0.08529052883386612,
      "rewards/rejected": -0.15476074814796448,
      "step": 2580
    },
    {
      "epoch": 0.18858994429679252,
      "grad_norm": 2.016288610592031,
      "learning_rate": 1.5719549837837187e-06,
      "log_odds_chosen": 1.294824242591858,
      "log_odds_ratio": -0.45556640625,
      "logits/chosen": -0.9857422113418579,
      "logits/rejected": -0.8753906488418579,
      "logps/chosen": -0.773144543170929,
      "logps/rejected": -1.7058594226837158,
      "loss": 0.9863,
      "nll_loss": 0.968945324420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.077392578125,
      "rewards/margins": 0.09327087551355362,
      "rewards/rejected": -0.1705322265625,
      "step": 2590
    },
    {
      "epoch": 0.18931809079986894,
      "grad_norm": 1.9160316798924124,
      "learning_rate": 1.5689290811054722e-06,
      "log_odds_chosen": 1.395898461341858,
      "log_odds_ratio": -0.37739259004592896,
      "logits/chosen": -0.9791015386581421,
      "logits/rejected": -0.8958984613418579,
      "logps/chosen": -0.6878906488418579,
      "logps/rejected": -1.644921898841858,
      "loss": 1.0017,
      "nll_loss": 1.013671875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06876220554113388,
      "rewards/margins": 0.09564208984375,
      "rewards/rejected": -0.16445311903953552,
      "step": 2600
    },
    {
      "epoch": 0.19004623730294534,
      "grad_norm": 3.096157852600544,
      "learning_rate": 1.5659205853725426e-06,
      "log_odds_chosen": 1.1210448741912842,
      "log_odds_ratio": -0.4522460997104645,
      "logits/chosen": -0.959179699420929,
      "logits/rejected": -0.8734375238418579,
      "logps/chosen": -0.6944335699081421,
      "logps/rejected": -1.493749976158142,
      "loss": 0.9654,
      "nll_loss": 0.923828125,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06953124701976776,
      "rewards/margins": 0.07993622124195099,
      "rewards/rejected": -0.14951172471046448,
      "step": 2610
    },
    {
      "epoch": 0.19077438380602177,
      "grad_norm": 1.7301517280448162,
      "learning_rate": 1.562929330329127e-06,
      "log_odds_chosen": 0.9090331792831421,
      "log_odds_ratio": -0.5238281488418579,
      "logits/chosen": -0.991992175579071,
      "logits/rejected": -0.908203125,
      "logps/chosen": -0.702929675579071,
      "logps/rejected": -1.3173828125,
      "loss": 0.9711,
      "nll_loss": 0.892578125,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07032470405101776,
      "rewards/margins": 0.06130218505859375,
      "rewards/rejected": -0.1317138671875,
      "step": 2620
    },
    {
      "epoch": 0.1915025303090982,
      "grad_norm": 2.21634226483171,
      "learning_rate": 1.5599551519340636e-06,
      "log_odds_chosen": 1.207910180091858,
      "log_odds_ratio": -0.4461914002895355,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.886914074420929,
      "logps/chosen": -0.678417980670929,
      "logps/rejected": -1.5068359375,
      "loss": 0.9737,
      "nll_loss": 0.9068359136581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06778564304113388,
      "rewards/margins": 0.08300171047449112,
      "rewards/rejected": -0.15070800483226776,
      "step": 2630
    },
    {
      "epoch": 0.19223067681217462,
      "grad_norm": 1.683991813663088,
      "learning_rate": 1.556997888323046e-06,
      "log_odds_chosen": 1.104821801185608,
      "log_odds_ratio": -0.4869140684604645,
      "logits/chosen": -0.954296886920929,
      "logits/rejected": -0.8648437261581421,
      "logps/chosen": -0.704296886920929,
      "logps/rejected": -1.497656226158142,
      "loss": 0.9859,
      "nll_loss": 0.966992199420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07041015475988388,
      "rewards/margins": 0.07934989780187607,
      "rewards/rejected": -0.14975586533546448,
      "step": 2640
    },
    {
      "epoch": 0.19295882331525102,
      "grad_norm": 1.8103352328012394,
      "learning_rate": 1.5540573797716226e-06,
      "log_odds_chosen": 1.421484351158142,
      "log_odds_ratio": -0.380615234375,
      "logits/chosen": -1.028906226158142,
      "logits/rejected": -0.9212890863418579,
      "logps/chosen": -0.6832031011581421,
      "logps/rejected": -1.6589844226837158,
      "loss": 0.9781,
      "nll_loss": 0.961132824420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06831054389476776,
      "rewards/margins": 0.09750976413488388,
      "rewards/rejected": -0.16584472358226776,
      "step": 2650
    },
    {
      "epoch": 0.19368696981832745,
      "grad_norm": 1.7308963669828101,
      "learning_rate": 1.5511334686589623e-06,
      "log_odds_chosen": 1.2762451171875,
      "log_odds_ratio": -0.43427735567092896,
      "logits/chosen": -0.994335949420929,
      "logits/rejected": -0.8857421875,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -1.641015648841858,
      "loss": 0.968,
      "nll_loss": 0.9408203363418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07161865383386612,
      "rewards/margins": 0.09243468940258026,
      "rewards/rejected": -0.16396483778953552,
      "step": 2660
    },
    {
      "epoch": 0.19441511632140387,
      "grad_norm": 1.713234417882242,
      "learning_rate": 1.548225999432367e-06,
      "log_odds_chosen": 1.102270483970642,
      "log_odds_ratio": -0.47285157442092896,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.8861328363418579,
      "logps/chosen": -0.6875,
      "logps/rejected": -1.4542968273162842,
      "loss": 0.9799,
      "nll_loss": 0.8736327886581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0687255859375,
      "rewards/margins": 0.0767059326171875,
      "rewards/rejected": -0.14536133408546448,
      "step": 2670
    },
    {
      "epoch": 0.19514326282448027,
      "grad_norm": 2.117196734393062,
      "learning_rate": 1.5453348185725114e-06,
      "log_odds_chosen": 1.164453148841858,
      "log_odds_ratio": -0.49433594942092896,
      "logits/chosen": -0.9976562261581421,
      "logits/rejected": -0.8814452886581421,
      "logps/chosen": -0.739062488079071,
      "logps/rejected": -1.560546875,
      "loss": 1.0065,
      "nll_loss": 1.0398437976837158,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07388915866613388,
      "rewards/margins": 0.08219604194164276,
      "rewards/rejected": -0.15607909858226776,
      "step": 2680
    },
    {
      "epoch": 0.1958714093275567,
      "grad_norm": 1.667294432246079,
      "learning_rate": 1.542459774559398e-06,
      "log_odds_chosen": 1.1334960460662842,
      "log_odds_ratio": -0.4444824159145355,
      "logits/chosen": -1.0007812976837158,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.6917968988418579,
      "logps/rejected": -1.4851562976837158,
      "loss": 0.9852,
      "nll_loss": 0.9066406488418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06918945163488388,
      "rewards/margins": 0.07927856594324112,
      "rewards/rejected": -0.14846190810203552,
      "step": 2690
    },
    {
      "epoch": 0.19659955583063313,
      "grad_norm": 1.884904503906586,
      "learning_rate": 1.539600717839002e-06,
      "log_odds_chosen": 1.4392578601837158,
      "log_odds_ratio": -0.37578123807907104,
      "logits/chosen": -1.0066406726837158,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -0.6773437261581421,
      "logps/rejected": -1.6953125,
      "loss": 0.9791,
      "nll_loss": 0.9134765863418579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06781005859375,
      "rewards/margins": 0.10184326022863388,
      "rewards/rejected": -0.1695556640625,
      "step": 2700
    },
    {
      "epoch": 0.19732770233370955,
      "grad_norm": 2.1456814461914755,
      "learning_rate": 1.536757500790597e-06,
      "log_odds_chosen": 1.4524414539337158,
      "log_odds_ratio": -0.41157227754592896,
      "logits/chosen": -0.9955078363418579,
      "logits/rejected": -0.863085925579071,
      "logps/chosen": -0.6771484613418579,
      "logps/rejected": -1.6945312023162842,
      "loss": 0.974,
      "nll_loss": 0.9195312261581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06773681938648224,
      "rewards/margins": 0.10166015475988388,
      "rewards/rejected": -0.16933593153953552,
      "step": 2710
    },
    {
      "epoch": 0.19805584883678595,
      "grad_norm": 1.9933224194500967,
      "learning_rate": 1.5339299776947407e-06,
      "log_odds_chosen": 1.044580101966858,
      "log_odds_ratio": -0.47314453125,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.8744140863418579,
      "logps/chosen": -0.729296863079071,
      "logps/rejected": -1.4314453601837158,
      "loss": 0.9788,
      "nll_loss": 0.941210925579071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07291259616613388,
      "rewards/margins": 0.070220947265625,
      "rewards/rejected": -0.14313964545726776,
      "step": 2720
    },
    {
      "epoch": 0.19878399533986238,
      "grad_norm": 1.8882083897874273,
      "learning_rate": 1.5311180047019054e-06,
      "log_odds_chosen": 1.301855444908142,
      "log_odds_ratio": -0.4417968690395355,
      "logits/chosen": -0.9585937261581421,
      "logits/rejected": -0.859179675579071,
      "logps/chosen": -0.634765625,
      "logps/rejected": -1.5457031726837158,
      "loss": 0.9828,
      "nll_loss": 0.876757800579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0634765625,
      "rewards/margins": 0.09110717475414276,
      "rewards/rejected": -0.15461425483226776,
      "step": 2730
    },
    {
      "epoch": 0.1995121418429388,
      "grad_norm": 1.9272035550408655,
      "learning_rate": 1.5283214398017402e-06,
      "log_odds_chosen": 1.11376953125,
      "log_odds_ratio": -0.47026365995407104,
      "logits/chosen": -0.9710937738418579,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -0.7298828363418579,
      "logps/rejected": -1.5146484375,
      "loss": 0.9807,
      "nll_loss": 0.8951171636581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07297363132238388,
      "rewards/margins": 0.07837524265050888,
      "rewards/rejected": -0.15146484971046448,
      "step": 2740
    },
    {
      "epoch": 0.2002402883460152,
      "grad_norm": 2.275440509102858,
      "learning_rate": 1.5255401427929477e-06,
      "log_odds_chosen": 1.2584960460662842,
      "log_odds_ratio": -0.455322265625,
      "logits/chosen": -0.9693359136581421,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.692089855670929,
      "logps/rejected": -1.5876953601837158,
      "loss": 0.9652,
      "nll_loss": 0.879101574420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06916503608226776,
      "rewards/margins": 0.08952026069164276,
      "rewards/rejected": -0.15871581435203552,
      "step": 2750
    },
    {
      "epoch": 0.20096843484909163,
      "grad_norm": 1.9131325251618527,
      "learning_rate": 1.5227739752537617e-06,
      "log_odds_chosen": 1.525390625,
      "log_odds_ratio": -0.3623046875,
      "logits/chosen": -1.04296875,
      "logits/rejected": -0.933398425579071,
      "logps/chosen": -0.645312488079071,
      "logps/rejected": -1.726171851158142,
      "loss": 0.9481,
      "nll_loss": 0.871289074420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06452636420726776,
      "rewards/margins": 0.10809326171875,
      "rewards/rejected": -0.17255859076976776,
      "step": 2760
    },
    {
      "epoch": 0.20169658135216806,
      "grad_norm": 1.872622775460701,
      "learning_rate": 1.5200228005130127e-06,
      "log_odds_chosen": 1.153906226158142,
      "log_odds_ratio": -0.4588867127895355,
      "logits/chosen": -0.988085925579071,
      "logits/rejected": -0.9052734375,
      "logps/chosen": -0.699902355670929,
      "logps/rejected": -1.511328101158142,
      "loss": 0.951,
      "nll_loss": 0.9457031488418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06999512016773224,
      "rewards/margins": 0.08110351860523224,
      "rewards/rejected": -0.15104980766773224,
      "step": 2770
    },
    {
      "epoch": 0.20242472785524449,
      "grad_norm": 1.8688286652629447,
      "learning_rate": 1.5172864836217631e-06,
      "log_odds_chosen": 1.135156273841858,
      "log_odds_ratio": -0.45585936307907104,
      "logits/chosen": -1.0173828601837158,
      "logits/rejected": -0.9332031011581421,
      "logps/chosen": -0.6884765625,
      "logps/rejected": -1.46484375,
      "loss": 0.9524,
      "nll_loss": 0.9007812738418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06885986030101776,
      "rewards/margins": 0.07772521674633026,
      "rewards/rejected": -0.1466064453125,
      "step": 2780
    },
    {
      "epoch": 0.20315287435832088,
      "grad_norm": 1.9921633276397985,
      "learning_rate": 1.514564891325506e-06,
      "log_odds_chosen": 1.201269507408142,
      "log_odds_ratio": -0.4375976622104645,
      "logits/chosen": -0.9580078125,
      "logits/rejected": -0.8740234375,
      "logps/chosen": -0.689453125,
      "logps/rejected": -1.4728515148162842,
      "loss": 0.9724,
      "nll_loss": 0.942187488079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06903076171875,
      "rewards/margins": 0.07842407375574112,
      "rewards/rejected": -0.14726562798023224,
      "step": 2790
    },
    {
      "epoch": 0.2038810208613973,
      "grad_norm": 1.7622652662269196,
      "learning_rate": 1.5118578920369086e-06,
      "log_odds_chosen": 1.521240234375,
      "log_odds_ratio": -0.3876953125,
      "logits/chosen": -0.991406261920929,
      "logits/rejected": -0.8910156488418579,
      "logps/chosen": -0.6944335699081421,
      "logps/rejected": -1.7703125476837158,
      "loss": 0.9604,
      "nll_loss": 0.924609363079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06939697265625,
      "rewards/margins": 0.10758056491613388,
      "rewards/rejected": -0.17702636122703552,
      "step": 2800
    },
    {
      "epoch": 0.20460916736447374,
      "grad_norm": 2.4610524568144307,
      "learning_rate": 1.5091653558090898e-06,
      "log_odds_chosen": 1.4079101085662842,
      "log_odds_ratio": -0.44477540254592896,
      "logits/chosen": -0.981249988079071,
      "logits/rejected": -0.8775390386581421,
      "logps/chosen": -0.721875011920929,
      "logps/rejected": -1.7531249523162842,
      "loss": 0.9698,
      "nll_loss": 1.011328101158142,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07215575873851776,
      "rewards/margins": 0.10315094143152237,
      "rewards/rejected": -0.17529296875,
      "step": 2810
    },
    {
      "epoch": 0.20533731386755016,
      "grad_norm": 1.855926984871671,
      "learning_rate": 1.506487154309419e-06,
      "log_odds_chosen": 1.3383667469024658,
      "log_odds_ratio": -0.4185546934604645,
      "logits/chosen": -0.9986327886581421,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.679980456829071,
      "logps/rejected": -1.5880858898162842,
      "loss": 0.9646,
      "nll_loss": 0.912304699420929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06801757961511612,
      "rewards/margins": 0.09089966118335724,
      "rewards/rejected": -0.15871581435203552,
      "step": 2820
    },
    {
      "epoch": 0.20606546037062656,
      "grad_norm": 1.89142819356753,
      "learning_rate": 1.5038231607938247e-06,
      "log_odds_chosen": 1.3450195789337158,
      "log_odds_ratio": -0.4310546815395355,
      "logits/chosen": -0.980664074420929,
      "logits/rejected": -0.892773449420929,
      "logps/chosen": -0.7310546636581421,
      "logps/rejected": -1.6671874523162842,
      "loss": 0.9944,
      "nll_loss": 0.9273437261581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07313232123851776,
      "rewards/margins": 0.09367065131664276,
      "rewards/rejected": -0.16665038466453552,
      "step": 2830
    },
    {
      "epoch": 0.206793606873703,
      "grad_norm": 1.8674812372554181,
      "learning_rate": 1.501173250081603e-06,
      "log_odds_chosen": 1.2644531726837158,
      "log_odds_ratio": -0.43437498807907104,
      "logits/chosen": -0.989453136920929,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.7007812261581421,
      "logps/rejected": -1.5935547351837158,
      "loss": 0.9816,
      "nll_loss": 0.9292968511581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07009277492761612,
      "rewards/margins": 0.08923645317554474,
      "rewards/rejected": -0.159423828125,
      "step": 2840
    },
    {
      "epoch": 0.20752175337677942,
      "grad_norm": 1.7319261331614564,
      "learning_rate": 1.4985372985307103e-06,
      "log_odds_chosen": 1.299462914466858,
      "log_odds_ratio": -0.4154296815395355,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.884570300579071,
      "logps/chosen": -0.680957019329071,
      "logps/rejected": -1.606835961341858,
      "loss": 0.9622,
      "nll_loss": 0.9185546636581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06815185397863388,
      "rewards/margins": 0.09235687553882599,
      "rewards/rejected": -0.16059570014476776,
      "step": 2850
    },
    {
      "epoch": 0.20824989987985582,
      "grad_norm": 1.7066555415301545,
      "learning_rate": 1.4959151840135313e-06,
      "log_odds_chosen": 1.4213135242462158,
      "log_odds_ratio": -0.37651365995407104,
      "logits/chosen": -0.9644531011581421,
      "logits/rejected": -0.8935546875,
      "logps/chosen": -0.60693359375,
      "logps/rejected": -1.5859375,
      "loss": 0.9563,
      "nll_loss": 0.883007824420929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06062011793255806,
      "rewards/margins": 0.09796142578125,
      "rewards/rejected": -0.15866699814796448,
      "step": 2860
    },
    {
      "epoch": 0.20897804638293224,
      "grad_norm": 2.040773021057978,
      "learning_rate": 1.4933067858931148e-06,
      "log_odds_chosen": 1.324804663658142,
      "log_odds_ratio": -0.4117675721645355,
      "logits/chosen": -0.977734386920929,
      "logits/rejected": -0.866015613079071,
      "logps/chosen": -0.6915038824081421,
      "logps/rejected": -1.615234375,
      "loss": 0.9692,
      "nll_loss": 0.916796863079071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06910400092601776,
      "rewards/margins": 0.09234619140625,
      "rewards/rejected": -0.1614990234375,
      "step": 2870
    },
    {
      "epoch": 0.20970619288600867,
      "grad_norm": 1.978854072925717,
      "learning_rate": 1.4907119849998597e-06,
      "log_odds_chosen": 1.1728515625,
      "log_odds_ratio": -0.4541015625,
      "logits/chosen": -0.949023425579071,
      "logits/rejected": -0.861132800579071,
      "logps/chosen": -0.662402331829071,
      "logps/rejected": -1.4402344226837158,
      "loss": 0.9556,
      "nll_loss": 0.924023449420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06625976413488388,
      "rewards/margins": 0.07781829684972763,
      "rewards/rejected": -0.14396972954273224,
      "step": 2880
    },
    {
      "epoch": 0.2104343393890851,
      "grad_norm": 1.7054023285399005,
      "learning_rate": 1.488130663608649e-06,
      "log_odds_chosen": 1.0991332530975342,
      "log_odds_ratio": -0.45112305879592896,
      "logits/chosen": -0.9798828363418579,
      "logits/rejected": -0.8675781488418579,
      "logps/chosen": -0.6244140863418579,
      "logps/rejected": -1.2873046398162842,
      "loss": 0.9543,
      "nll_loss": 0.9546874761581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06248779222369194,
      "rewards/margins": 0.06623993068933487,
      "rewards/rejected": -0.12863770127296448,
      "step": 2890
    },
    {
      "epoch": 0.2111624858921615,
      "grad_norm": 2.107837478420817,
      "learning_rate": 1.4855627054164149e-06,
      "log_odds_chosen": 1.473046898841858,
      "log_odds_ratio": -0.41289061307907104,
      "logits/chosen": -1.0134766101837158,
      "logits/rejected": -0.9185546636581421,
      "logps/chosen": -0.6890624761581421,
      "logps/rejected": -1.744531273841858,
      "loss": 0.9433,
      "nll_loss": 0.8968750238418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06887207180261612,
      "rewards/margins": 0.10548706352710724,
      "rewards/rejected": -0.17436523735523224,
      "step": 2900
    },
    {
      "epoch": 0.21189063239523792,
      "grad_norm": 2.2537114122368997,
      "learning_rate": 1.4830079955201294e-06,
      "log_odds_chosen": 1.234094262123108,
      "log_odds_ratio": -0.4718261659145355,
      "logits/chosen": -0.9853515625,
      "logits/rejected": -0.9085937738418579,
      "logps/chosen": -0.6859375238418579,
      "logps/rejected": -1.574609398841858,
      "loss": 0.9536,
      "nll_loss": 0.8763672113418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06864013522863388,
      "rewards/margins": 0.08886337280273438,
      "rewards/rejected": -0.15739746391773224,
      "step": 2910
    },
    {
      "epoch": 0.21261877889831435,
      "grad_norm": 1.8244686245259676,
      "learning_rate": 1.4804664203952103e-06,
      "log_odds_chosen": 1.08837890625,
      "log_odds_ratio": -0.4693359434604645,
      "logits/chosen": -0.9205077886581421,
      "logits/rejected": -0.8248046636581421,
      "logps/chosen": -0.69140625,
      "logps/rejected": -1.457617163658142,
      "loss": 0.9765,
      "nll_loss": 0.968945324420929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06918945163488388,
      "rewards/margins": 0.076568603515625,
      "rewards/rejected": -0.14577636122703552,
      "step": 2920
    },
    {
      "epoch": 0.21334692540139075,
      "grad_norm": 1.8006919322547585,
      "learning_rate": 1.4779378678743327e-06,
      "log_odds_chosen": 1.6355469226837158,
      "log_odds_ratio": -0.3731445372104645,
      "logits/chosen": -1.017578125,
      "logits/rejected": -0.887499988079071,
      "logps/chosen": -0.68359375,
      "logps/rejected": -1.875,
      "loss": 0.9595,
      "nll_loss": 0.8802734613418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06838379055261612,
      "rewards/margins": 0.11911620944738388,
      "rewards/rejected": -0.18740233778953552,
      "step": 2930
    },
    {
      "epoch": 0.21407507190446717,
      "grad_norm": 1.9131584993525343,
      "learning_rate": 1.4754222271266348e-06,
      "log_odds_chosen": 1.566796898841858,
      "log_odds_ratio": -0.38654786348342896,
      "logits/chosen": -0.9693359136581421,
      "logits/rejected": -0.880664050579071,
      "logps/chosen": -0.662792980670929,
      "logps/rejected": -1.7802734375,
      "loss": 0.9541,
      "nll_loss": 0.9365234375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06633301079273224,
      "rewards/margins": 0.11158446967601776,
      "rewards/rejected": -0.17802734673023224,
      "step": 2940
    },
    {
      "epoch": 0.2148032184075436,
      "grad_norm": 1.9460384896474459,
      "learning_rate": 1.4729193886373175e-06,
      "log_odds_chosen": 1.32861328125,
      "log_odds_ratio": -0.4120117127895355,
      "logits/chosen": -0.9390624761581421,
      "logits/rejected": -0.841601550579071,
      "logps/chosen": -0.681445300579071,
      "logps/rejected": -1.6281249523162842,
      "loss": 0.9538,
      "nll_loss": 0.9683593511581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06821288913488388,
      "rewards/margins": 0.09473876655101776,
      "rewards/rejected": -0.16289062798023224,
      "step": 2950
    },
    {
      "epoch": 0.21553136491062003,
      "grad_norm": 2.142437375052006,
      "learning_rate": 1.4704292441876156e-06,
      "log_odds_chosen": 1.2626953125,
      "log_odds_ratio": -0.42509764432907104,
      "logits/chosen": -0.972851574420929,
      "logits/rejected": -0.890625,
      "logps/chosen": -0.7411133050918579,
      "logps/rejected": -1.642578125,
      "loss": 0.9488,
      "nll_loss": 0.9574218988418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07414551079273224,
      "rewards/margins": 0.09000243991613388,
      "rewards/rejected": -0.16428223252296448,
      "step": 2960
    },
    {
      "epoch": 0.21625951141369643,
      "grad_norm": 2.416863996828751,
      "learning_rate": 1.4679516868351474e-06,
      "log_odds_chosen": 1.200097680091858,
      "log_odds_ratio": -0.46064454317092896,
      "logits/chosen": -1.010156273841858,
      "logits/rejected": -0.935351550579071,
      "logps/chosen": -0.687695324420929,
      "logps/rejected": -1.548828125,
      "loss": 0.9551,
      "nll_loss": 0.9525390863418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0687255859375,
      "rewards/margins": 0.086090087890625,
      "rewards/rejected": -0.15485839545726776,
      "step": 2970
    },
    {
      "epoch": 0.21698765791677285,
      "grad_norm": 1.6671627464746113,
      "learning_rate": 1.4654866108946234e-06,
      "log_odds_chosen": 1.0857422351837158,
      "log_odds_ratio": -0.46528321504592896,
      "logits/chosen": -0.9664062261581421,
      "logits/rejected": -0.9076172113418579,
      "logps/chosen": -0.661816418170929,
      "logps/rejected": -1.388671875,
      "loss": 0.9622,
      "nll_loss": 0.943164050579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06624756008386612,
      "rewards/margins": 0.0725608840584755,
      "rewards/rejected": -0.138916015625,
      "step": 2980
    },
    {
      "epoch": 0.21771580441984928,
      "grad_norm": 2.2636494828883826,
      "learning_rate": 1.4630339119189101e-06,
      "log_odds_chosen": 1.367285132408142,
      "log_odds_ratio": -0.4156738221645355,
      "logits/chosen": -0.9654296636581421,
      "logits/rejected": -0.8583984375,
      "logps/chosen": -0.6966797113418579,
      "logps/rejected": -1.6833984851837158,
      "loss": 0.9592,
      "nll_loss": 0.950390636920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06964111328125,
      "rewards/margins": 0.09891357272863388,
      "rewards/rejected": -0.16855469346046448,
      "step": 2990
    },
    {
      "epoch": 0.2184439509229257,
      "grad_norm": 2.1101380163406804,
      "learning_rate": 1.4605934866804429e-06,
      "log_odds_chosen": 1.328710913658142,
      "log_odds_ratio": -0.4356933534145355,
      "logits/chosen": -0.9716796875,
      "logits/rejected": -0.8720703125,
      "logps/chosen": -0.6787109375,
      "logps/rejected": -1.6240234375,
      "loss": 0.9662,
      "nll_loss": 0.9486328363418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06793212890625,
      "rewards/margins": 0.09464111179113388,
      "rewards/rejected": -0.16259765625,
      "step": 3000
    },
    {
      "epoch": 0.2191720974260021,
      "grad_norm": 2.0487337106120056,
      "learning_rate": 1.4581652331529784e-06,
      "log_odds_chosen": 1.3349609375,
      "log_odds_ratio": -0.4010253846645355,
      "logits/chosen": -0.97265625,
      "logits/rejected": -0.880078136920929,
      "logps/chosen": -0.6625000238418579,
      "logps/rejected": -1.585546851158142,
      "loss": 0.9323,
      "nll_loss": 0.91015625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06630859524011612,
      "rewards/margins": 0.09222412109375,
      "rewards/rejected": -0.15854492783546448,
      "step": 3010
    },
    {
      "epoch": 0.21990024392907853,
      "grad_norm": 1.7821297389824873,
      "learning_rate": 1.4557490504936778e-06,
      "log_odds_chosen": 1.4997069835662842,
      "log_odds_ratio": -0.4007812440395355,
      "logits/chosen": -0.9853515625,
      "logits/rejected": -0.8892577886581421,
      "logps/chosen": -0.677929699420929,
      "logps/rejected": -1.708593726158142,
      "loss": 0.9473,
      "nll_loss": 0.9619140625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06781005859375,
      "rewards/margins": 0.10306243598461151,
      "rewards/rejected": -0.1708984375,
      "step": 3020
    },
    {
      "epoch": 0.22062839043215496,
      "grad_norm": 2.2075025049368913,
      "learning_rate": 1.453344839025519e-06,
      "log_odds_chosen": 1.3333008289337158,
      "log_odds_ratio": -0.43901365995407104,
      "logits/chosen": -1.017968773841858,
      "logits/rejected": -0.8861328363418579,
      "logps/chosen": -0.707714855670929,
      "logps/rejected": -1.649023413658142,
      "loss": 0.9526,
      "nll_loss": 0.927734375,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07072754204273224,
      "rewards/margins": 0.09394989162683487,
      "rewards/rejected": -0.16486816108226776,
      "step": 3030
    },
    {
      "epoch": 0.22135653693523136,
      "grad_norm": 1.8552765813231715,
      "learning_rate": 1.4509525002200234e-06,
      "log_odds_chosen": 1.34716796875,
      "log_odds_ratio": -0.4063476622104645,
      "logits/chosen": -1.0349609851837158,
      "logits/rejected": -0.935351550579071,
      "logps/chosen": -0.7123047113418579,
      "logps/rejected": -1.6941406726837158,
      "loss": 0.9545,
      "nll_loss": 0.9576171636581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07130126655101776,
      "rewards/margins": 0.09808349609375,
      "rewards/rejected": -0.16938476264476776,
      "step": 3040
    },
    {
      "epoch": 0.22208468343830778,
      "grad_norm": 2.0386303284968292,
      "learning_rate": 1.4485719366802965e-06,
      "log_odds_chosen": 1.5283203125,
      "log_odds_ratio": -0.3778320252895355,
      "logits/chosen": -1.051367163658142,
      "logits/rejected": -0.966992199420929,
      "logps/chosen": -0.68408203125,
      "logps/rejected": -1.8097655773162842,
      "loss": 0.9659,
      "nll_loss": 0.9164062738418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06837157905101776,
      "rewards/margins": 0.112548828125,
      "rewards/rejected": -0.18100586533546448,
      "step": 3050
    },
    {
      "epoch": 0.2228128299413842,
      "grad_norm": 1.9756674859249355,
      "learning_rate": 1.4462030521243742e-06,
      "log_odds_chosen": 1.2770507335662842,
      "log_odds_ratio": -0.4183105528354645,
      "logits/chosen": -1.0255858898162842,
      "logits/rejected": -0.9388672113418579,
      "logps/chosen": -0.6094726324081421,
      "logps/rejected": -1.441992163658142,
      "loss": 0.9493,
      "nll_loss": 0.8822265863418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06102294847369194,
      "rewards/margins": 0.08314208686351776,
      "rewards/rejected": -0.14418944716453552,
      "step": 3060
    },
    {
      "epoch": 0.22354097644446064,
      "grad_norm": 2.026828091317089,
      "learning_rate": 1.443845751368867e-06,
      "log_odds_chosen": 1.4765625,
      "log_odds_ratio": -0.4332031309604645,
      "logits/chosen": -0.9624999761581421,
      "logits/rejected": -0.8792968988418579,
      "logps/chosen": -0.6673828363418579,
      "logps/rejected": -1.730078101158142,
      "loss": 0.9682,
      "nll_loss": 0.902148425579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06671142578125,
      "rewards/margins": 0.1063232421875,
      "rewards/rejected": -0.17294922471046448,
      "step": 3070
    },
    {
      "epoch": 0.22426912294753704,
      "grad_norm": 2.0704026248128464,
      "learning_rate": 1.4414999403128943e-06,
      "log_odds_chosen": 1.27685546875,
      "log_odds_ratio": -0.38818359375,
      "logits/chosen": -0.992382824420929,
      "logits/rejected": -0.8880859613418579,
      "logps/chosen": -0.642871081829071,
      "logps/rejected": -1.485937476158142,
      "loss": 0.9409,
      "nll_loss": 0.889453113079071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.064208984375,
      "rewards/margins": 0.08423309028148651,
      "rewards/rejected": -0.14853516221046448,
      "step": 3080
    },
    {
      "epoch": 0.22499726945061346,
      "grad_norm": 1.6596722166646387,
      "learning_rate": 1.439165525922309e-06,
      "log_odds_chosen": 1.521875023841858,
      "log_odds_ratio": -0.3441406190395355,
      "logits/chosen": -0.9955078363418579,
      "logits/rejected": -0.87890625,
      "logps/chosen": -0.63427734375,
      "logps/rejected": -1.6306641101837158,
      "loss": 0.9517,
      "nll_loss": 0.9166015386581421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06345214694738388,
      "rewards/margins": 0.09968261420726776,
      "rewards/rejected": -0.16303710639476776,
      "step": 3090
    },
    {
      "epoch": 0.2257254159536899,
      "grad_norm": 2.4207485804016358,
      "learning_rate": 1.4368424162141992e-06,
      "log_odds_chosen": 1.219140648841858,
      "log_odds_ratio": -0.4796386659145355,
      "logits/chosen": -0.9779297113418579,
      "logits/rejected": -0.8968750238418579,
      "logps/chosen": -0.713671863079071,
      "logps/rejected": -1.608007788658142,
      "loss": 0.9715,
      "nll_loss": 0.972851574420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07130126655101776,
      "rewards/margins": 0.08950195461511612,
      "rewards/rejected": -0.16074219346046448,
      "step": 3100
    },
    {
      "epoch": 0.2264535624567663,
      "grad_norm": 1.8252249236954992,
      "learning_rate": 1.434530520241665e-06,
      "log_odds_chosen": 1.3927733898162842,
      "log_odds_ratio": -0.4014648497104645,
      "logits/chosen": -0.9906250238418579,
      "logits/rejected": -0.877734363079071,
      "logps/chosen": -0.6524413824081421,
      "logps/rejected": -1.5988280773162842,
      "loss": 0.9402,
      "nll_loss": 0.886523425579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06525878608226776,
      "rewards/margins": 0.094482421875,
      "rewards/rejected": -0.1597900390625,
      "step": 3110
    },
    {
      "epoch": 0.22718170895984272,
      "grad_norm": 1.8812273728775581,
      "learning_rate": 1.4322297480788657e-06,
      "log_odds_chosen": 1.1748535633087158,
      "log_odds_ratio": -0.46625977754592896,
      "logits/chosen": -0.985156238079071,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -0.6996093988418579,
      "logps/rejected": -1.526953101158142,
      "loss": 0.9517,
      "nll_loss": 0.945507824420929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06998290866613388,
      "rewards/margins": 0.08274994045495987,
      "rewards/rejected": -0.1527099609375,
      "step": 3120
    },
    {
      "epoch": 0.22790985546291914,
      "grad_norm": 2.7344401519505057,
      "learning_rate": 1.4299400108063247e-06,
      "log_odds_chosen": 1.3515625,
      "log_odds_ratio": -0.44465333223342896,
      "logits/chosen": -0.9927734136581421,
      "logits/rejected": -0.8744140863418579,
      "logps/chosen": -0.732421875,
      "logps/rejected": -1.7041015625,
      "loss": 0.9683,
      "nll_loss": 0.920703113079071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07322998344898224,
      "rewards/margins": 0.09711913764476776,
      "rewards/rejected": -0.17045898735523224,
      "step": 3130
    },
    {
      "epoch": 0.22863800196599557,
      "grad_norm": 1.9995693445540086,
      "learning_rate": 1.4276612204964992e-06,
      "log_odds_chosen": 1.285375952720642,
      "log_odds_ratio": -0.43500977754592896,
      "logits/chosen": -1.0076172351837158,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.6634765863418579,
      "logps/rejected": -1.5261719226837158,
      "loss": 0.9348,
      "nll_loss": 0.853320300579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06633301079273224,
      "rewards/margins": 0.086273193359375,
      "rewards/rejected": -0.152587890625,
      "step": 3140
    },
    {
      "epoch": 0.22936614846907197,
      "grad_norm": 1.9731818610147924,
      "learning_rate": 1.4253932901995967e-06,
      "log_odds_chosen": 1.2380859851837158,
      "log_odds_ratio": -0.4385742247104645,
      "logits/chosen": -1.0076172351837158,
      "logits/rejected": -0.909375011920929,
      "logps/chosen": -0.7046874761581421,
      "logps/rejected": -1.595312476158142,
      "loss": 0.9175,
      "nll_loss": 0.8804687261581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07048340141773224,
      "rewards/margins": 0.08892212063074112,
      "rewards/rejected": -0.15939942002296448,
      "step": 3150
    },
    {
      "epoch": 0.2300942949721484,
      "grad_norm": 1.96315338446132,
      "learning_rate": 1.42313613392964e-06,
      "log_odds_chosen": 1.506250023841858,
      "log_odds_ratio": -0.36601561307907104,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.9140625,
      "logps/chosen": -0.656445324420929,
      "logps/rejected": -1.695898413658142,
      "loss": 0.9576,
      "nll_loss": 0.9486328363418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06558837741613388,
      "rewards/margins": 0.10384521633386612,
      "rewards/rejected": -0.16940918564796448,
      "step": 3160
    },
    {
      "epoch": 0.23082244147522482,
      "grad_norm": 1.9582604422326573,
      "learning_rate": 1.4208896666507756e-06,
      "log_odds_chosen": 1.4503905773162842,
      "log_odds_ratio": -0.40576171875,
      "logits/chosen": -1.0183594226837158,
      "logits/rejected": -0.8890625238418579,
      "logps/chosen": -0.7232421636581421,
      "logps/rejected": -1.7511718273162842,
      "loss": 0.9388,
      "nll_loss": 0.90234375,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07225342094898224,
      "rewards/margins": 0.10303344577550888,
      "rewards/rejected": -0.17524413764476776,
      "step": 3170
    },
    {
      "epoch": 0.23155058797830125,
      "grad_norm": 1.9001038651532063,
      "learning_rate": 1.4186538042638173e-06,
      "log_odds_chosen": 1.2781250476837158,
      "log_odds_ratio": -0.43427735567092896,
      "logits/chosen": -0.954882800579071,
      "logits/rejected": -0.8832031488418579,
      "logps/chosen": -0.687304675579071,
      "logps/rejected": -1.58203125,
      "loss": 0.9531,
      "nll_loss": 0.959179699420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06873778998851776,
      "rewards/margins": 0.089447021484375,
      "rewards/rejected": -0.15822753310203552,
      "step": 3180
    },
    {
      "epoch": 0.23227873448137765,
      "grad_norm": 1.8761442443485807,
      "learning_rate": 1.416428463593022e-06,
      "log_odds_chosen": 1.21142578125,
      "log_odds_ratio": -0.4537597596645355,
      "logits/chosen": -1.042578101158142,
      "logits/rejected": -0.927539050579071,
      "logps/chosen": -0.67724609375,
      "logps/rejected": -1.530664086341858,
      "loss": 0.9525,
      "nll_loss": 0.9273437261581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06779785454273224,
      "rewards/margins": 0.08539581298828125,
      "rewards/rejected": -0.15324707329273224,
      "step": 3190
    },
    {
      "epoch": 0.23300688098445407,
      "grad_norm": 2.102780096954173,
      "learning_rate": 1.414213562373095e-06,
      "log_odds_chosen": 1.2815430164337158,
      "log_odds_ratio": -0.4295410215854645,
      "logits/chosen": -1.0173828601837158,
      "logits/rejected": -0.908984363079071,
      "logps/chosen": -0.67578125,
      "logps/rejected": -1.608984351158142,
      "loss": 0.9245,
      "nll_loss": 0.95703125,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06760253757238388,
      "rewards/margins": 0.09331665188074112,
      "rewards/rejected": -0.1610107421875,
      "step": 3200
    },
    {
      "epoch": 0.2337350274875305,
      "grad_norm": 2.0596656084325704,
      "learning_rate": 1.4120090192364154e-06,
      "log_odds_chosen": 1.218847632408142,
      "log_odds_ratio": -0.43657225370407104,
      "logits/chosen": -0.9697265625,
      "logits/rejected": -0.8998047113418579,
      "logps/chosen": -0.7001953125,
      "logps/rejected": -1.5652344226837158,
      "loss": 0.9467,
      "nll_loss": 0.891796886920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07003173977136612,
      "rewards/margins": 0.08646240085363388,
      "rewards/rejected": -0.15642090141773224,
      "step": 3210
    },
    {
      "epoch": 0.2344631739906069,
      "grad_norm": 1.7776871068480575,
      "learning_rate": 1.4098147537004828e-06,
      "log_odds_chosen": 1.5828125476837158,
      "log_odds_ratio": -0.382080078125,
      "logits/chosen": -1.0400390625,
      "logits/rejected": -0.947460949420929,
      "logps/chosen": -0.667187511920929,
      "logps/rejected": -1.776757836341858,
      "loss": 0.9231,
      "nll_loss": 0.8199218511581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06669922173023224,
      "rewards/margins": 0.11105956882238388,
      "rewards/rejected": -0.17778320610523224,
      "step": 3220
    },
    {
      "epoch": 0.23519132049368333,
      "grad_norm": 2.2488850443210056,
      "learning_rate": 1.4076306861555735e-06,
      "log_odds_chosen": 1.41796875,
      "log_odds_ratio": -0.40283203125,
      "logits/chosen": -0.988085925579071,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.6537109613418579,
      "logps/rejected": -1.617773413658142,
      "loss": 0.9367,
      "nll_loss": 0.900390625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06531982123851776,
      "rewards/margins": 0.09633026272058487,
      "rewards/rejected": -0.16169433295726776,
      "step": 3230
    },
    {
      "epoch": 0.23591946699675975,
      "grad_norm": 1.9473739415490248,
      "learning_rate": 1.405456737852613e-06,
      "log_odds_chosen": 1.30908203125,
      "log_odds_ratio": -0.4225097596645355,
      "logits/chosen": -0.9857422113418579,
      "logits/rejected": -0.8968750238418579,
      "logps/chosen": -0.6573241949081421,
      "logps/rejected": -1.591406226158142,
      "loss": 0.9486,
      "nll_loss": 0.9212890863418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06569824367761612,
      "rewards/margins": 0.09355469048023224,
      "rewards/rejected": -0.15927734971046448,
      "step": 3240
    },
    {
      "epoch": 0.23664761349983618,
      "grad_norm": 1.9792943929256113,
      "learning_rate": 1.4032928308912468e-06,
      "log_odds_chosen": 1.3916015625,
      "log_odds_ratio": -0.4039062559604645,
      "logits/chosen": -0.9466797113418579,
      "logits/rejected": -0.8560546636581421,
      "logps/chosen": -0.658203125,
      "logps/rejected": -1.6072266101837158,
      "loss": 0.9077,
      "nll_loss": 0.819140613079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06582031399011612,
      "rewards/margins": 0.09481201320886612,
      "rewards/rejected": -0.16062012314796448,
      "step": 3250
    },
    {
      "epoch": 0.23737576000291258,
      "grad_norm": 1.9243620305329163,
      "learning_rate": 1.4011388882081175e-06,
      "log_odds_chosen": 1.3654296398162842,
      "log_odds_ratio": -0.43818360567092896,
      "logits/chosen": -0.9710937738418579,
      "logits/rejected": -0.910937488079071,
      "logps/chosen": -0.6949218511581421,
      "logps/rejected": -1.65234375,
      "loss": 0.9609,
      "nll_loss": 0.916210949420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06950683891773224,
      "rewards/margins": 0.09586334228515625,
      "rewards/rejected": -0.16533203423023224,
      "step": 3260
    },
    {
      "epoch": 0.238103906505989,
      "grad_norm": 1.8789820602704654,
      "learning_rate": 1.3989948335653378e-06,
      "log_odds_chosen": 1.082421898841858,
      "log_odds_ratio": -0.45166015625,
      "logits/chosen": -0.9453125,
      "logits/rejected": -0.8671875,
      "logps/chosen": -0.658984363079071,
      "logps/rejected": -1.3624999523162842,
      "loss": 0.9315,
      "nll_loss": 0.915234386920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06593017280101776,
      "rewards/margins": 0.07033081352710724,
      "rewards/rejected": -0.13623046875,
      "step": 3270
    },
    {
      "epoch": 0.23883205300906543,
      "grad_norm": 1.8270238165002866,
      "learning_rate": 1.3968605915391564e-06,
      "log_odds_chosen": 0.9653564691543579,
      "log_odds_ratio": -0.510009765625,
      "logits/chosen": -0.9849609136581421,
      "logits/rejected": -0.854687511920929,
      "logps/chosen": -0.7044922113418579,
      "logps/rejected": -1.341796875,
      "loss": 0.9267,
      "nll_loss": 0.9300781488418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0704345703125,
      "rewards/margins": 0.06384124606847763,
      "rewards/rejected": -0.13430175185203552,
      "step": 3280
    },
    {
      "epoch": 0.23956019951214183,
      "grad_norm": 1.7458016586679614,
      "learning_rate": 1.3947360875088132e-06,
      "log_odds_chosen": 1.3389160633087158,
      "log_odds_ratio": -0.4267822206020355,
      "logits/chosen": -1.005859375,
      "logits/rejected": -0.882031261920929,
      "logps/chosen": -0.655078113079071,
      "logps/rejected": -1.5925781726837158,
      "loss": 0.9279,
      "nll_loss": 0.9091796875,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0655517578125,
      "rewards/margins": 0.09366454929113388,
      "rewards/rejected": -0.15922851860523224,
      "step": 3290
    },
    {
      "epoch": 0.24028834601521826,
      "grad_norm": 1.9535989792097175,
      "learning_rate": 1.3926212476455828e-06,
      "log_odds_chosen": 1.395410180091858,
      "log_odds_ratio": -0.39301759004592896,
      "logits/chosen": -0.988085925579071,
      "logits/rejected": -0.8755859136581421,
      "logps/chosen": -0.716113269329071,
      "logps/rejected": -1.6892578601837158,
      "loss": 0.9456,
      "nll_loss": 0.9443359375,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.07160644233226776,
      "rewards/margins": 0.09724120795726776,
      "rewards/rejected": -0.1689453125,
      "step": 3300
    },
    {
      "epoch": 0.24101649251829468,
      "grad_norm": 2.098551401687419,
      "learning_rate": 1.3905159989019964e-06,
      "log_odds_chosen": 1.3742187023162842,
      "log_odds_ratio": -0.3951171934604645,
      "logits/chosen": -0.958789050579071,
      "logits/rejected": -0.864062488079071,
      "logps/chosen": -0.6548827886581421,
      "logps/rejected": -1.6208984851837158,
      "loss": 0.9566,
      "nll_loss": 0.8724609613418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06551513820886612,
      "rewards/margins": 0.09650878608226776,
      "rewards/rejected": -0.16203613579273224,
      "step": 3310
    },
    {
      "epoch": 0.2417446390213711,
      "grad_norm": 2.2667028369549502,
      "learning_rate": 1.3884202690012465e-06,
      "log_odds_chosen": 1.1271851062774658,
      "log_odds_ratio": -0.49238282442092896,
      "logits/chosen": -0.9697265625,
      "logits/rejected": -0.9140625,
      "logps/chosen": -0.70703125,
      "logps/rejected": -1.492773413658142,
      "loss": 0.9255,
      "nll_loss": 0.883007824420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07071533054113388,
      "rewards/margins": 0.07855224609375,
      "rewards/rejected": -0.14926758408546448,
      "step": 3320
    },
    {
      "epoch": 0.2424727855244475,
      "grad_norm": 1.8684644804444257,
      "learning_rate": 1.3863339864267636e-06,
      "log_odds_chosen": 1.131445288658142,
      "log_odds_ratio": -0.47309571504592896,
      "logits/chosen": -0.979687511920929,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.663281261920929,
      "logps/rejected": -1.4275391101837158,
      "loss": 0.9517,
      "nll_loss": 0.908984363079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06638183444738388,
      "rewards/margins": 0.07631225883960724,
      "rewards/rejected": -0.14262695610523224,
      "step": 3330
    },
    {
      "epoch": 0.24320093202752394,
      "grad_norm": 1.9487258090702118,
      "learning_rate": 1.3842570804119655e-06,
      "log_odds_chosen": 1.220239281654358,
      "log_odds_ratio": -0.4520019590854645,
      "logits/chosen": -1.0353515148162842,
      "logits/rejected": -0.938671886920929,
      "logps/chosen": -0.694531261920929,
      "logps/rejected": -1.5656249523162842,
      "loss": 0.9226,
      "nll_loss": 0.8564453125,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06944580376148224,
      "rewards/margins": 0.08715057373046875,
      "rewards/rejected": -0.1566162109375,
      "step": 3340
    },
    {
      "epoch": 0.24392907853060036,
      "grad_norm": 2.0037519577994223,
      "learning_rate": 1.3821894809301763e-06,
      "log_odds_chosen": 1.4854857921600342,
      "log_odds_ratio": -0.4117187559604645,
      "logits/chosen": -1.0166015625,
      "logits/rejected": -0.9117187261581421,
      "logps/chosen": -0.679882824420929,
      "logps/rejected": -1.7296874523162842,
      "loss": 0.9265,
      "nll_loss": 0.858691394329071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06796874850988388,
      "rewards/margins": 0.10483551025390625,
      "rewards/rejected": -0.17290039360523224,
      "step": 3350
    },
    {
      "epoch": 0.2446572250336768,
      "grad_norm": 2.3791373909526206,
      "learning_rate": 1.3801311186847081e-06,
      "log_odds_chosen": 1.338281273841858,
      "log_odds_ratio": -0.40966796875,
      "logits/chosen": -1.057031273841858,
      "logits/rejected": -0.9261718988418579,
      "logps/chosen": -0.6436523199081421,
      "logps/rejected": -1.578515648841858,
      "loss": 0.9323,
      "nll_loss": 0.8744140863418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06434325873851776,
      "rewards/margins": 0.09346923977136612,
      "rewards/rejected": -0.157958984375,
      "step": 3360
    },
    {
      "epoch": 0.2453853715367532,
      "grad_norm": 2.9431341239057374,
      "learning_rate": 1.378081925099109e-06,
      "log_odds_chosen": 1.499414086341858,
      "log_odds_ratio": -0.3798828125,
      "logits/chosen": -1.043554663658142,
      "logits/rejected": -0.905468761920929,
      "logps/chosen": -0.66748046875,
      "logps/rejected": -1.739843726158142,
      "loss": 0.9167,
      "nll_loss": 0.8755859136581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0667724609375,
      "rewards/margins": 0.107208251953125,
      "rewards/rejected": -0.17404785752296448,
      "step": 3370
    },
    {
      "epoch": 0.24611351803982962,
      "grad_norm": 1.9785449319589474,
      "learning_rate": 1.376041832307563e-06,
      "log_odds_chosen": 1.524999976158142,
      "log_odds_ratio": -0.37958985567092896,
      "logits/chosen": -1.0068359375,
      "logits/rejected": -0.8876953125,
      "logps/chosen": -0.6705077886581421,
      "logps/rejected": -1.7296874523162842,
      "loss": 0.9534,
      "nll_loss": 0.874804675579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06705322116613388,
      "rewards/margins": 0.10585937649011612,
      "rewards/rejected": -0.17312011122703552,
      "step": 3380
    },
    {
      "epoch": 0.24684166454290604,
      "grad_norm": 1.870208371949718,
      "learning_rate": 1.3740107731454524e-06,
      "log_odds_chosen": 1.1873047351837158,
      "log_odds_ratio": -0.4505371153354645,
      "logits/chosen": -0.994335949420929,
      "logits/rejected": -0.9296875,
      "logps/chosen": -0.669628918170929,
      "logps/rejected": -1.508398413658142,
      "loss": 0.9537,
      "nll_loss": 0.8500000238418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06699218600988388,
      "rewards/margins": 0.08387450873851776,
      "rewards/rejected": -0.15097656846046448,
      "step": 3390
    },
    {
      "epoch": 0.24756981104598244,
      "grad_norm": 2.070363334418945,
      "learning_rate": 1.3719886811400705e-06,
      "log_odds_chosen": 1.3897705078125,
      "log_odds_ratio": -0.42094725370407104,
      "logits/chosen": -0.983593761920929,
      "logits/rejected": -0.873828113079071,
      "logps/chosen": -0.6768554449081421,
      "logps/rejected": -1.6730468273162842,
      "loss": 0.9404,
      "nll_loss": 0.9085937738418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06765136867761612,
      "rewards/margins": 0.09942932426929474,
      "rewards/rejected": -0.1671142578125,
      "step": 3400
    },
    {
      "epoch": 0.24829795754905887,
      "grad_norm": 2.1482349420441684,
      "learning_rate": 1.3699754905014834e-06,
      "log_odds_chosen": 1.43603515625,
      "log_odds_ratio": -0.36176759004592896,
      "logits/chosen": -0.9775390625,
      "logits/rejected": -0.862109363079071,
      "logps/chosen": -0.6426757574081421,
      "logps/rejected": -1.6052734851837158,
      "loss": 0.9344,
      "nll_loss": 0.911914050579071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06424560397863388,
      "rewards/margins": 0.09621582180261612,
      "rewards/rejected": -0.16049805283546448,
      "step": 3410
    },
    {
      "epoch": 0.2490261040521353,
      "grad_norm": 2.141510424454952,
      "learning_rate": 1.3679711361135388e-06,
      "log_odds_chosen": 1.211181640625,
      "log_odds_ratio": -0.4310546815395355,
      "logits/chosen": -0.9970703125,
      "logits/rejected": -0.920703113079071,
      "logps/chosen": -0.7007812261581421,
      "logps/rejected": -1.550390601158142,
      "loss": 0.9168,
      "nll_loss": 0.8329101800918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.070068359375,
      "rewards/margins": 0.08489990234375,
      "rewards/rejected": -0.15500488877296448,
      "step": 3420
    },
    {
      "epoch": 0.24975425055521172,
      "grad_norm": 2.4080216592754056,
      "learning_rate": 1.3659755535250212e-06,
      "log_odds_chosen": 1.6338379383087158,
      "log_odds_ratio": -0.38127440214157104,
      "logits/chosen": -0.96875,
      "logits/rejected": -0.885546863079071,
      "logps/chosen": -0.6625000238418579,
      "logps/rejected": -1.8162109851837158,
      "loss": 0.918,
      "nll_loss": 0.832812488079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0662841796875,
      "rewards/margins": 0.11539916694164276,
      "rewards/rejected": -0.18173828721046448,
      "step": 3430
    },
    {
      "epoch": 0.2504823970582881,
      "grad_norm": 1.7590568219662346,
      "learning_rate": 1.3639886789409469e-06,
      "log_odds_chosen": 1.2498047351837158,
      "log_odds_ratio": -0.41923826932907104,
      "logits/chosen": -1.018164038658142,
      "logits/rejected": -0.9339843988418579,
      "logps/chosen": -0.666796863079071,
      "logps/rejected": -1.492578148841858,
      "loss": 0.939,
      "nll_loss": 0.8726562261581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06666259467601776,
      "rewards/margins": 0.08251953125,
      "rewards/rejected": -0.149169921875,
      "step": 3440
    },
    {
      "epoch": 0.2512105435613645,
      "grad_norm": 1.8599517577101523,
      "learning_rate": 1.3620104492139977e-06,
      "log_odds_chosen": 1.6375000476837158,
      "log_odds_ratio": -0.37993162870407104,
      "logits/chosen": -1.040429711341858,
      "logits/rejected": -0.948046863079071,
      "logps/chosen": -0.6851562261581421,
      "logps/rejected": -1.831640601158142,
      "loss": 0.9036,
      "nll_loss": 0.90625,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06843261420726776,
      "rewards/margins": 0.11461181938648224,
      "rewards/rejected": -0.18310546875,
      "step": 3450
    },
    {
      "epoch": 0.251938690064441,
      "grad_norm": 2.438198042937003,
      "learning_rate": 1.3600408018360918e-06,
      "log_odds_chosen": 1.582617163658142,
      "log_odds_ratio": -0.38139647245407104,
      "logits/chosen": -1.0314452648162842,
      "logits/rejected": -0.920117199420929,
      "logps/chosen": -0.671875,
      "logps/rejected": -1.796484351158142,
      "loss": 0.9084,
      "nll_loss": 0.8744140863418579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06718750298023224,
      "rewards/margins": 0.11269531399011612,
      "rewards/rejected": -0.17988280951976776,
      "step": 3460
    },
    {
      "epoch": 0.25266683656751737,
      "grad_norm": 1.7893320646909474,
      "learning_rate": 1.3580796749300878e-06,
      "log_odds_chosen": 1.408203125,
      "log_odds_ratio": -0.409423828125,
      "logits/chosen": -1.0066406726837158,
      "logits/rejected": -0.9253906011581421,
      "logps/chosen": -0.708789050579071,
      "logps/rejected": -1.661718726158142,
      "loss": 0.9128,
      "nll_loss": 0.887890636920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07076416164636612,
      "rewards/margins": 0.0953369140625,
      "rewards/rejected": -0.16618652641773224,
      "step": 3470
    },
    {
      "epoch": 0.2533949830705938,
      "grad_norm": 2.128657610487306,
      "learning_rate": 1.3561270072416209e-06,
      "log_odds_chosen": 1.4201171398162842,
      "log_odds_ratio": -0.41533201932907104,
      "logits/chosen": -1.011132836341858,
      "logits/rejected": -0.913867175579071,
      "logps/chosen": -0.692187488079071,
      "logps/rejected": -1.728906273841858,
      "loss": 0.9361,
      "nll_loss": 0.8677734136581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06925048679113388,
      "rewards/margins": 0.10357666015625,
      "rewards/rejected": -0.1727294921875,
      "step": 3480
    },
    {
      "epoch": 0.2541231295736702,
      "grad_norm": 1.8041119682970694,
      "learning_rate": 1.3541827381310652e-06,
      "log_odds_chosen": 1.2841796875,
      "log_odds_ratio": -0.4425292909145355,
      "logits/chosen": -0.9683593511581421,
      "logits/rejected": -0.8539062738418579,
      "logps/chosen": -0.7339843511581421,
      "logps/rejected": -1.633203148841858,
      "loss": 0.9178,
      "nll_loss": 0.9322265386581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0733642578125,
      "rewards/margins": 0.089874267578125,
      "rewards/rejected": -0.16330567002296448,
      "step": 3490
    },
    {
      "epoch": 0.2548512760767466,
      "grad_norm": 1.8833387818705296,
      "learning_rate": 1.3522468075656264e-06,
      "log_odds_chosen": 1.2472655773162842,
      "log_odds_ratio": -0.42578125,
      "logits/chosen": -1.010156273841858,
      "logits/rejected": -0.886523425579071,
      "logps/chosen": -0.7027343511581421,
      "logps/rejected": -1.543554663658142,
      "loss": 0.9257,
      "nll_loss": 0.826953113079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07027588039636612,
      "rewards/margins": 0.08406372368335724,
      "rewards/rejected": -0.15451660752296448,
      "step": 3500
    },
    {
      "epoch": 0.2555794225798231,
      "grad_norm": 1.9071480388205753,
      "learning_rate": 1.3503191561115553e-06,
      "log_odds_chosen": 1.1837890148162842,
      "log_odds_ratio": -0.4224609434604645,
      "logits/chosen": -1.055273413658142,
      "logits/rejected": -0.965039074420929,
      "logps/chosen": -0.680957019329071,
      "logps/rejected": -1.4607422351837158,
      "loss": 0.9192,
      "nll_loss": 0.809765636920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06807861477136612,
      "rewards/margins": 0.07801513373851776,
      "rewards/rejected": -0.14614257216453552,
      "step": 3510
    },
    {
      "epoch": 0.2563075690828995,
      "grad_norm": 2.219931969049714,
      "learning_rate": 1.348399724926484e-06,
      "log_odds_chosen": 1.265039086341858,
      "log_odds_ratio": -0.43745118379592896,
      "logits/chosen": -0.9808593988418579,
      "logits/rejected": -0.884570300579071,
      "logps/chosen": -0.649121105670929,
      "logps/rejected": -1.5558593273162842,
      "loss": 0.9313,
      "nll_loss": 0.8404296636581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06491699069738388,
      "rewards/margins": 0.09067382663488388,
      "rewards/rejected": -0.15556640923023224,
      "step": 3520
    },
    {
      "epoch": 0.2570357155859759,
      "grad_norm": 1.8339356722724671,
      "learning_rate": 1.346488455751882e-06,
      "log_odds_chosen": 1.3312499523162842,
      "log_odds_ratio": -0.4427734315395355,
      "logits/chosen": -1.008398413658142,
      "logits/rejected": -0.9072265625,
      "logps/chosen": -0.7255859375,
      "logps/rejected": -1.672265648841858,
      "loss": 0.9429,
      "nll_loss": 0.888476550579071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07257080078125,
      "rewards/margins": 0.0946044921875,
      "rewards/rejected": -0.16730956733226776,
      "step": 3530
    },
    {
      "epoch": 0.25776386208905233,
      "grad_norm": 2.051741764411779,
      "learning_rate": 1.3445852909056286e-06,
      "log_odds_chosen": 1.495019555091858,
      "log_odds_ratio": -0.37114256620407104,
      "logits/chosen": -0.962890625,
      "logits/rejected": -0.9111328125,
      "logps/chosen": -0.68115234375,
      "logps/rejected": -1.71484375,
      "loss": 0.9199,
      "nll_loss": 0.8687499761581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06800536811351776,
      "rewards/margins": 0.10347900539636612,
      "rewards/rejected": -0.17143554985523224,
      "step": 3540
    },
    {
      "epoch": 0.25849200859212873,
      "grad_norm": 2.1376568213975564,
      "learning_rate": 1.3426901732747024e-06,
      "log_odds_chosen": 1.2824218273162842,
      "log_odds_ratio": -0.42626953125,
      "logits/chosen": -1.003515601158142,
      "logits/rejected": -0.9208984375,
      "logps/chosen": -0.6385742425918579,
      "logps/rejected": -1.525781273841858,
      "loss": 0.9193,
      "nll_loss": 0.840624988079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06389160454273224,
      "rewards/margins": 0.08857421576976776,
      "rewards/rejected": -0.15244141221046448,
      "step": 3550
    },
    {
      "epoch": 0.25922015509520513,
      "grad_norm": 2.1283031384308146,
      "learning_rate": 1.3408030463079818e-06,
      "log_odds_chosen": 1.439062476158142,
      "log_odds_ratio": -0.3955078125,
      "logits/chosen": -0.988476574420929,
      "logits/rejected": -0.893359363079071,
      "logps/chosen": -0.6976562738418579,
      "logps/rejected": -1.6873047351837158,
      "loss": 0.9367,
      "nll_loss": 0.8794921636581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06972656399011612,
      "rewards/margins": 0.09901122748851776,
      "rewards/rejected": -0.16879883408546448,
      "step": 3560
    },
    {
      "epoch": 0.2599483015982816,
      "grad_norm": 1.9317890240768365,
      "learning_rate": 1.3389238540091568e-06,
      "log_odds_chosen": 1.162109375,
      "log_odds_ratio": -0.4800781309604645,
      "logits/chosen": -0.999804675579071,
      "logits/rejected": -0.9144531488418579,
      "logps/chosen": -0.695117175579071,
      "logps/rejected": -1.4783203601837158,
      "loss": 0.9294,
      "nll_loss": 0.878222644329071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06940917670726776,
      "rewards/margins": 0.0784759521484375,
      "rewards/rejected": -0.14775390923023224,
      "step": 3570
    },
    {
      "epoch": 0.260676448101358,
      "grad_norm": 2.145466794618356,
      "learning_rate": 1.337052540929751e-06,
      "log_odds_chosen": 1.366308569908142,
      "log_odds_ratio": -0.42158204317092896,
      "logits/chosen": -1.0291016101837158,
      "logits/rejected": -0.952343761920929,
      "logps/chosen": -0.698046863079071,
      "logps/rejected": -1.630468726158142,
      "loss": 0.9262,
      "nll_loss": 0.890625,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06981201469898224,
      "rewards/margins": 0.09336242824792862,
      "rewards/rejected": -0.16318359971046448,
      "step": 3580
    },
    {
      "epoch": 0.26140459460443444,
      "grad_norm": 2.2878805210850652,
      "learning_rate": 1.33518905216225e-06,
      "log_odds_chosen": 1.3230469226837158,
      "log_odds_ratio": -0.4310058653354645,
      "logits/chosen": -1.018164038658142,
      "logits/rejected": -0.912890613079071,
      "logps/chosen": -0.73046875,
      "logps/rejected": -1.6613280773162842,
      "loss": 0.9049,
      "nll_loss": 0.862500011920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07310791313648224,
      "rewards/margins": 0.09298095852136612,
      "rewards/rejected": -0.16616210341453552,
      "step": 3590
    },
    {
      "epoch": 0.26213274110751084,
      "grad_norm": 2.529065082726043,
      "learning_rate": 1.3333333333333332e-06,
      "log_odds_chosen": 1.219824194908142,
      "log_odds_ratio": -0.4585937559604645,
      "logits/chosen": -1.0314452648162842,
      "logits/rejected": -0.935351550579071,
      "logps/chosen": -0.7060546875,
      "logps/rejected": -1.5496094226837158,
      "loss": 0.9263,
      "nll_loss": 0.8666015863418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07060547173023224,
      "rewards/margins": 0.08421631157398224,
      "rewards/rejected": -0.15488281846046448,
      "step": 3600
    },
    {
      "epoch": 0.26286088761058723,
      "grad_norm": 2.14329552263589,
      "learning_rate": 1.3314853305972122e-06,
      "log_odds_chosen": 1.574804663658142,
      "log_odds_ratio": -0.36455076932907104,
      "logits/chosen": -0.979296863079071,
      "logits/rejected": -0.851367175579071,
      "logps/chosen": -0.660937488079071,
      "logps/rejected": -1.767578125,
      "loss": 0.9349,
      "nll_loss": 0.93359375,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06612548977136612,
      "rewards/margins": 0.11071167141199112,
      "rewards/rejected": -0.17680664360523224,
      "step": 3610
    },
    {
      "epoch": 0.2635890341136637,
      "grad_norm": 2.054969353533555,
      "learning_rate": 1.3296449906290671e-06,
      "log_odds_chosen": 1.6687500476837158,
      "log_odds_ratio": -0.34404295682907104,
      "logits/chosen": -1.037500023841858,
      "logits/rejected": -0.8951171636581421,
      "logps/chosen": -0.665332019329071,
      "logps/rejected": -1.8347656726837158,
      "loss": 0.9152,
      "nll_loss": 0.8707031011581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06657715141773224,
      "rewards/margins": 0.11689452826976776,
      "rewards/rejected": -0.18349608778953552,
      "step": 3620
    },
    {
      "epoch": 0.2643171806167401,
      "grad_norm": 2.0298369960702627,
      "learning_rate": 1.3278122606185844e-06,
      "log_odds_chosen": 1.4845702648162842,
      "log_odds_ratio": -0.43681639432907104,
      "logits/chosen": -1.007226586341858,
      "logits/rejected": -0.883593738079071,
      "logps/chosen": -0.654296875,
      "logps/rejected": -1.68359375,
      "loss": 0.9293,
      "nll_loss": 0.833984375,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06545410305261612,
      "rewards/margins": 0.10294189304113388,
      "rewards/rejected": -0.16826172173023224,
      "step": 3630
    },
    {
      "epoch": 0.2650453271198165,
      "grad_norm": 2.0786852622533303,
      "learning_rate": 1.3259870882635918e-06,
      "log_odds_chosen": 1.4045898914337158,
      "log_odds_ratio": -0.4500732421875,
      "logits/chosen": -0.9847656488418579,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.68505859375,
      "logps/rejected": -1.722070336341858,
      "loss": 0.9263,
      "nll_loss": 0.8955078125,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06846924126148224,
      "rewards/margins": 0.10381774604320526,
      "rewards/rejected": -0.17229004204273224,
      "step": 3640
    },
    {
      "epoch": 0.26577347362289294,
      "grad_norm": 2.065679727412843,
      "learning_rate": 1.3241694217637886e-06,
      "log_odds_chosen": 1.4490234851837158,
      "log_odds_ratio": -0.402587890625,
      "logits/chosen": -1.0089843273162842,
      "logits/rejected": -0.910937488079071,
      "logps/chosen": -0.6944335699081421,
      "logps/rejected": -1.706640601158142,
      "loss": 0.9267,
      "nll_loss": 0.916015625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06939697265625,
      "rewards/margins": 0.10113525390625,
      "rewards/rejected": -0.17048339545726776,
      "step": 3650
    },
    {
      "epoch": 0.26650162012596934,
      "grad_norm": 2.1227982487132735,
      "learning_rate": 1.3223592098145723e-06,
      "log_odds_chosen": 1.2158203125,
      "log_odds_ratio": -0.4048828184604645,
      "logits/chosen": -1.023828148841858,
      "logits/rejected": -0.932812511920929,
      "logps/chosen": -0.6910156011581421,
      "logps/rejected": -1.528906226158142,
      "loss": 0.9229,
      "nll_loss": 0.900585949420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06914062798023224,
      "rewards/margins": 0.08385010063648224,
      "rewards/rejected": -0.15285643935203552,
      "step": 3660
    },
    {
      "epoch": 0.26722976662904574,
      "grad_norm": 1.880816951977088,
      "learning_rate": 1.3205564016009555e-06,
      "log_odds_chosen": 1.0674560070037842,
      "log_odds_ratio": -0.4695800840854645,
      "logits/chosen": -1.016210913658142,
      "logits/rejected": -0.9375,
      "logps/chosen": -0.642578125,
      "logps/rejected": -1.3699219226837158,
      "loss": 0.9333,
      "nll_loss": 0.828320324420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06424560397863388,
      "rewards/margins": 0.07275619357824326,
      "rewards/rejected": -0.13706055283546448,
      "step": 3670
    },
    {
      "epoch": 0.2679579131321222,
      "grad_norm": 2.078438689330952,
      "learning_rate": 1.318760946791574e-06,
      "log_odds_chosen": 1.455322265625,
      "log_odds_ratio": -0.44965821504592896,
      "logits/chosen": -1.031835913658142,
      "logits/rejected": -0.869921863079071,
      "logps/chosen": -0.724316418170929,
      "logps/rejected": -1.792578101158142,
      "loss": 0.9201,
      "nll_loss": 0.915234386920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07241211086511612,
      "rewards/margins": 0.10657958686351776,
      "rewards/rejected": -0.17890624701976776,
      "step": 3680
    },
    {
      "epoch": 0.2686860596351986,
      "grad_norm": 1.770180295412941,
      "learning_rate": 1.316972795532786e-06,
      "log_odds_chosen": 1.380761742591858,
      "log_odds_ratio": -0.4234375059604645,
      "logits/chosen": -0.9789062738418579,
      "logits/rejected": -0.9085937738418579,
      "logps/chosen": -0.706835925579071,
      "logps/rejected": -1.6550781726837158,
      "loss": 0.9225,
      "nll_loss": 0.84521484375,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07066650688648224,
      "rewards/margins": 0.09483642876148224,
      "rewards/rejected": -0.16557617485523224,
      "step": 3690
    },
    {
      "epoch": 0.26941420613827505,
      "grad_norm": 2.080720067998385,
      "learning_rate": 1.3151918984428582e-06,
      "log_odds_chosen": 1.280615210533142,
      "log_odds_ratio": -0.44807130098342896,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.890625,
      "logps/chosen": -0.6983398199081421,
      "logps/rejected": -1.616601586341858,
      "loss": 0.9095,
      "nll_loss": 0.9105468988418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06979980319738388,
      "rewards/margins": 0.09171752631664276,
      "rewards/rejected": -0.16159668564796448,
      "step": 3700
    },
    {
      "epoch": 0.27014235264135145,
      "grad_norm": 2.4581478536055124,
      "learning_rate": 1.313418206606237e-06,
      "log_odds_chosen": 1.448144555091858,
      "log_odds_ratio": -0.39179688692092896,
      "logits/chosen": -1.032812476158142,
      "logits/rejected": -0.875,
      "logps/chosen": -0.646191418170929,
      "logps/rejected": -1.6785156726837158,
      "loss": 0.8938,
      "nll_loss": 0.780078113079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0645751953125,
      "rewards/margins": 0.103118896484375,
      "rewards/rejected": -0.16774901747703552,
      "step": 3710
    },
    {
      "epoch": 0.27087049914442785,
      "grad_norm": 1.8616826962580717,
      "learning_rate": 1.3116516715679057e-06,
      "log_odds_chosen": 1.378515601158142,
      "log_odds_ratio": -0.39208984375,
      "logits/chosen": -1.0302734375,
      "logits/rejected": -0.943359375,
      "logps/chosen": -0.630566418170929,
      "logps/rejected": -1.560156226158142,
      "loss": 0.9291,
      "nll_loss": 0.887499988079071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06309814751148224,
      "rewards/margins": 0.09299316257238388,
      "rewards/rejected": -0.15598145127296448,
      "step": 3720
    },
    {
      "epoch": 0.2715986456475043,
      "grad_norm": 2.0681530201690546,
      "learning_rate": 1.3098922453278258e-06,
      "log_odds_chosen": 1.2501952648162842,
      "log_odds_ratio": -0.3738769590854645,
      "logits/chosen": -0.9966796636581421,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.6156250238418579,
      "logps/rejected": -1.4021484851837158,
      "loss": 0.9013,
      "nll_loss": 0.83203125,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06157226487994194,
      "rewards/margins": 0.078765869140625,
      "rewards/rejected": -0.14033202826976776,
      "step": 3730
    },
    {
      "epoch": 0.2723267921505807,
      "grad_norm": 2.308783637756246,
      "learning_rate": 1.3081398803354573e-06,
      "log_odds_chosen": 1.2775390148162842,
      "log_odds_ratio": -0.4527831971645355,
      "logits/chosen": -1.035742163658142,
      "logits/rejected": -0.92578125,
      "logps/chosen": -0.7310546636581421,
      "logps/rejected": -1.651953101158142,
      "loss": 0.9044,
      "nll_loss": 0.903515636920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07314453274011612,
      "rewards/margins": 0.09195556491613388,
      "rewards/rejected": -0.16506347060203552,
      "step": 3740
    },
    {
      "epoch": 0.2730549386536571,
      "grad_norm": 2.319177821523628,
      "learning_rate": 1.3063945294843617e-06,
      "log_odds_chosen": 1.370996117591858,
      "log_odds_ratio": -0.38725584745407104,
      "logits/chosen": -0.971484363079071,
      "logits/rejected": -0.851757824420929,
      "logps/chosen": -0.658984363079071,
      "logps/rejected": -1.611718773841858,
      "loss": 0.8906,
      "nll_loss": 0.855664074420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06588134914636612,
      "rewards/margins": 0.09519042819738388,
      "rewards/rejected": -0.1611328125,
      "step": 3750
    },
    {
      "epoch": 0.27378308515673355,
      "grad_norm": 1.761299683171171,
      "learning_rate": 1.3046561461068843e-06,
      "log_odds_chosen": 1.4573853015899658,
      "log_odds_ratio": -0.41669923067092896,
      "logits/chosen": -1.0388672351837158,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.693359375,
      "logps/rejected": -1.772851586341858,
      "loss": 0.927,
      "nll_loss": 0.8705078363418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06943359225988388,
      "rewards/margins": 0.10798950493335724,
      "rewards/rejected": -0.17720946669578552,
      "step": 3760
    },
    {
      "epoch": 0.27451123165980995,
      "grad_norm": 2.8612497602223397,
      "learning_rate": 1.3029246839689124e-06,
      "log_odds_chosen": 1.4542968273162842,
      "log_odds_ratio": -0.3822265565395355,
      "logits/chosen": -1.013281226158142,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.658007800579071,
      "logps/rejected": -1.65625,
      "loss": 0.9125,
      "nll_loss": 0.887890636920929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06575927883386612,
      "rewards/margins": 0.099945068359375,
      "rewards/rejected": -0.16567382216453552,
      "step": 3770
    },
    {
      "epoch": 0.27523937816288635,
      "grad_norm": 2.555510954711944,
      "learning_rate": 1.3012000972647109e-06,
      "log_odds_chosen": 1.423681616783142,
      "log_odds_ratio": -0.4099365174770355,
      "logits/chosen": -0.9964843988418579,
      "logits/rejected": -0.8597656488418579,
      "logps/chosen": -0.6548827886581421,
      "logps/rejected": -1.695703148841858,
      "loss": 0.9301,
      "nll_loss": 0.867382824420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06550292670726776,
      "rewards/margins": 0.10407409816980362,
      "rewards/rejected": -0.16945800185203552,
      "step": 3780
    },
    {
      "epoch": 0.2759675246659628,
      "grad_norm": 1.9698754464810986,
      "learning_rate": 1.299482340611832e-06,
      "log_odds_chosen": 1.4599609375,
      "log_odds_ratio": -0.45283204317092896,
      "logits/chosen": -0.963671863079071,
      "logits/rejected": -0.8607422113418579,
      "logps/chosen": -0.711230456829071,
      "logps/rejected": -1.763281226158142,
      "loss": 0.9143,
      "nll_loss": 0.890820324420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07105712592601776,
      "rewards/margins": 0.10535888373851776,
      "rewards/rejected": -0.1763916015625,
      "step": 3790
    },
    {
      "epoch": 0.2766956711690392,
      "grad_norm": 2.0439385608108673,
      "learning_rate": 1.2977713690461003e-06,
      "log_odds_chosen": 1.425927758216858,
      "log_odds_ratio": -0.44755858182907104,
      "logits/chosen": -0.9779297113418579,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.7183593511581421,
      "logps/rejected": -1.7746093273162842,
      "loss": 0.9203,
      "nll_loss": 0.903515636920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07191161811351776,
      "rewards/margins": 0.10572052001953125,
      "rewards/rejected": -0.17753906548023224,
      "step": 3800
    },
    {
      "epoch": 0.2774238176721156,
      "grad_norm": 2.1809033960735817,
      "learning_rate": 1.296067138016669e-06,
      "log_odds_chosen": 1.251953125,
      "log_odds_ratio": -0.42158204317092896,
      "logits/chosen": -0.9775390625,
      "logits/rejected": -0.8910156488418579,
      "logps/chosen": -0.649121105670929,
      "logps/rejected": -1.493749976158142,
      "loss": 0.9036,
      "nll_loss": 0.8109375238418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06490478664636612,
      "rewards/margins": 0.08449707180261612,
      "rewards/rejected": -0.14948730170726776,
      "step": 3810
    },
    {
      "epoch": 0.27815196417519206,
      "grad_norm": 2.383663367704226,
      "learning_rate": 1.294369603381147e-06,
      "log_odds_chosen": 1.4202148914337158,
      "log_odds_ratio": -0.3775390684604645,
      "logits/chosen": -1.0173828601837158,
      "logits/rejected": -0.932421863079071,
      "logps/chosen": -0.6888672113418579,
      "logps/rejected": -1.6677734851837158,
      "loss": 0.9208,
      "nll_loss": 0.879101574420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06892089545726776,
      "rewards/margins": 0.09795226901769638,
      "rewards/rejected": -0.166748046875,
      "step": 3820
    },
    {
      "epoch": 0.27888011067826846,
      "grad_norm": 1.862087057097791,
      "learning_rate": 1.2926787214007981e-06,
      "log_odds_chosen": 1.414160132408142,
      "log_odds_ratio": -0.4173339903354645,
      "logits/chosen": -1.013281226158142,
      "logits/rejected": -0.870312511920929,
      "logps/chosen": -0.6859375238418579,
      "logps/rejected": -1.6845703125,
      "loss": 0.9364,
      "nll_loss": 0.9281250238418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06862793117761612,
      "rewards/margins": 0.09991760551929474,
      "rewards/rejected": -0.16848143935203552,
      "step": 3830
    },
    {
      "epoch": 0.2796082571813449,
      "grad_norm": 1.8573254019846177,
      "learning_rate": 1.2909944487358056e-06,
      "log_odds_chosen": 1.089941382408142,
      "log_odds_ratio": -0.46923828125,
      "logits/chosen": -0.9828125238418579,
      "logits/rejected": -0.865429699420929,
      "logps/chosen": -0.677734375,
      "logps/rejected": -1.4169921875,
      "loss": 0.909,
      "nll_loss": 0.867382824420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06781005859375,
      "rewards/margins": 0.07388915866613388,
      "rewards/rejected": -0.14157715439796448,
      "step": 3840
    },
    {
      "epoch": 0.2803364036844213,
      "grad_norm": 2.9560278293757656,
      "learning_rate": 1.2893167424406084e-06,
      "log_odds_chosen": 1.584375023841858,
      "log_odds_ratio": -0.37211912870407104,
      "logits/chosen": -1.0294921398162842,
      "logits/rejected": -0.8984375,
      "logps/chosen": -0.6563476324081421,
      "logps/rejected": -1.7556641101837158,
      "loss": 0.916,
      "nll_loss": 0.873242199420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06560058891773224,
      "rewards/margins": 0.10994873195886612,
      "rewards/rejected": -0.17558594048023224,
      "step": 3850
    },
    {
      "epoch": 0.2810645501874977,
      "grad_norm": 2.301857896508706,
      "learning_rate": 1.2876455599593008e-06,
      "log_odds_chosen": 1.272314429283142,
      "log_odds_ratio": -0.4703125059604645,
      "logits/chosen": -0.96875,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.7123047113418579,
      "logps/rejected": -1.6369140148162842,
      "loss": 0.899,
      "nll_loss": 0.907421886920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07126464694738388,
      "rewards/margins": 0.092315673828125,
      "rewards/rejected": -0.16362304985523224,
      "step": 3860
    },
    {
      "epoch": 0.28179269669057416,
      "grad_norm": 1.9693252976660351,
      "learning_rate": 1.285980859121099e-06,
      "log_odds_chosen": 1.489843726158142,
      "log_odds_ratio": -0.3890136778354645,
      "logits/chosen": -0.996874988079071,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -0.6749023199081421,
      "logps/rejected": -1.7138671875,
      "loss": 0.912,
      "nll_loss": 0.927539050579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06744384765625,
      "rewards/margins": 0.10399170219898224,
      "rewards/rejected": -0.17145995795726776,
      "step": 3870
    },
    {
      "epoch": 0.28252084319365056,
      "grad_norm": 1.8250221689884194,
      "learning_rate": 1.2843225981358712e-06,
      "log_odds_chosen": 1.29486083984375,
      "log_odds_ratio": -0.3955078125,
      "logits/chosen": -1.0125000476837158,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.661914050579071,
      "logps/rejected": -1.5587890148162842,
      "loss": 0.903,
      "nll_loss": 0.8369140625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06622314453125,
      "rewards/margins": 0.0897369384765625,
      "rewards/rejected": -0.15590819716453552,
      "step": 3880
    },
    {
      "epoch": 0.28324898969672696,
      "grad_norm": 1.9583934401346017,
      "learning_rate": 1.2826707355897317e-06,
      "log_odds_chosen": 1.431249976158142,
      "log_odds_ratio": -0.40107423067092896,
      "logits/chosen": -0.948437511920929,
      "logits/rejected": -0.8599609136581421,
      "logps/chosen": -0.7049804925918579,
      "logps/rejected": -1.7117187976837158,
      "loss": 0.9119,
      "nll_loss": 0.982617199420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07045898586511612,
      "rewards/margins": 0.10072021186351776,
      "rewards/rejected": -0.1712646484375,
      "step": 3890
    },
    {
      "epoch": 0.2839771361998034,
      "grad_norm": 2.390947632672128,
      "learning_rate": 1.281025230440697e-06,
      "log_odds_chosen": 1.03076171875,
      "log_odds_ratio": -0.46064454317092896,
      "logits/chosen": -0.994335949420929,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -0.6996093988418579,
      "logps/rejected": -1.390625,
      "loss": 0.9179,
      "nll_loss": 0.862109363079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0699462890625,
      "rewards/margins": 0.06906585395336151,
      "rewards/rejected": -0.13908691704273224,
      "step": 3900
    },
    {
      "epoch": 0.2847052827028798,
      "grad_norm": 1.906944086757529,
      "learning_rate": 1.2793860420144025e-06,
      "log_odds_chosen": 1.515039086341858,
      "log_odds_ratio": -0.4058593809604645,
      "logits/chosen": -0.9525390863418579,
      "logits/rejected": -0.833789050579071,
      "logps/chosen": -0.6563476324081421,
      "logps/rejected": -1.704687476158142,
      "loss": 0.8985,
      "nll_loss": 0.836132824420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06557617336511612,
      "rewards/margins": 0.10487060248851776,
      "rewards/rejected": -0.17050781846046448,
      "step": 3910
    },
    {
      "epoch": 0.2854334292059562,
      "grad_norm": 2.3331581164239474,
      "learning_rate": 1.2777531299998798e-06,
      "log_odds_chosen": 1.26416015625,
      "log_odds_ratio": -0.46088868379592896,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.887499988079071,
      "logps/chosen": -0.6910156011581421,
      "logps/rejected": -1.564453125,
      "loss": 0.9413,
      "nll_loss": 0.849414050579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06914062798023224,
      "rewards/margins": 0.0873565673828125,
      "rewards/rejected": -0.15642090141773224,
      "step": 3920
    },
    {
      "epoch": 0.28616157570903267,
      "grad_norm": 2.088158930255884,
      "learning_rate": 1.2761264544453928e-06,
      "log_odds_chosen": 1.27783203125,
      "log_odds_ratio": -0.42866212129592896,
      "logits/chosen": -0.9876953363418579,
      "logits/rejected": -0.868359386920929,
      "logps/chosen": -0.6851562261581421,
      "logps/rejected": -1.587890625,
      "loss": 0.9007,
      "nll_loss": 0.8384765386581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06846924126148224,
      "rewards/margins": 0.09032592922449112,
      "rewards/rejected": -0.15895995497703552,
      "step": 3930
    },
    {
      "epoch": 0.28688972221210907,
      "grad_norm": 2.0013029666363837,
      "learning_rate": 1.2745059757543324e-06,
      "log_odds_chosen": 1.3886229991912842,
      "log_odds_ratio": -0.43559569120407104,
      "logits/chosen": -0.989062488079071,
      "logits/rejected": -0.888867199420929,
      "logps/chosen": -0.7183593511581421,
      "logps/rejected": -1.6789062023162842,
      "loss": 0.8902,
      "nll_loss": 0.859570324420929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07181396335363388,
      "rewards/margins": 0.09621276706457138,
      "rewards/rejected": -0.16787108778953552,
      "step": 3940
    },
    {
      "epoch": 0.2876178687151855,
      "grad_norm": 1.980447538853639,
      "learning_rate": 1.272891654681168e-06,
      "log_odds_chosen": 1.460302710533142,
      "log_odds_ratio": -0.4136718809604645,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.8843749761581421,
      "logps/chosen": -0.739453136920929,
      "logps/rejected": -1.808984398841858,
      "loss": 0.8962,
      "nll_loss": 0.882617175579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07402344048023224,
      "rewards/margins": 0.10699462890625,
      "rewards/rejected": -0.18085937201976776,
      "step": 3950
    },
    {
      "epoch": 0.2883460152182619,
      "grad_norm": 2.1029680609042822,
      "learning_rate": 1.2712834523274563e-06,
      "log_odds_chosen": 1.3596680164337158,
      "log_odds_ratio": -0.4193359315395355,
      "logits/chosen": -0.9750000238418579,
      "logits/rejected": -0.8765624761581421,
      "logps/chosen": -0.6722656488418579,
      "logps/rejected": -1.596093773841858,
      "loss": 0.904,
      "nll_loss": 0.8792968988418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06724853813648224,
      "rewards/margins": 0.09249267727136612,
      "rewards/rejected": -0.15961913764476776,
      "step": 3960
    },
    {
      "epoch": 0.2890741617213383,
      "grad_norm": 2.010968121301939,
      "learning_rate": 1.2696813301379032e-06,
      "log_odds_chosen": 1.6116211414337158,
      "log_odds_ratio": -0.35407716035842896,
      "logits/chosen": -0.9740234613418579,
      "logits/rejected": -0.8427734375,
      "logps/chosen": -0.625,
      "logps/rejected": -1.7361328601837158,
      "loss": 0.9081,
      "nll_loss": 0.854687511920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06253661960363388,
      "rewards/margins": 0.11113891750574112,
      "rewards/rejected": -0.17360839247703552,
      "step": 3970
    },
    {
      "epoch": 0.2898023082244148,
      "grad_norm": 2.19757893074996,
      "learning_rate": 1.2680852498964829e-06,
      "log_odds_chosen": 1.2998046875,
      "log_odds_ratio": -0.45917969942092896,
      "logits/chosen": -0.971484363079071,
      "logits/rejected": -0.8765624761581421,
      "logps/chosen": -0.6751953363418579,
      "logps/rejected": -1.6300780773162842,
      "loss": 0.8971,
      "nll_loss": 0.8603515625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0675048828125,
      "rewards/margins": 0.09555206447839737,
      "rewards/rejected": -0.16301269829273224,
      "step": 3980
    },
    {
      "epoch": 0.29053045472749117,
      "grad_norm": 2.114679185301981,
      "learning_rate": 1.266495173722607e-06,
      "log_odds_chosen": 1.539648413658142,
      "log_odds_ratio": -0.3841308653354645,
      "logits/chosen": -1.01171875,
      "logits/rejected": -0.90234375,
      "logps/chosen": -0.654101550579071,
      "logps/rejected": -1.697851538658142,
      "loss": 0.8915,
      "nll_loss": 0.8187500238418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06541748344898224,
      "rewards/margins": 0.10440673679113388,
      "rewards/rejected": -0.16989746689796448,
      "step": 3990
    },
    {
      "epoch": 0.29125860123056757,
      "grad_norm": 3.003774127739079,
      "learning_rate": 1.2649110640673517e-06,
      "log_odds_chosen": 1.4013671875,
      "log_odds_ratio": -0.4100097715854645,
      "logits/chosen": -0.9789062738418579,
      "logits/rejected": -0.850781261920929,
      "logps/chosen": -0.6455078125,
      "logps/rejected": -1.581640601158142,
      "loss": 0.9143,
      "nll_loss": 0.8765624761581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06450195610523224,
      "rewards/margins": 0.09369506686925888,
      "rewards/rejected": -0.15817871689796448,
      "step": 4000
    },
    {
      "epoch": 0.291986747733644,
      "grad_norm": 1.9432782525194665,
      "learning_rate": 1.2633328837097308e-06,
      "log_odds_chosen": 1.6609375476837158,
      "log_odds_ratio": -0.35004884004592896,
      "logits/chosen": -1.032617211341858,
      "logits/rejected": -0.9076172113418579,
      "logps/chosen": -0.647656261920929,
      "logps/rejected": -1.8406250476837158,
      "loss": 0.9006,
      "nll_loss": 0.8480468988418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06474609673023224,
      "rewards/margins": 0.1192626953125,
      "rewards/rejected": -0.18405762314796448,
      "step": 4010
    },
    {
      "epoch": 0.2927148942367204,
      "grad_norm": 1.9009380432104825,
      "learning_rate": 1.2617605957530233e-06,
      "log_odds_chosen": 1.289648413658142,
      "log_odds_ratio": -0.43144530057907104,
      "logits/chosen": -1.006445288658142,
      "logits/rejected": -0.8998047113418579,
      "logps/chosen": -0.6607421636581421,
      "logps/rejected": -1.551171898841858,
      "loss": 0.8871,
      "nll_loss": 0.876171886920929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0660400390625,
      "rewards/margins": 0.08900757133960724,
      "rewards/rejected": -0.15500488877296448,
      "step": 4020
    },
    {
      "epoch": 0.2934430407397968,
      "grad_norm": 2.1431399739267727,
      "learning_rate": 1.2601941636211516e-06,
      "log_odds_chosen": 1.6111328601837158,
      "log_odds_ratio": -0.37641602754592896,
      "logits/chosen": -0.95703125,
      "logits/rejected": -0.872265636920929,
      "logps/chosen": -0.6869140863418579,
      "logps/rejected": -1.8468749523162842,
      "loss": 0.8911,
      "nll_loss": 0.808789074420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06871338188648224,
      "rewards/margins": 0.11597900092601776,
      "rewards/rejected": -0.18471679091453552,
      "step": 4030
    },
    {
      "epoch": 0.2941711872428733,
      "grad_norm": 1.782542270707798,
      "learning_rate": 1.2586335510551052e-06,
      "log_odds_chosen": 1.3860352039337158,
      "log_odds_ratio": -0.43657225370407104,
      "logits/chosen": -1.0087890625,
      "logits/rejected": -0.897656261920929,
      "logps/chosen": -0.7105468511581421,
      "logps/rejected": -1.7150390148162842,
      "loss": 0.9062,
      "nll_loss": 0.832812488079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07109375298023224,
      "rewards/margins": 0.10028076171875,
      "rewards/rejected": -0.17131347954273224,
      "step": 4040
    },
    {
      "epoch": 0.2948993337459497,
      "grad_norm": 2.023179698986602,
      "learning_rate": 1.2570787221094177e-06,
      "log_odds_chosen": 1.715429663658142,
      "log_odds_ratio": -0.3423828184604645,
      "logits/chosen": -0.9593750238418579,
      "logits/rejected": -0.8359375,
      "logps/chosen": -0.637011706829071,
      "logps/rejected": -1.837499976158142,
      "loss": 0.9159,
      "nll_loss": 0.8511718511581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.063720703125,
      "rewards/margins": 0.1201171875,
      "rewards/rejected": -0.18393555283546448,
      "step": 4050
    },
    {
      "epoch": 0.29562748024902613,
      "grad_norm": 1.9733336284907965,
      "learning_rate": 1.255529641148689e-06,
      "log_odds_chosen": 1.424218773841858,
      "log_odds_ratio": -0.3931518495082855,
      "logits/chosen": -0.970507800579071,
      "logits/rejected": -0.846484363079071,
      "logps/chosen": -0.620312511920929,
      "logps/rejected": -1.560937523841858,
      "loss": 0.9037,
      "nll_loss": 0.8568359613418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06204833835363388,
      "rewards/margins": 0.09418945014476776,
      "rewards/rejected": -0.15605469048023224,
      "step": 4060
    },
    {
      "epoch": 0.29635562675210253,
      "grad_norm": 2.334298693947026,
      "learning_rate": 1.2539862728441536e-06,
      "log_odds_chosen": 1.18115234375,
      "log_odds_ratio": -0.4518066346645355,
      "logits/chosen": -0.9761718511581421,
      "logits/rejected": -0.8642578125,
      "logps/chosen": -0.6849609613418579,
      "logps/rejected": -1.5076172351837158,
      "loss": 0.8958,
      "nll_loss": 0.897656261920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06846924126148224,
      "rewards/margins": 0.08220825344324112,
      "rewards/rejected": -0.15073242783546448,
      "step": 4070
    },
    {
      "epoch": 0.29708377325517893,
      "grad_norm": 2.0335569719244746,
      "learning_rate": 1.252448582170299e-06,
      "log_odds_chosen": 1.3641235828399658,
      "log_odds_ratio": -0.4237304627895355,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.877734363079071,
      "logps/chosen": -0.6128906011581421,
      "logps/rejected": -1.5457031726837158,
      "loss": 0.8798,
      "nll_loss": 0.785937488079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06124267727136612,
      "rewards/margins": 0.09330139309167862,
      "rewards/rejected": -0.154541015625,
      "step": 4080
    },
    {
      "epoch": 0.2978119197582554,
      "grad_norm": 2.149184128933456,
      "learning_rate": 1.2509165344015243e-06,
      "log_odds_chosen": 1.2607421875,
      "log_odds_ratio": -0.3938964903354645,
      "logits/chosen": -0.9789062738418579,
      "logits/rejected": -0.852343738079071,
      "logps/chosen": -0.6298828125,
      "logps/rejected": -1.476171851158142,
      "loss": 0.8922,
      "nll_loss": 0.8187500238418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06297607719898224,
      "rewards/margins": 0.084564208984375,
      "rewards/rejected": -0.14755859971046448,
      "step": 4090
    },
    {
      "epoch": 0.2985400662613318,
      "grad_norm": 2.4756301036059116,
      "learning_rate": 1.2493900951088486e-06,
      "log_odds_chosen": 1.378625512123108,
      "log_odds_ratio": -0.37871092557907104,
      "logits/chosen": -0.9292968511581421,
      "logits/rejected": -0.830078125,
      "logps/chosen": -0.6573241949081421,
      "logps/rejected": -1.5830078125,
      "loss": 0.9153,
      "nll_loss": 0.859570324420929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06575927883386612,
      "rewards/margins": 0.0926666259765625,
      "rewards/rejected": -0.15837402641773224,
      "step": 4100
    },
    {
      "epoch": 0.2992682127644082,
      "grad_norm": 2.3346024578564335,
      "learning_rate": 1.2478692301566601e-06,
      "log_odds_chosen": 1.7351562976837158,
      "log_odds_ratio": -0.35966795682907104,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.8623046875,
      "logps/chosen": -0.6783202886581421,
      "logps/rejected": -1.9562499523162842,
      "loss": 0.8957,
      "nll_loss": 0.820507824420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06782226264476776,
      "rewards/margins": 0.12806396186351776,
      "rewards/rejected": -0.19584961235523224,
      "step": 4110
    },
    {
      "epoch": 0.29999635926748464,
      "grad_norm": 2.093899272092189,
      "learning_rate": 1.2463539056995116e-06,
      "log_odds_chosen": 1.4303710460662842,
      "log_odds_ratio": -0.42631834745407104,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.8685547113418579,
      "logps/chosen": -0.6587890386581421,
      "logps/rejected": -1.679296851158142,
      "loss": 0.9119,
      "nll_loss": 0.8560546636581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06586913764476776,
      "rewards/margins": 0.10211181640625,
      "rewards/rejected": -0.16801758110523224,
      "step": 4120
    },
    {
      "epoch": 0.30072450577056103,
      "grad_norm": 2.1160267548154827,
      "learning_rate": 1.2448440881789541e-06,
      "log_odds_chosen": 1.455078125,
      "log_odds_ratio": -0.38618165254592896,
      "logits/chosen": -0.991406261920929,
      "logits/rejected": -0.8824218511581421,
      "logps/chosen": -0.6650390625,
      "logps/rejected": -1.6867187023162842,
      "loss": 0.8647,
      "nll_loss": 0.803906261920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06651611626148224,
      "rewards/margins": 0.10212402045726776,
      "rewards/rejected": -0.16862793266773224,
      "step": 4130
    },
    {
      "epoch": 0.30145265227363743,
      "grad_norm": 2.0121108703410564,
      "learning_rate": 1.2433397443204184e-06,
      "log_odds_chosen": 1.251855492591858,
      "log_odds_ratio": -0.4278808534145355,
      "logits/chosen": -0.962890625,
      "logits/rejected": -0.8490234613418579,
      "logps/chosen": -0.6845703125,
      "logps/rejected": -1.5427734851837158,
      "loss": 0.8885,
      "nll_loss": 0.8521484136581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06850586086511612,
      "rewards/margins": 0.08576659858226776,
      "rewards/rejected": -0.15432128310203552,
      "step": 4140
    },
    {
      "epoch": 0.3021807987767139,
      "grad_norm": 1.8965159591039014,
      "learning_rate": 1.2418408411301324e-06,
      "log_odds_chosen": 1.3927733898162842,
      "log_odds_ratio": -0.38300782442092896,
      "logits/chosen": -0.9759765863418579,
      "logits/rejected": -0.841601550579071,
      "logps/chosen": -0.652050793170929,
      "logps/rejected": -1.6193358898162842,
      "loss": 0.8692,
      "nll_loss": 0.873046875,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06522216647863388,
      "rewards/margins": 0.09677734225988388,
      "rewards/rejected": -0.16193847358226776,
      "step": 4150
    },
    {
      "epoch": 0.3029089452797903,
      "grad_norm": 2.031109667589268,
      "learning_rate": 1.2403473458920844e-06,
      "log_odds_chosen": 1.578222632408142,
      "log_odds_ratio": -0.4445556700229645,
      "logits/chosen": -0.9703124761581421,
      "logits/rejected": -0.8609374761581421,
      "logps/chosen": -0.716992199420929,
      "logps/rejected": -1.824609398841858,
      "loss": 0.9008,
      "nll_loss": 0.777148425579071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07174072414636612,
      "rewards/margins": 0.11082763969898224,
      "rewards/rejected": -0.18242187798023224,
      "step": 4160
    },
    {
      "epoch": 0.3036370917828667,
      "grad_norm": 2.124461824682984,
      "learning_rate": 1.2388592261650217e-06,
      "log_odds_chosen": 1.429540991783142,
      "log_odds_ratio": -0.4297851622104645,
      "logits/chosen": -1.0119140148162842,
      "logits/rejected": -0.8755859136581421,
      "logps/chosen": -0.743945300579071,
      "logps/rejected": -1.7462890148162842,
      "loss": 0.8695,
      "nll_loss": 0.8453124761581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0743408203125,
      "rewards/margins": 0.10030059516429901,
      "rewards/rejected": -0.17470702528953552,
      "step": 4170
    },
    {
      "epoch": 0.30436523828594314,
      "grad_norm": 2.076458114968415,
      "learning_rate": 1.2373764497794918e-06,
      "log_odds_chosen": 1.379003882408142,
      "log_odds_ratio": -0.42900389432907104,
      "logits/chosen": -0.949023425579071,
      "logits/rejected": -0.799023449420929,
      "logps/chosen": -0.6749023199081421,
      "logps/rejected": -1.6179687976837158,
      "loss": 0.882,
      "nll_loss": 0.86328125,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06744384765625,
      "rewards/margins": 0.09448547661304474,
      "rewards/rejected": -0.16181640326976776,
      "step": 4180
    },
    {
      "epoch": 0.30509338478901954,
      "grad_norm": 2.48336575201867,
      "learning_rate": 1.2358989848349217e-06,
      "log_odds_chosen": 1.4592773914337158,
      "log_odds_ratio": -0.388671875,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.8876953125,
      "logps/chosen": -0.645312488079071,
      "logps/rejected": -1.64453125,
      "loss": 0.879,
      "nll_loss": 0.80078125,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06447754055261612,
      "rewards/margins": 0.09986571967601776,
      "rewards/rejected": -0.16440430283546448,
      "step": 4190
    },
    {
      "epoch": 0.305821531292096,
      "grad_norm": 2.2489361849729494,
      "learning_rate": 1.2344267996967353e-06,
      "log_odds_chosen": 1.3708984851837158,
      "log_odds_ratio": -0.4219726622104645,
      "logits/chosen": -0.943164050579071,
      "logits/rejected": -0.8589843511581421,
      "logps/chosen": -0.6449218988418579,
      "logps/rejected": -1.6124999523162842,
      "loss": 0.8731,
      "nll_loss": 0.8408203125,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06450195610523224,
      "rewards/margins": 0.09684447944164276,
      "rewards/rejected": -0.16147461533546448,
      "step": 4200
    },
    {
      "epoch": 0.3065496777951724,
      "grad_norm": 2.1237729151961005,
      "learning_rate": 1.2329598629935076e-06,
      "log_odds_chosen": 1.2802734375,
      "log_odds_ratio": -0.42949217557907104,
      "logits/chosen": -0.9775390625,
      "logits/rejected": -0.8671875,
      "logps/chosen": -0.636523425579071,
      "logps/rejected": -1.4904296398162842,
      "loss": 0.8851,
      "nll_loss": 0.8203125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06364746391773224,
      "rewards/margins": 0.08545227348804474,
      "rewards/rejected": -0.14907225966453552,
      "step": 4210
    },
    {
      "epoch": 0.3072778242982488,
      "grad_norm": 2.065119171306644,
      "learning_rate": 1.2314981436141583e-06,
      "log_odds_chosen": 1.4506103992462158,
      "log_odds_ratio": -0.417724609375,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.8853515386581421,
      "logps/chosen": -0.6475585699081421,
      "logps/rejected": -1.6593749523162842,
      "loss": 0.8656,
      "nll_loss": 0.8218749761581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06474609673023224,
      "rewards/margins": 0.1010536178946495,
      "rewards/rejected": -0.165771484375,
      "step": 4220
    },
    {
      "epoch": 0.30800597080132525,
      "grad_norm": 2.0380344009811897,
      "learning_rate": 1.2300416107051802e-06,
      "log_odds_chosen": 1.5849609375,
      "log_odds_ratio": -0.3866210877895355,
      "logits/chosen": -1.0109374523162842,
      "logits/rejected": -0.886523425579071,
      "logps/chosen": -0.6751953363418579,
      "logps/rejected": -1.793359398841858,
      "loss": 0.9131,
      "nll_loss": 0.829882800579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06752929836511612,
      "rewards/margins": 0.11180420219898224,
      "rewards/rejected": -0.1793212890625,
      "step": 4230
    },
    {
      "epoch": 0.30873411730440165,
      "grad_norm": 2.285835766158039,
      "learning_rate": 1.2285902336679024e-06,
      "log_odds_chosen": 1.495703101158142,
      "log_odds_ratio": -0.4005371034145355,
      "logits/chosen": -0.9976562261581421,
      "logits/rejected": -0.9013671875,
      "logps/chosen": -0.6353515386581421,
      "logps/rejected": -1.6248047351837158,
      "loss": 0.8744,
      "nll_loss": 0.8404296636581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0634765625,
      "rewards/margins": 0.09897460788488388,
      "rewards/rejected": -0.1624755859375,
      "step": 4240
    },
    {
      "epoch": 0.30946226380747804,
      "grad_norm": 1.9718872704477093,
      "learning_rate": 1.2271439821557926e-06,
      "log_odds_chosen": 1.4089844226837158,
      "log_odds_ratio": -0.4078613221645355,
      "logits/chosen": -0.9976562261581421,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.6654297113418579,
      "logps/rejected": -1.701562523841858,
      "loss": 0.8766,
      "nll_loss": 0.805859386920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06647948920726776,
      "rewards/margins": 0.10369873046875,
      "rewards/rejected": -0.17023925483226776,
      "step": 4250
    },
    {
      "epoch": 0.3101904103105545,
      "grad_norm": 2.252332268191545,
      "learning_rate": 1.225702826071791e-06,
      "log_odds_chosen": 1.5226562023162842,
      "log_odds_ratio": -0.3954101502895355,
      "logits/chosen": -0.954882800579071,
      "logits/rejected": -0.839648425579071,
      "logps/chosen": -0.698925793170929,
      "logps/rejected": -1.732812523841858,
      "loss": 0.889,
      "nll_loss": 0.8902343511581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06986083835363388,
      "rewards/margins": 0.10344238579273224,
      "rewards/rejected": -0.17336425185203552,
      "step": 4260
    },
    {
      "epoch": 0.3109185568136309,
      "grad_norm": 1.9211632394092473,
      "learning_rate": 1.2242667355656797e-06,
      "log_odds_chosen": 1.467187523841858,
      "log_odds_ratio": -0.383544921875,
      "logits/chosen": -0.946484386920929,
      "logits/rejected": -0.859375,
      "logps/chosen": -0.636523425579071,
      "logps/rejected": -1.6613280773162842,
      "loss": 0.8854,
      "nll_loss": 0.864062488079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06357421725988388,
      "rewards/margins": 0.10262451320886612,
      "rewards/rejected": -0.16630859673023224,
      "step": 4270
    },
    {
      "epoch": 0.3116467033167073,
      "grad_norm": 3.1950683903101416,
      "learning_rate": 1.2228356810314862e-06,
      "log_odds_chosen": 1.430090308189392,
      "log_odds_ratio": -0.44428712129592896,
      "logits/chosen": -0.9615234136581421,
      "logits/rejected": -0.8720703125,
      "logps/chosen": -0.6982421875,
      "logps/rejected": -1.752343773841858,
      "loss": 0.9026,
      "nll_loss": 0.8701171875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06983642280101776,
      "rewards/margins": 0.10538940131664276,
      "rewards/rejected": -0.17524413764476776,
      "step": 4280
    },
    {
      "epoch": 0.31237484981978375,
      "grad_norm": 2.3402498377339014,
      "learning_rate": 1.2214096331049186e-06,
      "log_odds_chosen": 1.1662108898162842,
      "log_odds_ratio": -0.45561522245407104,
      "logits/chosen": -0.9818359613418579,
      "logits/rejected": -0.9039062261581421,
      "logps/chosen": -0.6988281011581421,
      "logps/rejected": -1.468164086341858,
      "loss": 0.8798,
      "nll_loss": 0.868945300579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06993408501148224,
      "rewards/margins": 0.07685241848230362,
      "rewards/rejected": -0.14670410752296448,
      "step": 4290
    },
    {
      "epoch": 0.31310299632286015,
      "grad_norm": 1.9221800339246609,
      "learning_rate": 1.2199885626608373e-06,
      "log_odds_chosen": 1.63818359375,
      "log_odds_ratio": -0.35249024629592896,
      "logits/chosen": -0.961718738079071,
      "logits/rejected": -0.845507800579071,
      "logps/chosen": -0.644726574420929,
      "logps/rejected": -1.7761719226837158,
      "loss": 0.8947,
      "nll_loss": 0.9326171875,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06450195610523224,
      "rewards/margins": 0.11326904594898224,
      "rewards/rejected": -0.1776123046875,
      "step": 4300
    },
    {
      "epoch": 0.3138311428259366,
      "grad_norm": 2.2775132313788222,
      "learning_rate": 1.2185724408107546e-06,
      "log_odds_chosen": 1.50390625,
      "log_odds_ratio": -0.41743165254592896,
      "logits/chosen": -0.985546886920929,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.7251952886581421,
      "logps/rejected": -1.7644531726837158,
      "loss": 0.8963,
      "nll_loss": 0.877148449420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07254638522863388,
      "rewards/margins": 0.10397949069738388,
      "rewards/rejected": -0.17641600966453552,
      "step": 4310
    },
    {
      "epoch": 0.314559289329013,
      "grad_norm": 2.464766740645121,
      "learning_rate": 1.2171612389003689e-06,
      "log_odds_chosen": 1.5910155773162842,
      "log_odds_ratio": -0.3652099668979645,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.87109375,
      "logps/chosen": -0.6219726800918579,
      "logps/rejected": -1.7374999523162842,
      "loss": 0.8766,
      "nll_loss": 0.831835925579071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.062255859375,
      "rewards/margins": 0.11158446967601776,
      "rewards/rejected": -0.17397460341453552,
      "step": 4320
    },
    {
      "epoch": 0.3152874358320894,
      "grad_norm": 2.0525809499950127,
      "learning_rate": 1.2157549285071297e-06,
      "log_odds_chosen": 1.4373047351837158,
      "log_odds_ratio": -0.410888671875,
      "logits/chosen": -0.9339843988418579,
      "logits/rejected": -0.836718738079071,
      "logps/chosen": -0.654101550579071,
      "logps/rejected": -1.668359398841858,
      "loss": 0.9031,
      "nll_loss": 0.8726562261581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06540527194738388,
      "rewards/margins": 0.1014404296875,
      "rewards/rejected": -0.16696777939796448,
      "step": 4330
    },
    {
      "epoch": 0.31601558233516586,
      "grad_norm": 1.896973951984371,
      "learning_rate": 1.2143534814378327e-06,
      "log_odds_chosen": 1.4669921398162842,
      "log_odds_ratio": -0.3897949159145355,
      "logits/chosen": -0.9878906011581421,
      "logits/rejected": -0.893359363079071,
      "logps/chosen": -0.661328136920929,
      "logps/rejected": -1.6886718273162842,
      "loss": 0.8857,
      "nll_loss": 0.841015636920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06613769382238388,
      "rewards/margins": 0.10271759331226349,
      "rewards/rejected": -0.16879883408546448,
      "step": 4340
    },
    {
      "epoch": 0.31674372883824226,
      "grad_norm": 1.9657386303024802,
      "learning_rate": 1.2129568697262454e-06,
      "log_odds_chosen": 1.68994140625,
      "log_odds_ratio": -0.34111326932907104,
      "logits/chosen": -0.9605468511581421,
      "logits/rejected": -0.8470703363418579,
      "logps/chosen": -0.668164074420929,
      "logps/rejected": -1.866796851158142,
      "loss": 0.8917,
      "nll_loss": 0.8833984136581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06678466498851776,
      "rewards/margins": 0.11994628608226776,
      "rewards/rejected": -0.186767578125,
      "step": 4350
    },
    {
      "epoch": 0.31747187534131865,
      "grad_norm": 3.417863502867882,
      "learning_rate": 1.2115650656307653e-06,
      "log_odds_chosen": 1.6335937976837158,
      "log_odds_ratio": -0.36455076932907104,
      "logits/chosen": -1.0027344226837158,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.664843738079071,
      "logps/rejected": -1.7878906726837158,
      "loss": 0.898,
      "nll_loss": 0.762499988079071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06635741889476776,
      "rewards/margins": 0.11236572265625,
      "rewards/rejected": -0.17880859971046448,
      "step": 4360
    },
    {
      "epoch": 0.3182000218443951,
      "grad_norm": 2.504394611462869,
      "learning_rate": 1.210178041632103e-06,
      "log_odds_chosen": 1.4513671398162842,
      "log_odds_ratio": -0.4164062440395355,
      "logits/chosen": -0.9330078363418579,
      "logits/rejected": -0.824023425579071,
      "logps/chosen": -0.6561523675918579,
      "logps/rejected": -1.6628906726837158,
      "loss": 0.8858,
      "nll_loss": 0.832226574420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.065673828125,
      "rewards/margins": 0.100738525390625,
      "rewards/rejected": -0.16628417372703552,
      "step": 4370
    },
    {
      "epoch": 0.3189281683474715,
      "grad_norm": 1.8810634976890226,
      "learning_rate": 1.2087957704309988e-06,
      "log_odds_chosen": 1.55859375,
      "log_odds_ratio": -0.3944091796875,
      "logits/chosen": -0.923046886920929,
      "logits/rejected": -0.8140624761581421,
      "logps/chosen": -0.6456054449081421,
      "logps/rejected": -1.705078125,
      "loss": 0.8665,
      "nll_loss": 0.788867175579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06455077975988388,
      "rewards/margins": 0.10596923530101776,
      "rewards/rejected": -0.17060546576976776,
      "step": 4380
    },
    {
      "epoch": 0.3196563148505479,
      "grad_norm": 2.2102323782714106,
      "learning_rate": 1.2074182249459642e-06,
      "log_odds_chosen": 1.691308617591858,
      "log_odds_ratio": -0.3719726502895355,
      "logits/chosen": -0.9886718988418579,
      "logits/rejected": -0.8648437261581421,
      "logps/chosen": -0.6771484613418579,
      "logps/rejected": -1.8957030773162842,
      "loss": 0.8648,
      "nll_loss": 0.8330078125,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06770019233226776,
      "rewards/margins": 0.12166748195886612,
      "rewards/rejected": -0.1895751953125,
      "step": 4390
    },
    {
      "epoch": 0.32038446135362436,
      "grad_norm": 2.1829900430349154,
      "learning_rate": 1.2060453783110545e-06,
      "log_odds_chosen": 1.3127930164337158,
      "log_odds_ratio": -0.40400391817092896,
      "logits/chosen": -0.968945324420929,
      "logits/rejected": -0.850781261920929,
      "logps/chosen": -0.634570300579071,
      "logps/rejected": -1.5166015625,
      "loss": 0.8658,
      "nll_loss": 0.888476550579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06341552734375,
      "rewards/margins": 0.08817748725414276,
      "rewards/rejected": -0.15166015923023224,
      "step": 4400
    },
    {
      "epoch": 0.32111260785670076,
      "grad_norm": 2.2476756720566375,
      "learning_rate": 1.2046772038736682e-06,
      "log_odds_chosen": 1.4152343273162842,
      "log_odds_ratio": -0.405029296875,
      "logits/chosen": -0.974804699420929,
      "logits/rejected": -0.881640613079071,
      "logps/chosen": -0.6502929925918579,
      "logps/rejected": -1.6064453125,
      "loss": 0.8988,
      "nll_loss": 0.807812511920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06501464545726776,
      "rewards/margins": 0.0955810546875,
      "rewards/rejected": -0.16066893935203552,
      "step": 4410
    },
    {
      "epoch": 0.3218407543597772,
      "grad_norm": 2.0704307264745987,
      "learning_rate": 1.2033136751923736e-06,
      "log_odds_chosen": 1.3499023914337158,
      "log_odds_ratio": -0.4185546934604645,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.873828113079071,
      "logps/chosen": -0.7044922113418579,
      "logps/rejected": -1.6570312976837158,
      "loss": 0.8806,
      "nll_loss": 0.819140613079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07038573920726776,
      "rewards/margins": 0.09517212212085724,
      "rewards/rejected": -0.1656494140625,
      "step": 4420
    },
    {
      "epoch": 0.3225689008628536,
      "grad_norm": 1.9892425531399507,
      "learning_rate": 1.201954766034762e-06,
      "log_odds_chosen": 1.424414038658142,
      "log_odds_ratio": -0.44121092557907104,
      "logits/chosen": -0.9779297113418579,
      "logits/rejected": -0.8753906488418579,
      "logps/chosen": -0.6836913824081421,
      "logps/rejected": -1.696874976158142,
      "loss": 0.8941,
      "nll_loss": 0.821484386920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.068359375,
      "rewards/margins": 0.10122070461511612,
      "rewards/rejected": -0.16960449516773224,
      "step": 4430
    },
    {
      "epoch": 0.32329704736593,
      "grad_norm": 3.6006385258020206,
      "learning_rate": 1.2006004503753285e-06,
      "log_odds_chosen": 1.3595702648162842,
      "log_odds_ratio": -0.4202636778354645,
      "logits/chosen": -0.9095703363418579,
      "logits/rejected": -0.8275390863418579,
      "logps/chosen": -0.675000011920929,
      "logps/rejected": -1.607812523841858,
      "loss": 0.886,
      "nll_loss": 0.888867199420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06749267876148224,
      "rewards/margins": 0.09318237006664276,
      "rewards/rejected": -0.16054686903953552,
      "step": 4440
    },
    {
      "epoch": 0.32402519386900647,
      "grad_norm": 2.5830185875644744,
      "learning_rate": 1.1992507023933782e-06,
      "log_odds_chosen": 1.631445288658142,
      "log_odds_ratio": -0.3603515625,
      "logits/chosen": -0.965039074420929,
      "logits/rejected": -0.8134765625,
      "logps/chosen": -0.667285144329071,
      "logps/rejected": -1.824609398841858,
      "loss": 0.9005,
      "nll_loss": 0.8824218511581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06674804538488388,
      "rewards/margins": 0.11579589545726776,
      "rewards/rejected": -0.18242187798023224,
      "step": 4450
    },
    {
      "epoch": 0.32475334037208287,
      "grad_norm": 2.1135191752351172,
      "learning_rate": 1.1979054964709597e-06,
      "log_odds_chosen": 1.4690673351287842,
      "log_odds_ratio": -0.3809570372104645,
      "logits/chosen": -1.044921875,
      "logits/rejected": -0.838671863079071,
      "logps/chosen": -0.64990234375,
      "logps/rejected": -1.685156226158142,
      "loss": 0.8842,
      "nll_loss": 0.818652331829071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06496582180261612,
      "rewards/margins": 0.10357513278722763,
      "rewards/rejected": -0.16848143935203552,
      "step": 4460
    },
    {
      "epoch": 0.32548148687515926,
      "grad_norm": 2.07056986607828,
      "learning_rate": 1.1965648071908207e-06,
      "log_odds_chosen": 1.7470703125,
      "log_odds_ratio": -0.32958984375,
      "logits/chosen": -1.0134766101837158,
      "logits/rejected": -0.896289050579071,
      "logps/chosen": -0.63916015625,
      "logps/rejected": -1.865625023841858,
      "loss": 0.8803,
      "nll_loss": 0.7881835699081421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06387939304113388,
      "rewards/margins": 0.12250976264476776,
      "rewards/rejected": -0.18635253608226776,
      "step": 4470
    },
    {
      "epoch": 0.3262096333782357,
      "grad_norm": 2.3024637185666488,
      "learning_rate": 1.1952286093343935e-06,
      "log_odds_chosen": 1.3767578601837158,
      "log_odds_ratio": -0.39287108182907104,
      "logits/chosen": -0.9750000238418579,
      "logits/rejected": -0.8423827886581421,
      "logps/chosen": -0.6591796875,
      "logps/rejected": -1.626562476158142,
      "loss": 0.9054,
      "nll_loss": 0.854687511920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06594238430261612,
      "rewards/margins": 0.09675292670726776,
      "rewards/rejected": -0.16276855766773224,
      "step": 4480
    },
    {
      "epoch": 0.3269377798813121,
      "grad_norm": 2.564076321797089,
      "learning_rate": 1.1938968778798005e-06,
      "log_odds_chosen": 1.4123046398162842,
      "log_odds_ratio": -0.429931640625,
      "logits/chosen": -0.9996093511581421,
      "logits/rejected": -0.8746093511581421,
      "logps/chosen": -0.65576171875,
      "logps/rejected": -1.6462891101837158,
      "loss": 0.879,
      "nll_loss": 0.83984375,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06557617336511612,
      "rewards/margins": 0.098968505859375,
      "rewards/rejected": -0.1646728515625,
      "step": 4490
    },
    {
      "epoch": 0.3276659263843885,
      "grad_norm": 2.2782524821209047,
      "learning_rate": 1.1925695879998878e-06,
      "log_odds_chosen": 1.3969237804412842,
      "log_odds_ratio": -0.41254884004592896,
      "logits/chosen": -0.9498046636581421,
      "logits/rejected": -0.798828125,
      "logps/chosen": -0.6552734375,
      "logps/rejected": -1.6296875476837158,
      "loss": 0.8699,
      "nll_loss": 0.939648449420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06550292670726776,
      "rewards/margins": 0.09744872897863388,
      "rewards/rejected": -0.16291503608226776,
      "step": 4500
    },
    {
      "epoch": 0.32839407288746497,
      "grad_norm": 2.4019593561693893,
      "learning_rate": 1.1912467150602794e-06,
      "log_odds_chosen": 1.332421898841858,
      "log_odds_ratio": -0.40092772245407104,
      "logits/chosen": -0.9521484375,
      "logits/rejected": -0.863085925579071,
      "logps/chosen": -0.650585949420929,
      "logps/rejected": -1.593164086341858,
      "loss": 0.8698,
      "nll_loss": 0.786914050579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06497802585363388,
      "rewards/margins": 0.094268798828125,
      "rewards/rejected": -0.15935058891773224,
      "step": 4510
    },
    {
      "epoch": 0.32912221939054137,
      "grad_norm": 2.0853329680026103,
      "learning_rate": 1.189928234617459e-06,
      "log_odds_chosen": 1.3391602039337158,
      "log_odds_ratio": -0.4043945372104645,
      "logits/chosen": -0.936328113079071,
      "logits/rejected": -0.804492175579071,
      "logps/chosen": -0.628710925579071,
      "logps/rejected": -1.5166015625,
      "loss": 0.8857,
      "nll_loss": 0.808789074420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06289062649011612,
      "rewards/margins": 0.08879394829273224,
      "rewards/rejected": -0.15163573622703552,
      "step": 4520
    },
    {
      "epoch": 0.32985036589361777,
      "grad_norm": 2.239330843999101,
      "learning_rate": 1.1886141224168716e-06,
      "log_odds_chosen": 1.5867187976837158,
      "log_odds_ratio": -0.3687500059604645,
      "logits/chosen": -1.0099608898162842,
      "logits/rejected": -0.889453113079071,
      "logps/chosen": -0.593066394329071,
      "logps/rejected": -1.6427733898162842,
      "loss": 0.8563,
      "nll_loss": 0.797656238079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05930175632238388,
      "rewards/margins": 0.10496826469898224,
      "rewards/rejected": -0.164306640625,
      "step": 4530
    },
    {
      "epoch": 0.3305785123966942,
      "grad_norm": 2.4957550857359103,
      "learning_rate": 1.1873043543910495e-06,
      "log_odds_chosen": 1.469335913658142,
      "log_odds_ratio": -0.43779295682907104,
      "logits/chosen": -0.9701172113418579,
      "logits/rejected": -0.846484363079071,
      "logps/chosen": -0.6878906488418579,
      "logps/rejected": -1.716406226158142,
      "loss": 0.8721,
      "nll_loss": 0.8662109375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06878662109375,
      "rewards/margins": 0.10273437201976776,
      "rewards/rejected": -0.171630859375,
      "step": 4540
    },
    {
      "epoch": 0.3313066588997706,
      "grad_norm": 2.038865881363933,
      "learning_rate": 1.1859989066577617e-06,
      "log_odds_chosen": 1.75830078125,
      "log_odds_ratio": -0.35771483182907104,
      "logits/chosen": -0.991992175579071,
      "logits/rejected": -0.852734386920929,
      "logps/chosen": -0.635546863079071,
      "logps/rejected": -1.853906273841858,
      "loss": 0.8895,
      "nll_loss": 0.8270508050918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06352539360523224,
      "rewards/margins": 0.12188720703125,
      "rewards/rejected": -0.1854248046875,
      "step": 4550
    },
    {
      "epoch": 0.3320348054028471,
      "grad_norm": 2.6070418764448737,
      "learning_rate": 1.1846977555181846e-06,
      "log_odds_chosen": 1.376074194908142,
      "log_odds_ratio": -0.4466308653354645,
      "logits/chosen": -1.0263671875,
      "logits/rejected": -0.9052734375,
      "logps/chosen": -0.6991211175918579,
      "logps/rejected": -1.6882812976837158,
      "loss": 0.8794,
      "nll_loss": 0.8667968511581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06978759914636612,
      "rewards/margins": 0.098968505859375,
      "rewards/rejected": -0.16875000298023224,
      "step": 4560
    },
    {
      "epoch": 0.3327629519059235,
      "grad_norm": 2.407773651801329,
      "learning_rate": 1.1834008774550946e-06,
      "log_odds_chosen": 1.5021483898162842,
      "log_odds_ratio": -0.3895507752895355,
      "logits/chosen": -0.941210925579071,
      "logits/rejected": -0.8443359136581421,
      "logps/chosen": -0.6607421636581421,
      "logps/rejected": -1.7197265625,
      "loss": 0.8819,
      "nll_loss": 0.8734375238418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06606445461511612,
      "rewards/margins": 0.10600586235523224,
      "rewards/rejected": -0.17219237983226776,
      "step": 4570
    },
    {
      "epoch": 0.3334910984089999,
      "grad_norm": 2.277772879150597,
      "learning_rate": 1.1821082491310835e-06,
      "log_odds_chosen": 1.3922851085662842,
      "log_odds_ratio": -0.4399169981479645,
      "logits/chosen": -0.9505859613418579,
      "logits/rejected": -0.845703125,
      "logps/chosen": -0.644824206829071,
      "logps/rejected": -1.626367211341858,
      "loss": 0.8681,
      "nll_loss": 0.864062488079071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06452636420726776,
      "rewards/margins": 0.09818725287914276,
      "rewards/rejected": -0.16252441704273224,
      "step": 4580
    },
    {
      "epoch": 0.33421924491207633,
      "grad_norm": 2.3603339414829607,
      "learning_rate": 1.1808198473867937e-06,
      "log_odds_chosen": 1.340673804283142,
      "log_odds_ratio": -0.4244628846645355,
      "logits/chosen": -0.98046875,
      "logits/rejected": -0.87890625,
      "logps/chosen": -0.6871093511581421,
      "logps/rejected": -1.626953125,
      "loss": 0.8486,
      "nll_loss": 0.8720703125,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06873778998851776,
      "rewards/margins": 0.09398803859949112,
      "rewards/rejected": -0.16264648735523224,
      "step": 4590
    },
    {
      "epoch": 0.33494739141515273,
      "grad_norm": 2.5944839025076405,
      "learning_rate": 1.179535649239177e-06,
      "log_odds_chosen": 1.4851562976837158,
      "log_odds_ratio": -0.3910156190395355,
      "logits/chosen": -1.013671875,
      "logits/rejected": -0.896679699420929,
      "logps/chosen": -0.6802734136581421,
      "logps/rejected": -1.7449219226837158,
      "loss": 0.8736,
      "nll_loss": 0.7977539300918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.068115234375,
      "rewards/margins": 0.10650940239429474,
      "rewards/rejected": -0.17453613877296448,
      "step": 4600
    },
    {
      "epoch": 0.3356755379182291,
      "grad_norm": 2.260596482611632,
      "learning_rate": 1.178255631879771e-06,
      "log_odds_chosen": 1.3974120616912842,
      "log_odds_ratio": -0.3992675840854645,
      "logits/chosen": -0.968554675579071,
      "logits/rejected": -0.8482421636581421,
      "logps/chosen": -0.6962890625,
      "logps/rejected": -1.6623046398162842,
      "loss": 0.8749,
      "nll_loss": 0.811328113079071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06964111328125,
      "rewards/margins": 0.09650573879480362,
      "rewards/rejected": -0.16606445610523224,
      "step": 4610
    },
    {
      "epoch": 0.3364036844213056,
      "grad_norm": 1.9536849245962038,
      "learning_rate": 1.1769797726729992e-06,
      "log_odds_chosen": 1.396337866783142,
      "log_odds_ratio": -0.41582030057907104,
      "logits/chosen": -0.973437488079071,
      "logits/rejected": -0.877734363079071,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -1.671484351158142,
      "loss": 0.8526,
      "nll_loss": 0.840624988079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07059326022863388,
      "rewards/margins": 0.09651489555835724,
      "rewards/rejected": -0.16706542670726776,
      "step": 4620
    },
    {
      "epoch": 0.337131830924382,
      "grad_norm": 2.230046565070823,
      "learning_rate": 1.1757080491544881e-06,
      "log_odds_chosen": 1.401757836341858,
      "log_odds_ratio": -0.39111328125,
      "logits/chosen": -0.9691406488418579,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.741992175579071,
      "logps/rejected": -1.6984374523162842,
      "loss": 0.8935,
      "nll_loss": 0.9501953125,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07418213039636612,
      "rewards/margins": 0.09567870944738388,
      "rewards/rejected": -0.16989746689796448,
      "step": 4630
    },
    {
      "epoch": 0.3378599774274584,
      "grad_norm": 1.9072093471572202,
      "learning_rate": 1.1744404390294068e-06,
      "log_odds_chosen": 1.7760741710662842,
      "log_odds_ratio": -0.33867186307907104,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.684374988079071,
      "logps/rejected": -1.946874976158142,
      "loss": 0.8751,
      "nll_loss": 0.841503918170929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06846924126148224,
      "rewards/margins": 0.12633056938648224,
      "rewards/rejected": -0.1947021484375,
      "step": 4640
    },
    {
      "epoch": 0.33858812393053483,
      "grad_norm": 2.9103737377139853,
      "learning_rate": 1.1731769201708264e-06,
      "log_odds_chosen": 1.50244140625,
      "log_odds_ratio": -0.3816894590854645,
      "logits/chosen": -1.046289086341858,
      "logits/rejected": -0.912890613079071,
      "logps/chosen": -0.6953125,
      "logps/rejected": -1.7990233898162842,
      "loss": 0.8579,
      "nll_loss": 0.8062499761581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06950683891773224,
      "rewards/margins": 0.11051025241613388,
      "rewards/rejected": -0.18017578125,
      "step": 4650
    },
    {
      "epoch": 0.33931627043361123,
      "grad_norm": 2.6792034460083145,
      "learning_rate": 1.1719174706180952e-06,
      "log_odds_chosen": 1.474707007408142,
      "log_odds_ratio": -0.43242186307907104,
      "logits/chosen": -0.9800781011581421,
      "logits/rejected": -0.9125000238418579,
      "logps/chosen": -0.6405273675918579,
      "logps/rejected": -1.6671874523162842,
      "loss": 0.9048,
      "nll_loss": 0.78955078125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0640869140625,
      "rewards/margins": 0.10258789360523224,
      "rewards/rejected": -0.16660156846046448,
      "step": 4660
    },
    {
      "epoch": 0.3400444169366877,
      "grad_norm": 2.5399208455884783,
      "learning_rate": 1.1706620685752386e-06,
      "log_odds_chosen": 1.257177710533142,
      "log_odds_ratio": -0.4805664122104645,
      "logits/chosen": -0.956835925579071,
      "logits/rejected": -0.867382824420929,
      "logps/chosen": -0.723828136920929,
      "logps/rejected": -1.591406226158142,
      "loss": 0.8877,
      "nll_loss": 0.84912109375,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07232666015625,
      "rewards/margins": 0.08670349419116974,
      "rewards/rejected": -0.15913085639476776,
      "step": 4670
    },
    {
      "epoch": 0.3407725634397641,
      "grad_norm": 2.047834837396672,
      "learning_rate": 1.1694106924093723e-06,
      "log_odds_chosen": 1.4031250476837158,
      "log_odds_ratio": -0.42241209745407104,
      "logits/chosen": -0.9638671875,
      "logits/rejected": -0.8892577886581421,
      "logps/chosen": -0.6836913824081421,
      "logps/rejected": -1.6593749523162842,
      "loss": 0.8832,
      "nll_loss": 0.8324218988418579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06833495944738388,
      "rewards/margins": 0.09761810302734375,
      "rewards/rejected": -0.16604003310203552,
      "step": 4680
    },
    {
      "epoch": 0.3415007099428405,
      "grad_norm": 2.137081132408231,
      "learning_rate": 1.1681633206491381e-06,
      "log_odds_chosen": 1.425878882408142,
      "log_odds_ratio": -0.4407714903354645,
      "logits/chosen": -0.9945312738418579,
      "logits/rejected": -0.896484375,
      "logps/chosen": -0.6192382574081421,
      "logps/rejected": -1.577539086341858,
      "loss": 0.8742,
      "nll_loss": 0.796679675579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06186523288488388,
      "rewards/margins": 0.09583739936351776,
      "rewards/rejected": -0.15771484375,
      "step": 4690
    },
    {
      "epoch": 0.34222885644591694,
      "grad_norm": 2.268331223595524,
      "learning_rate": 1.1669199319831564e-06,
      "log_odds_chosen": 1.5496094226837158,
      "log_odds_ratio": -0.3848632872104645,
      "logits/chosen": -0.967968761920929,
      "logits/rejected": -0.826367199420929,
      "logps/chosen": -0.67578125,
      "logps/rejected": -1.769921898841858,
      "loss": 0.8945,
      "nll_loss": 0.921093761920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06759033352136612,
      "rewards/margins": 0.10950927436351776,
      "rewards/rejected": -0.17702636122703552,
      "step": 4700
    },
    {
      "epoch": 0.34295700294899334,
      "grad_norm": 2.199788262175794,
      "learning_rate": 1.1656805052584958e-06,
      "log_odds_chosen": 1.501074194908142,
      "log_odds_ratio": -0.3841796815395355,
      "logits/chosen": -0.969921886920929,
      "logits/rejected": -0.8642578125,
      "logps/chosen": -0.658496081829071,
      "logps/rejected": -1.702539086341858,
      "loss": 0.915,
      "nll_loss": 0.903515636920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06584472954273224,
      "rewards/margins": 0.10447998344898224,
      "rewards/rejected": -0.17021484673023224,
      "step": 4710
    },
    {
      "epoch": 0.34368514945206974,
      "grad_norm": 1.9882092965372364,
      "learning_rate": 1.164445019479164e-06,
      "log_odds_chosen": 1.6267578601837158,
      "log_odds_ratio": -0.36640626192092896,
      "logits/chosen": -1.016210913658142,
      "logits/rejected": -0.873046875,
      "logps/chosen": -0.6875,
      "logps/rejected": -1.8671875,
      "loss": 0.8537,
      "nll_loss": 0.8765624761581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06871338188648224,
      "rewards/margins": 0.11800537258386612,
      "rewards/rejected": -0.18671874701976776,
      "step": 4720
    },
    {
      "epoch": 0.3444132959551462,
      "grad_norm": 2.354587239165765,
      "learning_rate": 1.1632134538046105e-06,
      "log_odds_chosen": 1.417626976966858,
      "log_odds_ratio": -0.4208984375,
      "logits/chosen": -0.9916015863418579,
      "logits/rejected": -0.8705078363418579,
      "logps/chosen": -0.710156261920929,
      "logps/rejected": -1.7265625,
      "loss": 0.8859,
      "nll_loss": 0.879101574420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07102050632238388,
      "rewards/margins": 0.10151366889476776,
      "rewards/rejected": -0.17268066108226776,
      "step": 4730
    },
    {
      "epoch": 0.3451414424582226,
      "grad_norm": 1.9464886596116469,
      "learning_rate": 1.1619857875482536e-06,
      "log_odds_chosen": 1.6466796398162842,
      "log_odds_ratio": -0.33378905057907104,
      "logits/chosen": -1.005859375,
      "logits/rejected": -0.8714843988418579,
      "logps/chosen": -0.6234375238418579,
      "logps/rejected": -1.748046875,
      "loss": 0.8798,
      "nll_loss": 0.804882824420929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06234131008386612,
      "rewards/margins": 0.11249999701976776,
      "rewards/rejected": -0.17478027939796448,
      "step": 4740
    },
    {
      "epoch": 0.345869588961299,
      "grad_norm": 3.075646143944494,
      "learning_rate": 1.1607620001760185e-06,
      "log_odds_chosen": 1.3464844226837158,
      "log_odds_ratio": -0.45014649629592896,
      "logits/chosen": -1.003320336341858,
      "logits/rejected": -0.9019531011581421,
      "logps/chosen": -0.678906261920929,
      "logps/rejected": -1.6335937976837158,
      "loss": 0.868,
      "nll_loss": 0.8482421636581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06790771335363388,
      "rewards/margins": 0.09535522758960724,
      "rewards/rejected": -0.16337890923023224,
      "step": 4750
    },
    {
      "epoch": 0.34659773546437544,
      "grad_norm": 2.59789663066583,
      "learning_rate": 1.1595420713048968e-06,
      "log_odds_chosen": 1.5583984851837158,
      "log_odds_ratio": -0.3541503846645355,
      "logits/chosen": -1.022851586341858,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.631152331829071,
      "logps/rejected": -1.6746094226837158,
      "loss": 0.8678,
      "nll_loss": 0.8160156011581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06315918266773224,
      "rewards/margins": 0.10427246242761612,
      "rewards/rejected": -0.16745606064796448,
      "step": 4760
    },
    {
      "epoch": 0.34732588196745184,
      "grad_norm": 2.225948237085341,
      "learning_rate": 1.1583259807015182e-06,
      "log_odds_chosen": 1.2248046398162842,
      "log_odds_ratio": -0.44438475370407104,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.884960949420929,
      "logps/chosen": -0.646191418170929,
      "logps/rejected": -1.4443359375,
      "loss": 0.872,
      "nll_loss": 0.8084961175918579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06462402641773224,
      "rewards/margins": 0.07988281548023224,
      "rewards/rejected": -0.14439697563648224,
      "step": 4770
    },
    {
      "epoch": 0.34805402847052824,
      "grad_norm": 2.128644866021711,
      "learning_rate": 1.1571137082807434e-06,
      "log_odds_chosen": 1.5554687976837158,
      "log_odds_ratio": -0.4063964784145355,
      "logits/chosen": -0.976367175579071,
      "logits/rejected": -0.866406261920929,
      "logps/chosen": -0.6356445550918579,
      "logps/rejected": -1.732812523841858,
      "loss": 0.8591,
      "nll_loss": 0.792187511920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06363525241613388,
      "rewards/margins": 0.10974731296300888,
      "rewards/rejected": -0.17331543564796448,
      "step": 4780
    },
    {
      "epoch": 0.3487821749736047,
      "grad_norm": 2.3594232231758054,
      "learning_rate": 1.155905234104269e-06,
      "log_odds_chosen": 1.499230980873108,
      "log_odds_ratio": -0.3814697265625,
      "logits/chosen": -0.9603515863418579,
      "logits/rejected": -0.8681640625,
      "logps/chosen": -0.676953136920929,
      "logps/rejected": -1.678125023841858,
      "loss": 0.8651,
      "nll_loss": 0.8160156011581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06767578423023224,
      "rewards/margins": 0.10012435913085938,
      "rewards/rejected": -0.16779784858226776,
      "step": 4790
    },
    {
      "epoch": 0.3495103214766811,
      "grad_norm": 2.3491760577048253,
      "learning_rate": 1.1547005383792516e-06,
      "log_odds_chosen": 1.634124755859375,
      "log_odds_ratio": -0.330810546875,
      "logits/chosen": -0.9574218988418579,
      "logits/rejected": -0.840624988079071,
      "logps/chosen": -0.645800769329071,
      "logps/rejected": -1.775390625,
      "loss": 0.8776,
      "nll_loss": 0.8687499761581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06458739936351776,
      "rewards/margins": 0.113037109375,
      "rewards/rejected": -0.17751464247703552,
      "step": 4800
    },
    {
      "epoch": 0.35023846797975755,
      "grad_norm": 1.973863230179253,
      "learning_rate": 1.1534996014569446e-06,
      "log_odds_chosen": 1.4659912586212158,
      "log_odds_ratio": -0.38203126192092896,
      "logits/chosen": -1.0087890625,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -0.6405273675918579,
      "logps/rejected": -1.6513671875,
      "loss": 0.8743,
      "nll_loss": 0.793749988079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06407471001148224,
      "rewards/margins": 0.101226806640625,
      "rewards/rejected": -0.165283203125,
      "step": 4810
    },
    {
      "epoch": 0.35096661448283395,
      "grad_norm": 2.0577446395614634,
      "learning_rate": 1.1523024038313547e-06,
      "log_odds_chosen": 1.294921875,
      "log_odds_ratio": -0.435791015625,
      "logits/chosen": -0.9732421636581421,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.6376953125,
      "logps/rejected": -1.5041015148162842,
      "loss": 0.8745,
      "nll_loss": 0.828320324420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06373290717601776,
      "rewards/margins": 0.08674316108226776,
      "rewards/rejected": -0.15046386420726776,
      "step": 4820
    },
    {
      "epoch": 0.35169476098591035,
      "grad_norm": 2.6926917489276163,
      "learning_rate": 1.1511089261379083e-06,
      "log_odds_chosen": 1.4591796398162842,
      "log_odds_ratio": -0.39726561307907104,
      "logits/chosen": -1.0671875476837158,
      "logits/rejected": -0.907031238079071,
      "logps/chosen": -0.7035156488418579,
      "logps/rejected": -1.751562476158142,
      "loss": 0.8633,
      "nll_loss": 0.8384765386581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07032470405101776,
      "rewards/margins": 0.1048583984375,
      "rewards/rejected": -0.17524413764476776,
      "step": 4830
    },
    {
      "epoch": 0.3524229074889868,
      "grad_norm": 2.291019400783551,
      "learning_rate": 1.149919149152138e-06,
      "log_odds_chosen": 1.4088866710662842,
      "log_odds_ratio": -0.4153808653354645,
      "logits/chosen": -0.9287109375,
      "logits/rejected": -0.84375,
      "logps/chosen": -0.645703136920929,
      "logps/rejected": -1.598046898841858,
      "loss": 0.863,
      "nll_loss": 0.804492175579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06461181491613388,
      "rewards/margins": 0.09534911811351776,
      "rewards/rejected": -0.15986327826976776,
      "step": 4840
    },
    {
      "epoch": 0.3531510539920632,
      "grad_norm": 2.2606220666895687,
      "learning_rate": 1.148733053788381e-06,
      "log_odds_chosen": 1.420800805091858,
      "log_odds_ratio": -0.3935546875,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.8619140386581421,
      "logps/chosen": -0.6590820550918579,
      "logps/rejected": -1.666406273841858,
      "loss": 0.8788,
      "nll_loss": 0.817187488079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06586913764476776,
      "rewards/margins": 0.10074005275964737,
      "rewards/rejected": -0.16667480766773224,
      "step": 4850
    },
    {
      "epoch": 0.3538792004951396,
      "grad_norm": 2.3504972713576477,
      "learning_rate": 1.1475506210984938e-06,
      "log_odds_chosen": 1.300878882408142,
      "log_odds_ratio": -0.45026856660842896,
      "logits/chosen": -1.018945336341858,
      "logits/rejected": -0.9291015863418579,
      "logps/chosen": -0.6982421875,
      "logps/rejected": -1.591406226158142,
      "loss": 0.8908,
      "nll_loss": 0.865039050579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06978759914636612,
      "rewards/margins": 0.0892486572265625,
      "rewards/rejected": -0.1590576171875,
      "step": 4860
    },
    {
      "epoch": 0.35460734699821606,
      "grad_norm": 2.2880743419705962,
      "learning_rate": 1.1463718322705807e-06,
      "log_odds_chosen": 0.985644519329071,
      "log_odds_ratio": -0.53369140625,
      "logits/chosen": -0.959765613079071,
      "logits/rejected": -0.8207031488418579,
      "logps/chosen": -0.722460925579071,
      "logps/rejected": -1.4189453125,
      "loss": 0.8452,
      "nll_loss": 0.8466796875,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07225342094898224,
      "rewards/margins": 0.06957397609949112,
      "rewards/rejected": -0.14189453423023224,
      "step": 4870
    },
    {
      "epoch": 0.35533549350129245,
      "grad_norm": 2.8795386427103398,
      "learning_rate": 1.1451966686277364e-06,
      "log_odds_chosen": 1.190820336341858,
      "log_odds_ratio": -0.45061033964157104,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.883984386920929,
      "logps/chosen": -0.6895507574081421,
      "logps/rejected": -1.504296898841858,
      "loss": 0.8681,
      "nll_loss": 0.8501952886581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0689697265625,
      "rewards/margins": 0.08150024712085724,
      "rewards/rejected": -0.15048828721046448,
      "step": 4880
    },
    {
      "epoch": 0.35606364000436885,
      "grad_norm": 1.9986231423832015,
      "learning_rate": 1.1440251116268034e-06,
      "log_odds_chosen": 1.464746117591858,
      "log_odds_ratio": -0.40302735567092896,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.8837890625,
      "logps/chosen": -0.656933605670929,
      "logps/rejected": -1.705468773841858,
      "loss": 0.8518,
      "nll_loss": 0.8050781488418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06569824367761612,
      "rewards/margins": 0.10481567680835724,
      "rewards/rejected": -0.17043456435203552,
      "step": 4890
    },
    {
      "epoch": 0.3567917865074453,
      "grad_norm": 2.6600615434643466,
      "learning_rate": 1.1428571428571428e-06,
      "log_odds_chosen": 1.7117187976837158,
      "log_odds_ratio": -0.3340820372104645,
      "logits/chosen": -0.989062488079071,
      "logits/rejected": -0.879101574420929,
      "logps/chosen": -0.610546886920929,
      "logps/rejected": -1.823828101158142,
      "loss": 0.8539,
      "nll_loss": 0.7662109136581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06103515625,
      "rewards/margins": 0.121337890625,
      "rewards/rejected": -0.18256835639476776,
      "step": 4900
    },
    {
      "epoch": 0.3575199330105217,
      "grad_norm": 2.214181737715167,
      "learning_rate": 1.14169274403942e-06,
      "log_odds_chosen": 1.3644530773162842,
      "log_odds_ratio": -0.4229492247104645,
      "logits/chosen": -0.9537109136581421,
      "logits/rejected": -0.858593761920929,
      "logps/chosen": -0.6826171875,
      "logps/rejected": -1.6257812976837158,
      "loss": 0.8884,
      "nll_loss": 0.8960937261581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06820068508386612,
      "rewards/margins": 0.0943603515625,
      "rewards/rejected": -0.16249999403953552,
      "step": 4910
    },
    {
      "epoch": 0.35824807951359816,
      "grad_norm": 2.0686486680448475,
      "learning_rate": 1.140531897024402e-06,
      "log_odds_chosen": 1.179235816001892,
      "log_odds_ratio": -0.46845704317092896,
      "logits/chosen": -1.0265624523162842,
      "logits/rejected": -0.951171875,
      "logps/chosen": -0.6571289300918579,
      "logps/rejected": -1.465234398841858,
      "loss": 0.8642,
      "nll_loss": 0.830859363079071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06571044772863388,
      "rewards/margins": 0.08081817626953125,
      "rewards/rejected": -0.14655761420726776,
      "step": 4920
    },
    {
      "epoch": 0.35897622601667456,
      "grad_norm": 2.1231901713936514,
      "learning_rate": 1.13937458379177e-06,
      "log_odds_chosen": 1.32666015625,
      "log_odds_ratio": -0.4737792909145355,
      "logits/chosen": -0.9955078363418579,
      "logits/rejected": -0.8853515386581421,
      "logps/chosen": -0.741406261920929,
      "logps/rejected": -1.7136719226837158,
      "loss": 0.8584,
      "nll_loss": 0.8648437261581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07415771484375,
      "rewards/margins": 0.09732665866613388,
      "rewards/rejected": -0.17141112685203552,
      "step": 4930
    },
    {
      "epoch": 0.35970437251975096,
      "grad_norm": 2.7596862788320116,
      "learning_rate": 1.1382207864489444e-06,
      "log_odds_chosen": 1.460546851158142,
      "log_odds_ratio": -0.37553709745407104,
      "logits/chosen": -0.9556640386581421,
      "logits/rejected": -0.8578125238418579,
      "logps/chosen": -0.6451171636581421,
      "logps/rejected": -1.633203148841858,
      "loss": 0.8596,
      "nll_loss": 0.82568359375,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06451416015625,
      "rewards/margins": 0.09881591796875,
      "rewards/rejected": -0.16323241591453552,
      "step": 4940
    },
    {
      "epoch": 0.3604325190228274,
      "grad_norm": 2.521389359075402,
      "learning_rate": 1.1370704872299223e-06,
      "log_odds_chosen": 1.4609375,
      "log_odds_ratio": -0.4232421815395355,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.9105468988418579,
      "logps/chosen": -0.678417980670929,
      "logps/rejected": -1.701562523841858,
      "loss": 0.8595,
      "nll_loss": 0.8326171636581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06784667819738388,
      "rewards/margins": 0.10222168266773224,
      "rewards/rejected": -0.16994628310203552,
      "step": 4950
    },
    {
      "epoch": 0.3611606655259038,
      "grad_norm": 2.584395192815235,
      "learning_rate": 1.1359236684941295e-06,
      "log_odds_chosen": 1.677636742591858,
      "log_odds_ratio": -0.35871583223342896,
      "logits/chosen": -0.928906261920929,
      "logits/rejected": -0.8324218988418579,
      "logps/chosen": -0.6353515386581421,
      "logps/rejected": -1.790429711341858,
      "loss": 0.8568,
      "nll_loss": 0.818164050579071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06345214694738388,
      "rewards/margins": 0.11562804877758026,
      "rewards/rejected": -0.17917481064796448,
      "step": 4960
    },
    {
      "epoch": 0.3618888120289802,
      "grad_norm": 2.27351688374288,
      "learning_rate": 1.1347803127252839e-06,
      "log_odds_chosen": 1.357031226158142,
      "log_odds_ratio": -0.4119628965854645,
      "logits/chosen": -0.9048827886581421,
      "logits/rejected": -0.831250011920929,
      "logps/chosen": -0.6314452886581421,
      "logps/rejected": -1.5712890625,
      "loss": 0.8505,
      "nll_loss": 0.792187511920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06313476711511612,
      "rewards/margins": 0.09393920749425888,
      "rewards/rejected": -0.1571044921875,
      "step": 4970
    },
    {
      "epoch": 0.36261695853205667,
      "grad_norm": 2.7553717501760637,
      "learning_rate": 1.1336404025302715e-06,
      "log_odds_chosen": 1.403417944908142,
      "log_odds_ratio": -0.4190917909145355,
      "logits/chosen": -0.9869140386581421,
      "logits/rejected": -0.848828136920929,
      "logps/chosen": -0.7333008050918579,
      "logps/rejected": -1.7468750476837158,
      "loss": 0.8738,
      "nll_loss": 0.808789074420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07329101860523224,
      "rewards/margins": 0.10150146484375,
      "rewards/rejected": -0.1746826171875,
      "step": 4980
    },
    {
      "epoch": 0.36334510503513306,
      "grad_norm": 2.0247286834775333,
      "learning_rate": 1.1325039206380352e-06,
      "log_odds_chosen": 1.5318725109100342,
      "log_odds_ratio": -0.402587890625,
      "logits/chosen": -0.990429699420929,
      "logits/rejected": -0.902539074420929,
      "logps/chosen": -0.64697265625,
      "logps/rejected": -1.6769530773162842,
      "loss": 0.8402,
      "nll_loss": 0.785351574420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.064697265625,
      "rewards/margins": 0.10296936333179474,
      "rewards/rejected": -0.16777344048023224,
      "step": 4990
    },
    {
      "epoch": 0.36407325153820946,
      "grad_norm": 2.926531941062753,
      "learning_rate": 1.131370849898476e-06,
      "log_odds_chosen": 1.635156273841858,
      "log_odds_ratio": -0.3763183653354645,
      "logits/chosen": -0.989453136920929,
      "logits/rejected": -0.8794921636581421,
      "logps/chosen": -0.680957019329071,
      "logps/rejected": -1.8406250476837158,
      "loss": 0.8686,
      "nll_loss": 0.830078125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06817626953125,
      "rewards/margins": 0.115966796875,
      "rewards/rejected": -0.18410643935203552,
      "step": 5000
    },
    {
      "epoch": 0.36407325153820946,
      "eval_log_odds_chosen": 1.1701252460479736,
      "eval_log_odds_ratio": -0.47759079933166504,
      "eval_logits/chosen": -0.8645420074462891,
      "eval_logits/rejected": -0.7723644971847534,
      "eval_logps/chosen": -0.7588062882423401,
      "eval_logps/rejected": -1.6227697134017944,
      "eval_loss": 1.188714861869812,
      "eval_nll_loss": 1.136938452720642,
      "eval_rewards/accuracies": 0.7338129281997681,
      "eval_rewards/chosen": -0.07588158547878265,
      "eval_rewards/margins": 0.08640365302562714,
      "eval_rewards/rejected": -0.16227255761623383,
      "eval_runtime": 1383.4765,
      "eval_samples_per_second": 70.728,
      "eval_steps_per_second": 1.105,
      "step": 5000
    },
    {
      "epoch": 0.3648013980412859,
      "grad_norm": 2.3535375170036317,
      "learning_rate": 1.130241173281366e-06,
      "log_odds_chosen": 1.64990234375,
      "log_odds_ratio": -0.333984375,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.8388671875,
      "logps/chosen": -0.6468750238418579,
      "logps/rejected": -1.8107421398162842,
      "loss": 0.8539,
      "nll_loss": 0.822265625,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06473388522863388,
      "rewards/margins": 0.1165771484375,
      "rewards/rejected": -0.18120117485523224,
      "step": 5010
    },
    {
      "epoch": 0.3655295445443623,
      "grad_norm": 2.44878220125548,
      "learning_rate": 1.1291148738752732e-06,
      "log_odds_chosen": 1.5876953601837158,
      "log_odds_ratio": -0.3704589903354645,
      "logits/chosen": -0.9789062738418579,
      "logits/rejected": -0.872851550579071,
      "logps/chosen": -0.6705077886581421,
      "logps/rejected": -1.790624976158142,
      "loss": 0.8625,
      "nll_loss": 0.8013671636581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06706543266773224,
      "rewards/margins": 0.11213378608226776,
      "rewards/rejected": -0.17900390923023224,
      "step": 5020
    },
    {
      "epoch": 0.36625769104743877,
      "grad_norm": 2.2139248309326556,
      "learning_rate": 1.1279919348864981e-06,
      "log_odds_chosen": 1.2345092296600342,
      "log_odds_ratio": -0.42353516817092896,
      "logits/chosen": -0.9994140863418579,
      "logits/rejected": -0.896484375,
      "logps/chosen": -0.641796886920929,
      "logps/rejected": -1.4910156726837158,
      "loss": 0.8778,
      "nll_loss": 0.8443359136581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06418456882238388,
      "rewards/margins": 0.08502807468175888,
      "rewards/rejected": -0.14912109076976776,
      "step": 5030
    },
    {
      "epoch": 0.36698583755051517,
      "grad_norm": 2.654248476639483,
      "learning_rate": 1.126872339638022e-06,
      "log_odds_chosen": 1.370507836341858,
      "log_odds_ratio": -0.44648438692092896,
      "logits/chosen": -0.978515625,
      "logits/rejected": -0.8236328363418579,
      "logps/chosen": -0.673632800579071,
      "logps/rejected": -1.6650390625,
      "loss": 0.8513,
      "nll_loss": 0.8384765386581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06735839694738388,
      "rewards/margins": 0.09934081882238388,
      "rewards/rejected": -0.16655273735523224,
      "step": 5040
    },
    {
      "epoch": 0.36771398405359157,
      "grad_norm": 2.3372480313333694,
      "learning_rate": 1.1257560715684668e-06,
      "log_odds_chosen": 1.3493163585662842,
      "log_odds_ratio": -0.40576171875,
      "logits/chosen": -0.9833984375,
      "logits/rejected": -0.915234386920929,
      "logps/chosen": -0.6495116949081421,
      "logps/rejected": -1.572656273841858,
      "loss": 0.8457,
      "nll_loss": 0.7876952886581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06496582180261612,
      "rewards/margins": 0.09226684272289276,
      "rewards/rejected": -0.15720215439796448,
      "step": 5050
    },
    {
      "epoch": 0.368442130556668,
      "grad_norm": 1.977211344779332,
      "learning_rate": 1.1246431142310665e-06,
      "log_odds_chosen": 1.3971679210662842,
      "log_odds_ratio": -0.4151367247104645,
      "logits/chosen": -1.045507788658142,
      "logits/rejected": -0.9369140863418579,
      "logps/chosen": -0.6903320550918579,
      "logps/rejected": -1.644921898841858,
      "loss": 0.8586,
      "nll_loss": 0.785351574420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06906738132238388,
      "rewards/margins": 0.09538574516773224,
      "rewards/rejected": -0.16447754204273224,
      "step": 5060
    },
    {
      "epoch": 0.3691702770597444,
      "grad_norm": 2.177191359912977,
      "learning_rate": 1.1235334512926484e-06,
      "log_odds_chosen": 1.4939453601837158,
      "log_odds_ratio": -0.42192381620407104,
      "logits/chosen": -0.9603515863418579,
      "logits/rejected": -0.855273425579071,
      "logps/chosen": -0.681835949420929,
      "logps/rejected": -1.7644531726837158,
      "loss": 0.8598,
      "nll_loss": 0.8919922113418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06812743842601776,
      "rewards/margins": 0.108245849609375,
      "rewards/rejected": -0.17644043266773224,
      "step": 5070
    },
    {
      "epoch": 0.3698984235628208,
      "grad_norm": 2.5153858377715475,
      "learning_rate": 1.1224270665326274e-06,
      "log_odds_chosen": 1.3351562023162842,
      "log_odds_ratio": -0.4427734315395355,
      "logits/chosen": -0.998046875,
      "logits/rejected": -0.8978515863418579,
      "logps/chosen": -0.705371081829071,
      "logps/rejected": -1.637109398841858,
      "loss": 0.8531,
      "nll_loss": 0.8583984375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07048340141773224,
      "rewards/margins": 0.09320831298828125,
      "rewards/rejected": -0.16374512016773224,
      "step": 5080
    },
    {
      "epoch": 0.3706265700658973,
      "grad_norm": 2.051676711874061,
      "learning_rate": 1.12132394384201e-06,
      "log_odds_chosen": 1.244482398033142,
      "log_odds_ratio": -0.4573730528354645,
      "logits/chosen": -1.0001952648162842,
      "logits/rejected": -0.9126952886581421,
      "logps/chosen": -0.69580078125,
      "logps/rejected": -1.5212891101837158,
      "loss": 0.8718,
      "nll_loss": 0.8037109375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06954345852136612,
      "rewards/margins": 0.08260498195886612,
      "rewards/rejected": -0.15212401747703552,
      "step": 5090
    },
    {
      "epoch": 0.3713547165689737,
      "grad_norm": 2.781487063599797,
      "learning_rate": 1.1202240672224076e-06,
      "log_odds_chosen": 1.4113280773162842,
      "log_odds_ratio": -0.4249023497104645,
      "logits/chosen": -0.9564453363418579,
      "logits/rejected": -0.840039074420929,
      "logps/chosen": -0.6839843988418579,
      "logps/rejected": -1.6921875476837158,
      "loss": 0.841,
      "nll_loss": 0.813281238079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06838379055261612,
      "rewards/margins": 0.10087890923023224,
      "rewards/rejected": -0.16926269233226776,
      "step": 5100
    },
    {
      "epoch": 0.3720828630720501,
      "grad_norm": 2.2563448081551405,
      "learning_rate": 1.1191274207850654e-06,
      "log_odds_chosen": 1.7119140625,
      "log_odds_ratio": -0.3570800721645355,
      "logits/chosen": -1.005859375,
      "logits/rejected": -0.891796886920929,
      "logps/chosen": -0.6742187738418579,
      "logps/rejected": -1.878515601158142,
      "loss": 0.8547,
      "nll_loss": 0.813281238079071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06746826320886612,
      "rewards/margins": 0.1204833984375,
      "rewards/rejected": -0.18789061903953552,
      "step": 5110
    },
    {
      "epoch": 0.37281100957512653,
      "grad_norm": 2.4284409653259384,
      "learning_rate": 1.1180339887498948e-06,
      "log_odds_chosen": 1.558203101158142,
      "log_odds_ratio": -0.3634277284145355,
      "logits/chosen": -0.9546874761581421,
      "logits/rejected": -0.8599609136581421,
      "logps/chosen": -0.6377929449081421,
      "logps/rejected": -1.7236328125,
      "loss": 0.8606,
      "nll_loss": 0.8526366949081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06383056938648224,
      "rewards/margins": 0.10850830376148224,
      "rewards/rejected": -0.1724853515625,
      "step": 5120
    },
    {
      "epoch": 0.3735391560782029,
      "grad_norm": 2.6812582174843387,
      "learning_rate": 1.1169437554445213e-06,
      "log_odds_chosen": 1.698339819908142,
      "log_odds_ratio": -0.3580566346645355,
      "logits/chosen": -1.0115234851837158,
      "logits/rejected": -0.880664050579071,
      "logps/chosen": -0.6419922113418579,
      "logps/rejected": -1.8849608898162842,
      "loss": 0.8489,
      "nll_loss": 0.7813476324081421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06422118842601776,
      "rewards/margins": 0.12435302883386612,
      "rewards/rejected": -0.18867187201976776,
      "step": 5130
    },
    {
      "epoch": 0.3742673025812793,
      "grad_norm": 2.4862136382664404,
      "learning_rate": 1.1158567053033413e-06,
      "log_odds_chosen": 1.4710571765899658,
      "log_odds_ratio": -0.3946289122104645,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.8638671636581421,
      "logps/chosen": -0.6792968511581421,
      "logps/rejected": -1.732031226158142,
      "loss": 0.8784,
      "nll_loss": 0.837109386920929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06791992485523224,
      "rewards/margins": 0.10528869926929474,
      "rewards/rejected": -0.1732177734375,
      "step": 5140
    },
    {
      "epoch": 0.3749954490843558,
      "grad_norm": 2.0383397149277473,
      "learning_rate": 1.1147728228665882e-06,
      "log_odds_chosen": 1.587890625,
      "log_odds_ratio": -0.43159180879592896,
      "logits/chosen": -1.0060546398162842,
      "logits/rejected": -0.920117199420929,
      "logps/chosen": -0.7032226324081421,
      "logps/rejected": -1.853124976158142,
      "loss": 0.8451,
      "nll_loss": 0.916015625,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07028808444738388,
      "rewards/margins": 0.11504516750574112,
      "rewards/rejected": -0.18552246689796448,
      "step": 5150
    },
    {
      "epoch": 0.3757235955874322,
      "grad_norm": 2.0202737863307325,
      "learning_rate": 1.1136920927794092e-06,
      "log_odds_chosen": 1.4416015148162842,
      "log_odds_ratio": -0.3780273497104645,
      "logits/chosen": -1.005468726158142,
      "logits/rejected": -0.9097656011581421,
      "logps/chosen": -0.7099609375,
      "logps/rejected": -1.717187523841858,
      "loss": 0.8413,
      "nll_loss": 0.797070324420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07099609076976776,
      "rewards/margins": 0.10081176459789276,
      "rewards/rejected": -0.17180176079273224,
      "step": 5160
    },
    {
      "epoch": 0.37645174209050863,
      "grad_norm": 2.075554566608684,
      "learning_rate": 1.1126144997909508e-06,
      "log_odds_chosen": 1.422265648841858,
      "log_odds_ratio": -0.45966798067092896,
      "logits/chosen": -1.0177733898162842,
      "logits/rejected": -0.850390613079071,
      "logps/chosen": -0.714648425579071,
      "logps/rejected": -1.7820312976837158,
      "loss": 0.8392,
      "nll_loss": 0.7748047113418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.0714111328125,
      "rewards/margins": 0.10695800930261612,
      "rewards/rejected": -0.17822265625,
      "step": 5170
    },
    {
      "epoch": 0.37717988859358503,
      "grad_norm": 2.409387075944228,
      "learning_rate": 1.1115400287534568e-06,
      "log_odds_chosen": 1.501074194908142,
      "log_odds_ratio": -0.40776365995407104,
      "logits/chosen": -0.9671875238418579,
      "logits/rejected": -0.866015613079071,
      "logps/chosen": -0.6753906011581421,
      "logps/rejected": -1.717187523841858,
      "loss": 0.8506,
      "nll_loss": 0.787890613079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06751708686351776,
      "rewards/margins": 0.10436401516199112,
      "rewards/rejected": -0.17185059189796448,
      "step": 5180
    },
    {
      "epoch": 0.37790803509666143,
      "grad_norm": 2.3856835923912136,
      "learning_rate": 1.110468664621372e-06,
      "log_odds_chosen": 1.381250023841858,
      "log_odds_ratio": -0.4151855409145355,
      "logits/chosen": -0.9605468511581421,
      "logits/rejected": -0.857617199420929,
      "logps/chosen": -0.699023425579071,
      "logps/rejected": -1.6902344226837158,
      "loss": 0.8474,
      "nll_loss": 0.8355468511581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06988525390625,
      "rewards/margins": 0.099334716796875,
      "rewards/rejected": -0.16914062201976776,
      "step": 5190
    },
    {
      "epoch": 0.3786361815997379,
      "grad_norm": 2.1466138578300913,
      "learning_rate": 1.1094003924504583e-06,
      "log_odds_chosen": 1.5236327648162842,
      "log_odds_ratio": -0.40576171875,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.8783203363418579,
      "logps/chosen": -0.6722656488418579,
      "logps/rejected": -1.767578125,
      "loss": 0.8734,
      "nll_loss": 0.8359375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06724853813648224,
      "rewards/margins": 0.1094970703125,
      "rewards/rejected": -0.17680664360523224,
      "step": 5200
    },
    {
      "epoch": 0.3793643281028143,
      "grad_norm": 2.164557343261468,
      "learning_rate": 1.1083351973969191e-06,
      "log_odds_chosen": 1.773046851158142,
      "log_odds_ratio": -0.3685546815395355,
      "logits/chosen": -1.0167968273162842,
      "logits/rejected": -0.897265613079071,
      "logps/chosen": -0.7002929449081421,
      "logps/rejected": -1.9738280773162842,
      "loss": 0.8319,
      "nll_loss": 0.8173828125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06999512016773224,
      "rewards/margins": 0.12736816704273224,
      "rewards/rejected": -0.197265625,
      "step": 5210
    },
    {
      "epoch": 0.3800924746058907,
      "grad_norm": 2.0634713896027344,
      "learning_rate": 1.107273064716533e-06,
      "log_odds_chosen": 1.5671875476837158,
      "log_odds_ratio": -0.3814330995082855,
      "logits/chosen": -0.9828125238418579,
      "logits/rejected": -0.877734363079071,
      "logps/chosen": -0.66064453125,
      "logps/rejected": -1.728124976158142,
      "loss": 0.8351,
      "nll_loss": 0.8119140863418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06601562350988388,
      "rewards/margins": 0.10691528022289276,
      "rewards/rejected": -0.17280273139476776,
      "step": 5220
    },
    {
      "epoch": 0.38082062110896714,
      "grad_norm": 2.2451371934051543,
      "learning_rate": 1.1062139797637962e-06,
      "log_odds_chosen": 1.5558593273162842,
      "log_odds_ratio": -0.36680907011032104,
      "logits/chosen": -0.9755859375,
      "logits/rejected": -0.856640636920929,
      "logps/chosen": -0.636425793170929,
      "logps/rejected": -1.7537109851837158,
      "loss": 0.8673,
      "nll_loss": 0.750781238079071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06363525241613388,
      "rewards/margins": 0.11171875149011612,
      "rewards/rejected": -0.17543944716453552,
      "step": 5230
    },
    {
      "epoch": 0.38154876761204354,
      "grad_norm": 2.2535092391384195,
      "learning_rate": 1.1051579279910751e-06,
      "log_odds_chosen": 1.4794921875,
      "log_odds_ratio": -0.389892578125,
      "logits/chosen": -0.988085925579071,
      "logits/rejected": -0.851757824420929,
      "logps/chosen": -0.6246093511581421,
      "logps/rejected": -1.646484375,
      "loss": 0.8478,
      "nll_loss": 0.8056640625,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06253661960363388,
      "rewards/margins": 0.1021728515625,
      "rewards/rejected": -0.16459961235523224,
      "step": 5240
    },
    {
      "epoch": 0.38227691411511994,
      "grad_norm": 2.630002270859958,
      "learning_rate": 1.1041048949477667e-06,
      "log_odds_chosen": 1.4599609375,
      "log_odds_ratio": -0.39641112089157104,
      "logits/chosen": -1.0310547351837158,
      "logits/rejected": -0.9185546636581421,
      "logps/chosen": -0.62890625,
      "logps/rejected": -1.628515601158142,
      "loss": 0.8349,
      "nll_loss": 0.80126953125,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06291504204273224,
      "rewards/margins": 0.09992675483226776,
      "rewards/rejected": -0.16291503608226776,
      "step": 5250
    },
    {
      "epoch": 0.3830050606181964,
      "grad_norm": 2.4228796681648843,
      "learning_rate": 1.1030548662794673e-06,
      "log_odds_chosen": 1.704687476158142,
      "log_odds_ratio": -0.346923828125,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.8880859613418579,
      "logps/chosen": -0.6083008050918579,
      "logps/rejected": -1.7960937023162842,
      "loss": 0.8486,
      "nll_loss": 0.7333984375,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06081543117761612,
      "rewards/margins": 0.11872558295726776,
      "rewards/rejected": -0.17958983778953552,
      "step": 5260
    },
    {
      "epoch": 0.3837332071212728,
      "grad_norm": 3.7688205117048783,
      "learning_rate": 1.102007827727152e-06,
      "log_odds_chosen": 1.723242163658142,
      "log_odds_ratio": -0.38530272245407104,
      "logits/chosen": -1.012109398841858,
      "logits/rejected": -0.917773425579071,
      "logps/chosen": -0.607128918170929,
      "logps/rejected": -1.82421875,
      "loss": 0.8531,
      "nll_loss": 0.8080078363418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06071777269244194,
      "rewards/margins": 0.121826171875,
      "rewards/rejected": -0.18247070908546448,
      "step": 5270
    },
    {
      "epoch": 0.38446135362434924,
      "grad_norm": 2.0199515879498158,
      "learning_rate": 1.1009637651263607e-06,
      "log_odds_chosen": 1.3438231945037842,
      "log_odds_ratio": -0.4375976622104645,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.8804687261581421,
      "logps/chosen": -0.6415039300918579,
      "logps/rejected": -1.55078125,
      "loss": 0.8491,
      "nll_loss": 0.7808593511581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06423339992761612,
      "rewards/margins": 0.09075317531824112,
      "rewards/rejected": -0.15495605766773224,
      "step": 5280
    },
    {
      "epoch": 0.38518950012742564,
      "grad_norm": 2.8066042740912094,
      "learning_rate": 1.0999226644063927e-06,
      "log_odds_chosen": 1.506933569908142,
      "log_odds_ratio": -0.39204102754592896,
      "logits/chosen": -0.9410156011581421,
      "logits/rejected": -0.8939453363418579,
      "logps/chosen": -0.6029297113418579,
      "logps/rejected": -1.6208984851837158,
      "loss": 0.839,
      "nll_loss": 0.7352539300918579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06029052659869194,
      "rewards/margins": 0.10187987983226776,
      "rewards/rejected": -0.16213378310203552,
      "step": 5290
    },
    {
      "epoch": 0.38591764663050204,
      "grad_norm": 2.369989436955428,
      "learning_rate": 1.0988845115895123e-06,
      "log_odds_chosen": 1.604089379310608,
      "log_odds_ratio": -0.39653319120407104,
      "logits/chosen": -1.011132836341858,
      "logits/rejected": -0.8792968988418579,
      "logps/chosen": -0.6424804925918579,
      "logps/rejected": -1.7595703601837158,
      "loss": 0.8469,
      "nll_loss": 0.824414074420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06429443508386612,
      "rewards/margins": 0.11170196533203125,
      "rewards/rejected": -0.17600098252296448,
      "step": 5300
    },
    {
      "epoch": 0.3866457931335785,
      "grad_norm": 2.594537085533605,
      "learning_rate": 1.0978492927901574e-06,
      "log_odds_chosen": 1.4656250476837158,
      "log_odds_ratio": -0.40019530057907104,
      "logits/chosen": -0.9710937738418579,
      "logits/rejected": -0.849414050579071,
      "logps/chosen": -0.6631835699081421,
      "logps/rejected": -1.6535155773162842,
      "loss": 0.8645,
      "nll_loss": 0.7982422113418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06635741889476776,
      "rewards/margins": 0.09897460788488388,
      "rewards/rejected": -0.16538086533546448,
      "step": 5310
    },
    {
      "epoch": 0.3873739396366549,
      "grad_norm": 2.4186888768174786,
      "learning_rate": 1.0968169942141634e-06,
      "log_odds_chosen": 1.6306641101837158,
      "log_odds_ratio": -0.3559814393520355,
      "logits/chosen": -0.9703124761581421,
      "logits/rejected": -0.8353515863418579,
      "logps/chosen": -0.6532226800918579,
      "logps/rejected": -1.7742187976837158,
      "loss": 0.8334,
      "nll_loss": 0.7835937738418579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06528320163488388,
      "rewards/margins": 0.11210326850414276,
      "rewards/rejected": -0.17741699516773224,
      "step": 5320
    },
    {
      "epoch": 0.3881020861397313,
      "grad_norm": 3.777127780525136,
      "learning_rate": 1.0957876021579874e-06,
      "log_odds_chosen": 1.3818359375,
      "log_odds_ratio": -0.41767579317092896,
      "logits/chosen": -0.971484363079071,
      "logits/rejected": -0.8824218511581421,
      "logps/chosen": -0.7079101800918579,
      "logps/rejected": -1.6886718273162842,
      "loss": 0.8391,
      "nll_loss": 0.8335937261581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07083740085363388,
      "rewards/margins": 0.09788818657398224,
      "rewards/rejected": -0.16877441108226776,
      "step": 5330
    },
    {
      "epoch": 0.38883023264280775,
      "grad_norm": 2.3535647466806524,
      "learning_rate": 1.0947611030079466e-06,
      "log_odds_chosen": 1.6271483898162842,
      "log_odds_ratio": -0.36933594942092896,
      "logits/chosen": -0.991015613079071,
      "logits/rejected": -0.8589843511581421,
      "logps/chosen": -0.6412109136581421,
      "logps/rejected": -1.8017578125,
      "loss": 0.8296,
      "nll_loss": 0.8072265386581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06412353366613388,
      "rewards/margins": 0.11595459282398224,
      "rewards/rejected": -0.18020018935203552,
      "step": 5340
    },
    {
      "epoch": 0.38955837914588415,
      "grad_norm": 3.3066965319210326,
      "learning_rate": 1.0937374832394612e-06,
      "log_odds_chosen": 1.7453124523162842,
      "log_odds_ratio": -0.35114747285842896,
      "logits/chosen": -0.9585937261581421,
      "logits/rejected": -0.8570312261581421,
      "logps/chosen": -0.667285144329071,
      "logps/rejected": -1.900390625,
      "loss": 0.8399,
      "nll_loss": 0.794726550579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06669922173023224,
      "rewards/margins": 0.12348632514476776,
      "rewards/rejected": -0.19016113877296448,
      "step": 5350
    },
    {
      "epoch": 0.39028652564896055,
      "grad_norm": 2.1590312864549848,
      "learning_rate": 1.092716729416306e-06,
      "log_odds_chosen": 1.1875,
      "log_odds_ratio": -0.44482421875,
      "logits/chosen": -0.9449218511581421,
      "logits/rejected": -0.872851550579071,
      "logps/chosen": -0.7066406011581421,
      "logps/rejected": -1.521093726158142,
      "loss": 0.8449,
      "nll_loss": 0.7875000238418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07069091498851776,
      "rewards/margins": 0.08162841945886612,
      "rewards/rejected": -0.15227051079273224,
      "step": 5360
    },
    {
      "epoch": 0.391014672152037,
      "grad_norm": 2.303663788098858,
      "learning_rate": 1.0916988281898703e-06,
      "log_odds_chosen": 1.7605469226837158,
      "log_odds_ratio": -0.35795897245407104,
      "logits/chosen": -0.989062488079071,
      "logits/rejected": -0.905078113079071,
      "logps/chosen": -0.649121105670929,
      "logps/rejected": -1.89453125,
      "loss": 0.8386,
      "nll_loss": 0.8056640625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06486816704273224,
      "rewards/margins": 0.1243896484375,
      "rewards/rejected": -0.18930664658546448,
      "step": 5370
    },
    {
      "epoch": 0.3917428186551134,
      "grad_norm": 2.5151319562423753,
      "learning_rate": 1.0906837662984237e-06,
      "log_odds_chosen": 1.7705078125,
      "log_odds_ratio": -0.3629394471645355,
      "logits/chosen": -0.982226550579071,
      "logits/rejected": -0.8939453363418579,
      "logps/chosen": -0.6534179449081421,
      "logps/rejected": -1.9128906726837158,
      "loss": 0.832,
      "nll_loss": 0.840039074420929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06533203274011612,
      "rewards/margins": 0.12606200575828552,
      "rewards/rejected": -0.19140625,
      "step": 5380
    },
    {
      "epoch": 0.39247096515818986,
      "grad_norm": 2.553797225452928,
      "learning_rate": 1.089671530566391e-06,
      "log_odds_chosen": 1.490136742591858,
      "log_odds_ratio": -0.4043945372104645,
      "logits/chosen": -1.034765601158142,
      "logits/rejected": -0.9244140386581421,
      "logps/chosen": -0.708203136920929,
      "logps/rejected": -1.785546898841858,
      "loss": 0.8121,
      "nll_loss": 0.749804675579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07084961235523224,
      "rewards/margins": 0.10776367038488388,
      "rewards/rejected": -0.17851562798023224,
      "step": 5390
    },
    {
      "epoch": 0.39319911166126625,
      "grad_norm": 3.0297695711525674,
      "learning_rate": 1.0886621079036346e-06,
      "log_odds_chosen": 1.4514648914337158,
      "log_odds_ratio": -0.3819335997104645,
      "logits/chosen": -0.972851574420929,
      "logits/rejected": -0.8628906011581421,
      "logps/chosen": -0.6341797113418579,
      "logps/rejected": -1.5935547351837158,
      "loss": 0.8142,
      "nll_loss": 0.8121093511581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06343994289636612,
      "rewards/margins": 0.09586791694164276,
      "rewards/rejected": -0.15939942002296448,
      "step": 5400
    },
    {
      "epoch": 0.39392725816434265,
      "grad_norm": 2.543936426766196,
      "learning_rate": 1.0876554853047417e-06,
      "log_odds_chosen": 1.3517577648162842,
      "log_odds_ratio": -0.42314451932907104,
      "logits/chosen": -1.016992211341858,
      "logits/rejected": -0.8990234136581421,
      "logps/chosen": -0.64501953125,
      "logps/rejected": -1.591210961341858,
      "loss": 0.8294,
      "nll_loss": 0.810546875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06456299126148224,
      "rewards/margins": 0.09447021782398224,
      "rewards/rejected": -0.15913085639476776,
      "step": 5410
    },
    {
      "epoch": 0.3946554046674191,
      "grad_norm": 2.4627130591280753,
      "learning_rate": 1.0866516498483225e-06,
      "log_odds_chosen": 1.372656226158142,
      "log_odds_ratio": -0.43183594942092896,
      "logits/chosen": -0.992968738079071,
      "logits/rejected": -0.890625,
      "logps/chosen": -0.6153320074081421,
      "logps/rejected": -1.58203125,
      "loss": 0.8693,
      "nll_loss": 0.7835937738418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0615234375,
      "rewards/margins": 0.09664306789636612,
      "rewards/rejected": -0.15822753310203552,
      "step": 5420
    },
    {
      "epoch": 0.3953835511704955,
      "grad_norm": 3.036728481403032,
      "learning_rate": 1.0856505886963116e-06,
      "log_odds_chosen": 1.448632836341858,
      "log_odds_ratio": -0.4195312559604645,
      "logits/chosen": -0.9917968511581421,
      "logits/rejected": -0.8763672113418579,
      "logps/chosen": -0.6767578125,
      "logps/rejected": -1.712499976158142,
      "loss": 0.8227,
      "nll_loss": 0.7906249761581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06773681938648224,
      "rewards/margins": 0.10356445610523224,
      "rewards/rejected": -0.17128905653953552,
      "step": 5430
    },
    {
      "epoch": 0.3961116976735719,
      "grad_norm": 2.2198622782056288,
      "learning_rate": 1.0846522890932808e-06,
      "log_odds_chosen": 1.6552734375,
      "log_odds_ratio": -0.38703614473342896,
      "logits/chosen": -0.979296863079071,
      "logits/rejected": -0.84765625,
      "logps/chosen": -0.68017578125,
      "logps/rejected": -1.865234375,
      "loss": 0.8404,
      "nll_loss": 0.7811523675918579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06800536811351776,
      "rewards/margins": 0.11862792819738388,
      "rewards/rejected": -0.1866455078125,
      "step": 5440
    },
    {
      "epoch": 0.39683984417664836,
      "grad_norm": 2.2845839406615154,
      "learning_rate": 1.0836567383657542e-06,
      "log_odds_chosen": 1.440039038658142,
      "log_odds_ratio": -0.4117675721645355,
      "logits/chosen": -1.00390625,
      "logits/rejected": -0.904492199420929,
      "logps/chosen": -0.7108398675918579,
      "logps/rejected": -1.7439453601837158,
      "loss": 0.834,
      "nll_loss": 0.787402331829071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07111816108226776,
      "rewards/margins": 0.103240966796875,
      "rewards/rejected": -0.17438964545726776,
      "step": 5450
    },
    {
      "epoch": 0.39756799067972476,
      "grad_norm": 2.5025237220211634,
      "learning_rate": 1.0826639239215334e-06,
      "log_odds_chosen": 1.519921898841858,
      "log_odds_ratio": -0.36738282442092896,
      "logits/chosen": -1.0037109851837158,
      "logits/rejected": -0.873242199420929,
      "logps/chosen": -0.646289050579071,
      "logps/rejected": -1.6902344226837158,
      "loss": 0.8016,
      "nll_loss": 0.756640613079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06466064602136612,
      "rewards/margins": 0.10429687798023224,
      "rewards/rejected": -0.16892090439796448,
      "step": 5460
    },
    {
      "epoch": 0.39829613718280116,
      "grad_norm": 3.2183972587161946,
      "learning_rate": 1.0816738332490292e-06,
      "log_odds_chosen": 1.4484374523162842,
      "log_odds_ratio": -0.3704589903354645,
      "logits/chosen": -1.0007812976837158,
      "logits/rejected": -0.902148425579071,
      "logps/chosen": -0.627148449420929,
      "logps/rejected": -1.611718773841858,
      "loss": 0.8599,
      "nll_loss": 0.7818359136581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06270752102136612,
      "rewards/margins": 0.09840087592601776,
      "rewards/rejected": -0.16098633408546448,
      "step": 5470
    },
    {
      "epoch": 0.3990242836858776,
      "grad_norm": 3.4213556027434766,
      "learning_rate": 1.0806864539165982e-06,
      "log_odds_chosen": 1.204687476158142,
      "log_odds_ratio": -0.49980467557907104,
      "logits/chosen": -0.9769531488418579,
      "logits/rejected": -0.8955078125,
      "logps/chosen": -0.7427734136581421,
      "logps/rejected": -1.5714843273162842,
      "loss": 0.8552,
      "nll_loss": 0.8798828125,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07431640475988388,
      "rewards/margins": 0.08291015774011612,
      "rewards/rejected": -0.15720215439796448,
      "step": 5480
    },
    {
      "epoch": 0.399752430188954,
      "grad_norm": 2.443504489597527,
      "learning_rate": 1.0797017735718878e-06,
      "log_odds_chosen": 1.4277465343475342,
      "log_odds_ratio": -0.404052734375,
      "logits/chosen": -0.9966796636581421,
      "logits/rejected": -0.886914074420929,
      "logps/chosen": -0.652050793170929,
      "logps/rejected": -1.630859375,
      "loss": 0.8516,
      "nll_loss": 0.839062511920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06513671576976776,
      "rewards/margins": 0.09802856296300888,
      "rewards/rejected": -0.1630859375,
      "step": 5490
    },
    {
      "epoch": 0.4004805766920304,
      "grad_norm": 2.2466202910288726,
      "learning_rate": 1.0787197799411874e-06,
      "log_odds_chosen": 1.627050757408142,
      "log_odds_ratio": -0.3813720643520355,
      "logits/chosen": -1.0236327648162842,
      "logits/rejected": -0.9091796875,
      "logps/chosen": -0.635937511920929,
      "logps/rejected": -1.7900390625,
      "loss": 0.824,
      "nll_loss": 0.7769531011581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06356201320886612,
      "rewards/margins": 0.11553039401769638,
      "rewards/rejected": -0.17917481064796448,
      "step": 5500
    },
    {
      "epoch": 0.40120872319510686,
      "grad_norm": 2.1990567655997943,
      "learning_rate": 1.0777404608287846e-06,
      "log_odds_chosen": 1.381445288658142,
      "log_odds_ratio": -0.3998046815395355,
      "logits/chosen": -1.003515601158142,
      "logits/rejected": -0.8902343511581421,
      "logps/chosen": -0.6689453125,
      "logps/rejected": -1.613867163658142,
      "loss": 0.8479,
      "nll_loss": 0.8099609613418579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06693115085363388,
      "rewards/margins": 0.09456177055835724,
      "rewards/rejected": -0.16145019233226776,
      "step": 5510
    },
    {
      "epoch": 0.40193686969818326,
      "grad_norm": 3.916825028555095,
      "learning_rate": 1.0767638041163309e-06,
      "log_odds_chosen": 1.6003906726837158,
      "log_odds_ratio": -0.3753906190395355,
      "logits/chosen": -1.049218773841858,
      "logits/rejected": -0.9183593988418579,
      "logps/chosen": -0.6639648675918579,
      "logps/rejected": -1.741796851158142,
      "loss": 0.8315,
      "nll_loss": 0.8001953363418579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06641845405101776,
      "rewards/margins": 0.10783691704273224,
      "rewards/rejected": -0.17416992783546448,
      "step": 5520
    },
    {
      "epoch": 0.4026650162012597,
      "grad_norm": 6.261971330193659,
      "learning_rate": 1.0757897977622107e-06,
      "log_odds_chosen": 1.4181640148162842,
      "log_odds_ratio": -0.4051757752895355,
      "logits/chosen": -1.0203125476837158,
      "logits/rejected": -0.891406238079071,
      "logps/chosen": -0.67626953125,
      "logps/rejected": -1.703125,
      "loss": 0.8477,
      "nll_loss": 0.8492187261581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06763915717601776,
      "rewards/margins": 0.10260009765625,
      "rewards/rejected": -0.17026367783546448,
      "step": 5530
    },
    {
      "epoch": 0.4033931627043361,
      "grad_norm": 2.3894935630346428,
      "learning_rate": 1.074818429800918e-06,
      "log_odds_chosen": 1.2545897960662842,
      "log_odds_ratio": -0.4541992247104645,
      "logits/chosen": -0.9927734136581421,
      "logits/rejected": -0.8880859613418579,
      "logps/chosen": -0.69482421875,
      "logps/rejected": -1.580078125,
      "loss": 0.8372,
      "nll_loss": 0.8134765625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06948242336511612,
      "rewards/margins": 0.08867187798023224,
      "rewards/rejected": -0.15805664658546448,
      "step": 5540
    },
    {
      "epoch": 0.4041213092074125,
      "grad_norm": 2.346748315815649,
      "learning_rate": 1.073849688342439e-06,
      "log_odds_chosen": 1.521582007408142,
      "log_odds_ratio": -0.37138670682907104,
      "logits/chosen": -1.0263671875,
      "logits/rejected": -0.9046875238418579,
      "logps/chosen": -0.645312488079071,
      "logps/rejected": -1.677343726158142,
      "loss": 0.8354,
      "nll_loss": 0.70166015625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0645751953125,
      "rewards/margins": 0.10330810397863388,
      "rewards/rejected": -0.16774901747703552,
      "step": 5550
    },
    {
      "epoch": 0.40484945571048897,
      "grad_norm": 2.6668992200157087,
      "learning_rate": 1.07288356157164e-06,
      "log_odds_chosen": 1.75341796875,
      "log_odds_ratio": -0.3414062559604645,
      "logits/chosen": -1.008203148841858,
      "logits/rejected": -0.885937511920929,
      "logps/chosen": -0.6328125,
      "logps/rejected": -1.8771483898162842,
      "loss": 0.8222,
      "nll_loss": 0.7481445074081421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06337890774011612,
      "rewards/margins": 0.124359130859375,
      "rewards/rejected": -0.1876220703125,
      "step": 5560
    },
    {
      "epoch": 0.40557760221356537,
      "grad_norm": 2.2934257825842184,
      "learning_rate": 1.0719200377476648e-06,
      "log_odds_chosen": 1.606542944908142,
      "log_odds_ratio": -0.4273925721645355,
      "logits/chosen": -0.9947265386581421,
      "logits/rejected": -0.884960949420929,
      "logps/chosen": -0.6792968511581421,
      "logps/rejected": -1.862890601158142,
      "loss": 0.8,
      "nll_loss": 0.7730468511581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06783447414636612,
      "rewards/margins": 0.118438720703125,
      "rewards/rejected": -0.18635253608226776,
      "step": 5570
    },
    {
      "epoch": 0.40630574871664177,
      "grad_norm": 2.9068652384038116,
      "learning_rate": 1.0709591052033317e-06,
      "log_odds_chosen": 1.352441430091858,
      "log_odds_ratio": -0.42802733182907104,
      "logits/chosen": -0.96484375,
      "logits/rejected": -0.858203113079071,
      "logps/chosen": -0.6324218511581421,
      "logps/rejected": -1.5597655773162842,
      "loss": 0.8283,
      "nll_loss": 0.791796863079071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06318359076976776,
      "rewards/margins": 0.09282837063074112,
      "rewards/rejected": -0.15595702826976776,
      "step": 5580
    },
    {
      "epoch": 0.4070338952197182,
      "grad_norm": 2.4150019667857636,
      "learning_rate": 1.0700007523445435e-06,
      "log_odds_chosen": 1.520849585533142,
      "log_odds_ratio": -0.40605467557907104,
      "logits/chosen": -0.9673827886581421,
      "logits/rejected": -0.862109363079071,
      "logps/chosen": -0.600781261920929,
      "logps/rejected": -1.6365234851837158,
      "loss": 0.8051,
      "nll_loss": 0.752734363079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06011962890625,
      "rewards/margins": 0.10360260307788849,
      "rewards/rejected": -0.16379395127296448,
      "step": 5590
    },
    {
      "epoch": 0.4077620417227946,
      "grad_norm": 2.3035202409571123,
      "learning_rate": 1.0690449676496976e-06,
      "log_odds_chosen": 1.5573241710662842,
      "log_odds_ratio": -0.4097656309604645,
      "logits/chosen": -0.971484363079071,
      "logits/rejected": -0.882617175579071,
      "logps/chosen": -0.6787109375,
      "logps/rejected": -1.7960937023162842,
      "loss": 0.8364,
      "nll_loss": 0.7982422113418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06787109375,
      "rewards/margins": 0.11170653998851776,
      "rewards/rejected": -0.17951659858226776,
      "step": 5600
    },
    {
      "epoch": 0.408490188225871,
      "grad_norm": 2.6012049564037074,
      "learning_rate": 1.0680917396691054e-06,
      "log_odds_chosen": 1.7761719226837158,
      "log_odds_ratio": -0.3301025331020355,
      "logits/chosen": -0.9634765386581421,
      "logits/rejected": -0.805859386920929,
      "logps/chosen": -0.6258789300918579,
      "logps/rejected": -1.877343773841858,
      "loss": 0.8145,
      "nll_loss": 0.7943359613418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06258545070886612,
      "rewards/margins": 0.125,
      "rewards/rejected": -0.18759766221046448,
      "step": 5610
    },
    {
      "epoch": 0.4092183347289475,
      "grad_norm": 3.5947587434108232,
      "learning_rate": 1.0671410570244164e-06,
      "log_odds_chosen": 1.6227538585662842,
      "log_odds_ratio": -0.374755859375,
      "logits/chosen": -0.9898437261581421,
      "logits/rejected": -0.845507800579071,
      "logps/chosen": -0.6607421636581421,
      "logps/rejected": -1.8175780773162842,
      "loss": 0.8398,
      "nll_loss": 0.812304675579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06607665866613388,
      "rewards/margins": 0.11577758938074112,
      "rewards/rejected": -0.181884765625,
      "step": 5620
    },
    {
      "epoch": 0.4099464812320239,
      "grad_norm": 2.747795949554436,
      "learning_rate": 1.0661929084080466e-06,
      "log_odds_chosen": 1.211035132408142,
      "log_odds_ratio": -0.4453125,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.883984386920929,
      "logps/chosen": -0.68017578125,
      "logps/rejected": -1.5164062976837158,
      "loss": 0.8557,
      "nll_loss": 0.8199218511581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06809081882238388,
      "rewards/margins": 0.083740234375,
      "rewards/rejected": -0.15166015923023224,
      "step": 5630
    },
    {
      "epoch": 0.41067462773510033,
      "grad_norm": 2.363212131876386,
      "learning_rate": 1.0652472825826149e-06,
      "log_odds_chosen": 1.4697265625,
      "log_odds_ratio": -0.3851074278354645,
      "logits/chosen": -0.9683593511581421,
      "logits/rejected": -0.8394531011581421,
      "logps/chosen": -0.681347668170929,
      "logps/rejected": -1.731835961341858,
      "loss": 0.8399,
      "nll_loss": 0.792187511920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06815185397863388,
      "rewards/margins": 0.10513152927160263,
      "rewards/rejected": -0.17314453423023224,
      "step": 5640
    },
    {
      "epoch": 0.4114027742381767,
      "grad_norm": 2.295440412104529,
      "learning_rate": 1.0643041683803828e-06,
      "log_odds_chosen": 1.2716796398162842,
      "log_odds_ratio": -0.43134766817092896,
      "logits/chosen": -0.9580078125,
      "logits/rejected": -0.857421875,
      "logps/chosen": -0.7017577886581421,
      "logps/rejected": -1.624414086341858,
      "loss": 0.8357,
      "nll_loss": 0.8802734613418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07021484524011612,
      "rewards/margins": 0.09215088188648224,
      "rewards/rejected": -0.16237792372703552,
      "step": 5650
    },
    {
      "epoch": 0.4121309207412531,
      "grad_norm": 2.265025615139446,
      "learning_rate": 1.063363554702701e-06,
      "log_odds_chosen": 1.548059105873108,
      "log_odds_ratio": -0.37885743379592896,
      "logits/chosen": -0.979296863079071,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -0.6333984136581421,
      "logps/rejected": -1.7009766101837158,
      "loss": 0.875,
      "nll_loss": 0.824414074420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06328125298023224,
      "rewards/margins": 0.10682983696460724,
      "rewards/rejected": -0.17014160752296448,
      "step": 5660
    },
    {
      "epoch": 0.4128590672443296,
      "grad_norm": 2.5230988020173095,
      "learning_rate": 1.0624254305194609e-06,
      "log_odds_chosen": 1.4622070789337158,
      "log_odds_ratio": -0.3829589784145355,
      "logits/chosen": -0.980664074420929,
      "logits/rejected": -0.8529297113418579,
      "logps/chosen": -0.6703125238418579,
      "logps/rejected": -1.6769530773162842,
      "loss": 0.844,
      "nll_loss": 0.7953125238418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06702880561351776,
      "rewards/margins": 0.10074462741613388,
      "rewards/rejected": -0.16765137016773224,
      "step": 5670
    },
    {
      "epoch": 0.413587213747406,
      "grad_norm": 2.521259555067162,
      "learning_rate": 1.0614897848685505e-06,
      "log_odds_chosen": 1.605078101158142,
      "log_odds_ratio": -0.3622070252895355,
      "logits/chosen": -1.0119140148162842,
      "logits/rejected": -0.8988281488418579,
      "logps/chosen": -0.6859375238418579,
      "logps/rejected": -1.8419921398162842,
      "loss": 0.8333,
      "nll_loss": 0.838085949420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06857910007238388,
      "rewards/margins": 0.11553955078125,
      "rewards/rejected": -0.18398436903953552,
      "step": 5680
    },
    {
      "epoch": 0.4143153602504824,
      "grad_norm": 2.3320231678900054,
      "learning_rate": 1.0605566068553173e-06,
      "log_odds_chosen": 1.3623046875,
      "log_odds_ratio": -0.44096678495407104,
      "logits/chosen": -0.9527343511581421,
      "logits/rejected": -0.8539062738418579,
      "logps/chosen": -0.682910144329071,
      "logps/rejected": -1.634179711341858,
      "loss": 0.8189,
      "nll_loss": 0.767382800579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06831054389476776,
      "rewards/margins": 0.09521484375,
      "rewards/rejected": -0.16352538764476776,
      "step": 5690
    },
    {
      "epoch": 0.41504350675355883,
      "grad_norm": 2.1986475813785615,
      "learning_rate": 1.0596258856520351e-06,
      "log_odds_chosen": 1.350683569908142,
      "log_odds_ratio": -0.40473634004592896,
      "logits/chosen": -0.970703125,
      "logits/rejected": -0.866015613079071,
      "logps/chosen": -0.6861327886581421,
      "logps/rejected": -1.6208984851837158,
      "loss": 0.8217,
      "nll_loss": 0.7837890386581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06857910007238388,
      "rewards/margins": 0.09345702826976776,
      "rewards/rejected": -0.16213378310203552,
      "step": 5700
    },
    {
      "epoch": 0.41577165325663523,
      "grad_norm": 2.981820620538686,
      "learning_rate": 1.0586976104973764e-06,
      "log_odds_chosen": 1.575903296470642,
      "log_odds_ratio": -0.39228516817092896,
      "logits/chosen": -0.966601550579071,
      "logits/rejected": -0.8705078363418579,
      "logps/chosen": -0.683300793170929,
      "logps/rejected": -1.810937523841858,
      "loss": 0.821,
      "nll_loss": 0.7572265863418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06831054389476776,
      "rewards/margins": 0.11281432956457138,
      "rewards/rejected": -0.18107910454273224,
      "step": 5710
    },
    {
      "epoch": 0.41649979975971163,
      "grad_norm": 2.455152323343703,
      "learning_rate": 1.05777177069589e-06,
      "log_odds_chosen": 1.724218726158142,
      "log_odds_ratio": -0.34956055879592896,
      "logits/chosen": -0.951367199420929,
      "logits/rejected": -0.8716796636581421,
      "logps/chosen": -0.64453125,
      "logps/rejected": -1.8074219226837158,
      "loss": 0.8332,
      "nll_loss": 0.817187488079071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06442870944738388,
      "rewards/margins": 0.11641235649585724,
      "rewards/rejected": -0.18081054091453552,
      "step": 5720
    },
    {
      "epoch": 0.4172279462627881,
      "grad_norm": 2.287756485606037,
      "learning_rate": 1.0568483556174834e-06,
      "log_odds_chosen": 1.70703125,
      "log_odds_ratio": -0.31791990995407104,
      "logits/chosen": -1.0115234851837158,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.5770508050918579,
      "logps/rejected": -1.750390648841858,
      "loss": 0.8144,
      "nll_loss": 0.7613281011581421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05769043043255806,
      "rewards/margins": 0.11713866889476776,
      "rewards/rejected": -0.17490234971046448,
      "step": 5730
    },
    {
      "epoch": 0.4179560927658645,
      "grad_norm": 2.994369876484584,
      "learning_rate": 1.055927354696909e-06,
      "log_odds_chosen": 1.5388672351837158,
      "log_odds_ratio": -0.36699217557907104,
      "logits/chosen": -0.9322265386581421,
      "logits/rejected": -0.834765613079071,
      "logps/chosen": -0.654101550579071,
      "logps/rejected": -1.7042968273162842,
      "loss": 0.8302,
      "nll_loss": 0.75830078125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0654296875,
      "rewards/margins": 0.10507812350988388,
      "rewards/rejected": -0.17050781846046448,
      "step": 5740
    },
    {
      "epoch": 0.4186842392689409,
      "grad_norm": 2.387712945823637,
      "learning_rate": 1.0550087574332592e-06,
      "log_odds_chosen": 1.63330078125,
      "log_odds_ratio": -0.35710448026657104,
      "logits/chosen": -0.9677734375,
      "logits/rejected": -0.8755859136581421,
      "logps/chosen": -0.62255859375,
      "logps/rejected": -1.784765601158142,
      "loss": 0.8139,
      "nll_loss": 0.7666015625,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06229247897863388,
      "rewards/margins": 0.11617431789636612,
      "rewards/rejected": -0.17844238877296448,
      "step": 5750
    },
    {
      "epoch": 0.41941238577201734,
      "grad_norm": 2.2651065438241846,
      "learning_rate": 1.0540925533894598e-06,
      "log_odds_chosen": 1.670019507408142,
      "log_odds_ratio": -0.3768554627895355,
      "logits/chosen": -1.0244140625,
      "logits/rejected": -0.866992175579071,
      "logps/chosen": -0.633593738079071,
      "logps/rejected": -1.814843773841858,
      "loss": 0.8224,
      "nll_loss": 0.7728515863418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.0633544921875,
      "rewards/margins": 0.11815796047449112,
      "rewards/rejected": -0.1815185546875,
      "step": 5760
    },
    {
      "epoch": 0.42014053227509374,
      "grad_norm": 2.245877868602975,
      "learning_rate": 1.053178732191775e-06,
      "log_odds_chosen": 1.9031250476837158,
      "log_odds_ratio": -0.3037109375,
      "logits/chosen": -1.016015648841858,
      "logits/rejected": -0.8998047113418579,
      "logps/chosen": -0.592578113079071,
      "logps/rejected": -1.946874976158142,
      "loss": 0.832,
      "nll_loss": 0.8082031011581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05928955227136612,
      "rewards/margins": 0.13525390625,
      "rewards/rejected": -0.19448241591453552,
      "step": 5770
    },
    {
      "epoch": 0.4208686787781702,
      "grad_norm": 2.7630570113921147,
      "learning_rate": 1.0522672835293127e-06,
      "log_odds_chosen": 1.5861327648162842,
      "log_odds_ratio": -0.36406248807907104,
      "logits/chosen": -1.014257788658142,
      "logits/rejected": -0.867968738079071,
      "logps/chosen": -0.6849609613418579,
      "logps/rejected": -1.7707030773162842,
      "loss": 0.8223,
      "nll_loss": 0.8486328125,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06853027641773224,
      "rewards/margins": 0.10849609225988388,
      "rewards/rejected": -0.1768798828125,
      "step": 5780
    },
    {
      "epoch": 0.4215968252812466,
      "grad_norm": 2.338542671225521,
      "learning_rate": 1.0513581971535365e-06,
      "log_odds_chosen": 1.4328124523162842,
      "log_odds_ratio": -0.36865234375,
      "logits/chosen": -0.9833984375,
      "logits/rejected": -0.833984375,
      "logps/chosen": -0.6669921875,
      "logps/rejected": -1.651953101158142,
      "loss": 0.8136,
      "nll_loss": 0.7391601800918579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06662597507238388,
      "rewards/margins": 0.09865722805261612,
      "rewards/rejected": -0.165283203125,
      "step": 5790
    },
    {
      "epoch": 0.422324971784323,
      "grad_norm": 2.772525065876872,
      "learning_rate": 1.0504514628777803e-06,
      "log_odds_chosen": 1.5867187976837158,
      "log_odds_ratio": -0.346435546875,
      "logits/chosen": -0.9541015625,
      "logits/rejected": -0.850390613079071,
      "logps/chosen": -0.5887695550918579,
      "logps/rejected": -1.637109398841858,
      "loss": 0.8292,
      "nll_loss": 0.7708984613418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05880127102136612,
      "rewards/margins": 0.1048583984375,
      "rewards/rejected": -0.16376952826976776,
      "step": 5800
    },
    {
      "epoch": 0.42305311828739944,
      "grad_norm": 2.4762325089529442,
      "learning_rate": 1.0495470705767713e-06,
      "log_odds_chosen": 1.489160180091858,
      "log_odds_ratio": -0.42890626192092896,
      "logits/chosen": -0.921093761920929,
      "logits/rejected": -0.848437488079071,
      "logps/chosen": -0.675488293170929,
      "logps/rejected": -1.7109375,
      "loss": 0.8302,
      "nll_loss": 0.8177734613418579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06757812201976776,
      "rewards/margins": 0.10354156792163849,
      "rewards/rejected": -0.17104491591453552,
      "step": 5810
    },
    {
      "epoch": 0.42378126479047584,
      "grad_norm": 2.3867965816115126,
      "learning_rate": 1.0486450101861527e-06,
      "log_odds_chosen": 1.5458984375,
      "log_odds_ratio": -0.36601561307907104,
      "logits/chosen": -0.949414074420929,
      "logits/rejected": -0.8910156488418579,
      "logps/chosen": -0.6405273675918579,
      "logps/rejected": -1.7013671398162842,
      "loss": 0.8274,
      "nll_loss": 0.78173828125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0640869140625,
      "rewards/margins": 0.10590820014476776,
      "rewards/rejected": -0.1700439453125,
      "step": 5820
    },
    {
      "epoch": 0.42450941129355224,
      "grad_norm": 7.6675961372490855,
      "learning_rate": 1.0477452717020143e-06,
      "log_odds_chosen": 1.7697265148162842,
      "log_odds_ratio": -0.3580078184604645,
      "logits/chosen": -0.9808593988418579,
      "logits/rejected": -0.853710949420929,
      "logps/chosen": -0.6439453363418579,
      "logps/rejected": -1.9054687023162842,
      "loss": 0.8329,
      "nll_loss": 0.778613269329071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06439208984375,
      "rewards/margins": 0.12613525986671448,
      "rewards/rejected": -0.19038085639476776,
      "step": 5830
    },
    {
      "epoch": 0.4252375577966287,
      "grad_norm": 2.5801685731453587,
      "learning_rate": 1.0468478451804272e-06,
      "log_odds_chosen": 1.8093750476837158,
      "log_odds_ratio": -0.360107421875,
      "logits/chosen": -0.9896484613418579,
      "logits/rejected": -0.837890625,
      "logps/chosen": -0.648730456829071,
      "logps/rejected": -1.980859398841858,
      "loss": 0.7904,
      "nll_loss": 0.790234386920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06488037109375,
      "rewards/margins": 0.13327637314796448,
      "rewards/rejected": -0.19804687798023224,
      "step": 5840
    },
    {
      "epoch": 0.4259657042997051,
      "grad_norm": 2.855837915897883,
      "learning_rate": 1.0459527207369814e-06,
      "log_odds_chosen": 1.8306152820587158,
      "log_odds_ratio": -0.31025391817092896,
      "logits/chosen": -0.991015613079071,
      "logits/rejected": -0.8541015386581421,
      "logps/chosen": -0.654980480670929,
      "logps/rejected": -1.935937523841858,
      "loss": 0.8178,
      "nll_loss": 0.808789074420929,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.06545410305261612,
      "rewards/margins": 0.12813110649585724,
      "rewards/rejected": -0.19357910752296448,
      "step": 5850
    },
    {
      "epoch": 0.4266938508027815,
      "grad_norm": 2.2446120882896707,
      "learning_rate": 1.0450598885463281e-06,
      "log_odds_chosen": 1.6154296398162842,
      "log_odds_ratio": -0.39287108182907104,
      "logits/chosen": -0.992382824420929,
      "logits/rejected": -0.883593738079071,
      "logps/chosen": -0.615917980670929,
      "logps/rejected": -1.7078125476837158,
      "loss": 0.7967,
      "nll_loss": 0.749804675579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06159668043255806,
      "rewards/margins": 0.10918579250574112,
      "rewards/rejected": -0.17070312798023224,
      "step": 5860
    },
    {
      "epoch": 0.42742199730585795,
      "grad_norm": 2.5655256287774337,
      "learning_rate": 1.0441693388417282e-06,
      "log_odds_chosen": 1.435449242591858,
      "log_odds_ratio": -0.404296875,
      "logits/chosen": -0.968945324420929,
      "logits/rejected": -0.8501952886581421,
      "logps/chosen": -0.669921875,
      "logps/rejected": -1.669921875,
      "loss": 0.822,
      "nll_loss": 0.8232421875,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06700439751148224,
      "rewards/margins": 0.09995117038488388,
      "rewards/rejected": -0.16704101860523224,
      "step": 5870
    },
    {
      "epoch": 0.42815014380893435,
      "grad_norm": 2.4402563066248812,
      "learning_rate": 1.0432810619146023e-06,
      "log_odds_chosen": 1.4692871570587158,
      "log_odds_ratio": -0.40263670682907104,
      "logits/chosen": -0.9912109375,
      "logits/rejected": -0.87109375,
      "logps/chosen": -0.61767578125,
      "logps/rejected": -1.6064453125,
      "loss": 0.8183,
      "nll_loss": 0.7269531488418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.061767578125,
      "rewards/margins": 0.09886474907398224,
      "rewards/rejected": -0.16064453125,
      "step": 5880
    },
    {
      "epoch": 0.4288782903120108,
      "grad_norm": 3.1585484128908092,
      "learning_rate": 1.042395048114086e-06,
      "log_odds_chosen": 1.652734398841858,
      "log_odds_ratio": -0.36931151151657104,
      "logits/chosen": -0.9814453125,
      "logits/rejected": -0.8414062261581421,
      "logps/chosen": -0.6712890863418579,
      "logps/rejected": -1.8083984851837158,
      "loss": 0.8194,
      "nll_loss": 0.823046863079071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06712646782398224,
      "rewards/margins": 0.11383056640625,
      "rewards/rejected": -0.18088379502296448,
      "step": 5890
    },
    {
      "epoch": 0.4296064368150872,
      "grad_norm": 2.44360496780276,
      "learning_rate": 1.041511287846591e-06,
      "log_odds_chosen": 1.6759765148162842,
      "log_odds_ratio": -0.3524414002895355,
      "logits/chosen": -0.9683593511581421,
      "logits/rejected": -0.8427734375,
      "logps/chosen": -0.634082019329071,
      "logps/rejected": -1.7998046875,
      "loss": 0.818,
      "nll_loss": 0.770703136920929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.0633544921875,
      "rewards/margins": 0.11667480319738388,
      "rewards/rejected": -0.18002930283546448,
      "step": 5900
    },
    {
      "epoch": 0.4303345833181636,
      "grad_norm": 2.2602207464888635,
      "learning_rate": 1.0406297715753674e-06,
      "log_odds_chosen": 1.5607421398162842,
      "log_odds_ratio": -0.367431640625,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.8570312261581421,
      "logps/chosen": -0.6958984136581421,
      "logps/rejected": -1.8132812976837158,
      "loss": 0.8295,
      "nll_loss": 0.804882824420929,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06954345852136612,
      "rewards/margins": 0.11174316704273224,
      "rewards/rejected": -0.18120117485523224,
      "step": 5910
    },
    {
      "epoch": 0.43106272982124005,
      "grad_norm": 2.5284144535242232,
      "learning_rate": 1.0397504898200726e-06,
      "log_odds_chosen": 1.915624976158142,
      "log_odds_ratio": -0.3387451171875,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.863085925579071,
      "logps/chosen": -0.6670898199081421,
      "logps/rejected": -2.0347657203674316,
      "loss": 0.8199,
      "nll_loss": 0.863085925579071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06662597507238388,
      "rewards/margins": 0.13682861626148224,
      "rewards/rejected": -0.20341797173023224,
      "step": 5920
    },
    {
      "epoch": 0.43179087632431645,
      "grad_norm": 2.740820482575086,
      "learning_rate": 1.0388734331563415e-06,
      "log_odds_chosen": 1.873046875,
      "log_odds_ratio": -0.3523925840854645,
      "logits/chosen": -1.0027344226837158,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.6766601800918579,
      "logps/rejected": -2.0550780296325684,
      "loss": 0.8333,
      "nll_loss": 0.764453113079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06766357272863388,
      "rewards/margins": 0.13796386122703552,
      "rewards/rejected": -0.20556640625,
      "step": 5930
    },
    {
      "epoch": 0.43251902282739285,
      "grad_norm": 2.6688921555030376,
      "learning_rate": 1.037998592215364e-06,
      "log_odds_chosen": 1.675195336341858,
      "log_odds_ratio": -0.3687744140625,
      "logits/chosen": -1.016015648841858,
      "logits/rejected": -0.8687499761581421,
      "logps/chosen": -0.624804675579071,
      "logps/rejected": -1.7976562976837158,
      "loss": 0.8247,
      "nll_loss": 0.799023449420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06245117262005806,
      "rewards/margins": 0.11710204929113388,
      "rewards/rejected": -0.17963866889476776,
      "step": 5940
    },
    {
      "epoch": 0.4332471693304693,
      "grad_norm": 2.7512305890571525,
      "learning_rate": 1.037125957683463e-06,
      "log_odds_chosen": 1.6776854991912842,
      "log_odds_ratio": -0.3680419921875,
      "logits/chosen": -1.001562476158142,
      "logits/rejected": -0.89453125,
      "logps/chosen": -0.642578125,
      "logps/rejected": -1.8076171875,
      "loss": 0.8236,
      "nll_loss": 0.808398425579071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06425781548023224,
      "rewards/margins": 0.11637572944164276,
      "rewards/rejected": -0.18073730170726776,
      "step": 5950
    },
    {
      "epoch": 0.4339753158335457,
      "grad_norm": 2.3609810330250363,
      "learning_rate": 1.0362555203016794e-06,
      "log_odds_chosen": 1.778906226158142,
      "log_odds_ratio": -0.341064453125,
      "logits/chosen": -1.0125000476837158,
      "logits/rejected": -0.876171886920929,
      "logps/chosen": -0.605761706829071,
      "logps/rejected": -1.8917968273162842,
      "loss": 0.8071,
      "nll_loss": 0.727343738079071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06049804762005806,
      "rewards/margins": 0.12871094048023224,
      "rewards/rejected": -0.18916015326976776,
      "step": 5960
    },
    {
      "epoch": 0.4347034623366221,
      "grad_norm": 2.659037766682072,
      "learning_rate": 1.035387270865359e-06,
      "log_odds_chosen": 1.6171875,
      "log_odds_ratio": -0.3770507872104645,
      "logits/chosen": -1.029296875,
      "logits/rejected": -0.8935546875,
      "logps/chosen": -0.6747070550918579,
      "logps/rejected": -1.8185546398162842,
      "loss": 0.7879,
      "nll_loss": 0.733593761920929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06746826320886612,
      "rewards/margins": 0.11451415717601776,
      "rewards/rejected": -0.18205566704273224,
      "step": 5970
    },
    {
      "epoch": 0.43543160883969856,
      "grad_norm": 2.0530548686015426,
      "learning_rate": 1.0345212002237434e-06,
      "log_odds_chosen": 1.529296875,
      "log_odds_ratio": -0.3731445372104645,
      "logits/chosen": -0.9771484136581421,
      "logits/rejected": -0.870312511920929,
      "logps/chosen": -0.6673828363418579,
      "logps/rejected": -1.7507812976837158,
      "loss": 0.8309,
      "nll_loss": 0.80859375,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06679687649011612,
      "rewards/margins": 0.10825195163488388,
      "rewards/rejected": -0.17499999701976776,
      "step": 5980
    },
    {
      "epoch": 0.43615975534277496,
      "grad_norm": 2.7407686393931727,
      "learning_rate": 1.0336572992795644e-06,
      "log_odds_chosen": 1.477929711341858,
      "log_odds_ratio": -0.39973145723342896,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.924023449420929,
      "logps/chosen": -0.6514648199081421,
      "logps/rejected": -1.6857421398162842,
      "loss": 0.7916,
      "nll_loss": 0.7759765386581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06512451171875,
      "rewards/margins": 0.10338745266199112,
      "rewards/rejected": -0.16850586235523224,
      "step": 5990
    },
    {
      "epoch": 0.4368879018458514,
      "grad_norm": 2.68627681411438,
      "learning_rate": 1.0327955589886444e-06,
      "log_odds_chosen": 1.7111327648162842,
      "log_odds_ratio": -0.3501830995082855,
      "logits/chosen": -0.984179675579071,
      "logits/rejected": -0.884570300579071,
      "logps/chosen": -0.6753906011581421,
      "logps/rejected": -1.9181640148162842,
      "loss": 0.8133,
      "nll_loss": 0.7728515863418579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0675048828125,
      "rewards/margins": 0.12418212741613388,
      "rewards/rejected": -0.19179686903953552,
      "step": 6000
    },
    {
      "epoch": 0.4376160483489278,
      "grad_norm": 2.221105654678892,
      "learning_rate": 1.0319359703594971e-06,
      "log_odds_chosen": 1.485937476158142,
      "log_odds_ratio": -0.42236328125,
      "logits/chosen": -1.0166015625,
      "logits/rejected": -0.8773437738418579,
      "logps/chosen": -0.683300793170929,
      "logps/rejected": -1.7546875476837158,
      "loss": 0.8015,
      "nll_loss": 0.739062488079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06837157905101776,
      "rewards/margins": 0.10714111477136612,
      "rewards/rejected": -0.17546387016773224,
      "step": 6010
    },
    {
      "epoch": 0.4383441948520042,
      "grad_norm": 2.2531751260285096,
      "learning_rate": 1.0310785244529341e-06,
      "log_odds_chosen": 1.8624999523162842,
      "log_odds_ratio": -0.3697753846645355,
      "logits/chosen": -0.9833984375,
      "logits/rejected": -0.886523425579071,
      "logps/chosen": -0.6329101324081421,
      "logps/rejected": -1.9406249523162842,
      "loss": 0.8066,
      "nll_loss": 0.722363293170929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06331787258386612,
      "rewards/margins": 0.13077393174171448,
      "rewards/rejected": -0.19401855766773224,
      "step": 6020
    },
    {
      "epoch": 0.43907234135508066,
      "grad_norm": 2.828297663151744,
      "learning_rate": 1.0302232123816746e-06,
      "log_odds_chosen": 1.682031273841858,
      "log_odds_ratio": -0.3722900450229645,
      "logits/chosen": -1.013671875,
      "logits/rejected": -0.8990234136581421,
      "logps/chosen": -0.666015625,
      "logps/rejected": -1.830078125,
      "loss": 0.8163,
      "nll_loss": 0.757617175579071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06657715141773224,
      "rewards/margins": 0.11646728217601776,
      "rewards/rejected": -0.18303222954273224,
      "step": 6030
    },
    {
      "epoch": 0.43980048785815706,
      "grad_norm": 2.7015406509441813,
      "learning_rate": 1.0293700253099576e-06,
      "log_odds_chosen": 1.442285180091858,
      "log_odds_ratio": -0.38188475370407104,
      "logits/chosen": -1.000390648841858,
      "logits/rejected": -0.8580077886581421,
      "logps/chosen": -0.6400390863418579,
      "logps/rejected": -1.673828125,
      "loss": 0.8199,
      "nll_loss": 0.730664074420929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06405029445886612,
      "rewards/margins": 0.10338439792394638,
      "rewards/rejected": -0.16740722954273224,
      "step": 6040
    },
    {
      "epoch": 0.44052863436123346,
      "grad_norm": 2.278778621378478,
      "learning_rate": 1.02851895445316e-06,
      "log_odds_chosen": 1.652246117591858,
      "log_odds_ratio": -0.3895019590854645,
      "logits/chosen": -1.0048828125,
      "logits/rejected": -0.884570300579071,
      "logps/chosen": -0.630175769329071,
      "logps/rejected": -1.827734351158142,
      "loss": 0.8066,
      "nll_loss": 0.719531238079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06296386569738388,
      "rewards/margins": 0.1197509765625,
      "rewards/rejected": -0.18283692002296448,
      "step": 6050
    },
    {
      "epoch": 0.4412567808643099,
      "grad_norm": 2.990846438429488,
      "learning_rate": 1.0276699910774159e-06,
      "log_odds_chosen": 1.510351538658142,
      "log_odds_ratio": -0.3731933534145355,
      "logits/chosen": -1.038671851158142,
      "logits/rejected": -0.9375,
      "logps/chosen": -0.6451171636581421,
      "logps/rejected": -1.6921875476837158,
      "loss": 0.8016,
      "nll_loss": 0.7748047113418579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06455077975988388,
      "rewards/margins": 0.10469970852136612,
      "rewards/rejected": -0.16914062201976776,
      "step": 6060
    },
    {
      "epoch": 0.4419849273673863,
      "grad_norm": 2.760751042320254,
      "learning_rate": 1.0268231264992398e-06,
      "log_odds_chosen": 1.5283203125,
      "log_odds_ratio": -0.403564453125,
      "logits/chosen": -0.9935547113418579,
      "logits/rejected": -0.875,
      "logps/chosen": -0.6537109613418579,
      "logps/rejected": -1.755859375,
      "loss": 0.8021,
      "nll_loss": 0.788281261920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06541748344898224,
      "rewards/margins": 0.11030273139476776,
      "rewards/rejected": -0.17585448920726776,
      "step": 6070
    },
    {
      "epoch": 0.4427130738704627,
      "grad_norm": 2.4766671780232974,
      "learning_rate": 1.0259783520851542e-06,
      "log_odds_chosen": 1.5,
      "log_odds_ratio": -0.39892578125,
      "logits/chosen": -0.999804675579071,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.678027331829071,
      "logps/rejected": -1.7287108898162842,
      "loss": 0.8056,
      "nll_loss": 0.78173828125,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06776122748851776,
      "rewards/margins": 0.10504150390625,
      "rewards/rejected": -0.17292480170726776,
      "step": 6080
    },
    {
      "epoch": 0.44344122037353917,
      "grad_norm": 2.976776027909396,
      "learning_rate": 1.0251356592513193e-06,
      "log_odds_chosen": 1.8347656726837158,
      "log_odds_ratio": -0.33769530057907104,
      "logits/chosen": -0.9576171636581421,
      "logits/rejected": -0.8760741949081421,
      "logps/chosen": -0.742480456829071,
      "logps/rejected": -2.1019530296325684,
      "loss": 0.8098,
      "nll_loss": 0.795117199420929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.07425536960363388,
      "rewards/margins": 0.13583984971046448,
      "rewards/rejected": -0.21015624701976776,
      "step": 6090
    },
    {
      "epoch": 0.44416936687661557,
      "grad_norm": 2.5263244184580786,
      "learning_rate": 1.0242950394631678e-06,
      "log_odds_chosen": 1.4890625476837158,
      "log_odds_ratio": -0.3907714784145355,
      "logits/chosen": -0.9742187261581421,
      "logits/rejected": -0.9017578363418579,
      "logps/chosen": -0.6375976800918579,
      "logps/rejected": -1.6466796398162842,
      "loss": 0.8165,
      "nll_loss": 0.7314453125,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06380615383386612,
      "rewards/margins": 0.10077514499425888,
      "rewards/rejected": -0.16452637314796448,
      "step": 6100
    },
    {
      "epoch": 0.44489751337969197,
      "grad_norm": 2.5795975704678153,
      "learning_rate": 1.0234564842350404e-06,
      "log_odds_chosen": 1.5652344226837158,
      "log_odds_ratio": -0.381591796875,
      "logits/chosen": -0.9974609613418579,
      "logits/rejected": -0.8763672113418579,
      "logps/chosen": -0.661425769329071,
      "logps/rejected": -1.7257812023162842,
      "loss": 0.8219,
      "nll_loss": 0.793749988079071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06611327826976776,
      "rewards/margins": 0.10645751655101776,
      "rewards/rejected": -0.17265625298023224,
      "step": 6110
    },
    {
      "epoch": 0.4456256598827684,
      "grad_norm": 2.4367829689496334,
      "learning_rate": 1.0226199851298272e-06,
      "log_odds_chosen": 1.420312523841858,
      "log_odds_ratio": -0.3935546875,
      "logits/chosen": -1.0,
      "logits/rejected": -0.8837890625,
      "logps/chosen": -0.6397460699081421,
      "logps/rejected": -1.611718773841858,
      "loss": 0.8057,
      "nll_loss": 0.767871081829071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06401367485523224,
      "rewards/margins": 0.09704895317554474,
      "rewards/rejected": -0.16105957329273224,
      "step": 6120
    },
    {
      "epoch": 0.4463538063858448,
      "grad_norm": 2.307918122422047,
      "learning_rate": 1.0217855337586106e-06,
      "log_odds_chosen": 1.7584960460662842,
      "log_odds_ratio": -0.37714844942092896,
      "logits/chosen": -0.9876953363418579,
      "logits/rejected": -0.8910156488418579,
      "logps/chosen": -0.6294921636581421,
      "logps/rejected": -1.8916015625,
      "loss": 0.8067,
      "nll_loss": 0.7626953125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06290283054113388,
      "rewards/margins": 0.12617798149585724,
      "rewards/rejected": -0.189208984375,
      "step": 6130
    },
    {
      "epoch": 0.4470819528889213,
      "grad_norm": 2.6717437023642168,
      "learning_rate": 1.0209531217803119e-06,
      "log_odds_chosen": 1.630468726158142,
      "log_odds_ratio": -0.364013671875,
      "logits/chosen": -0.9951171875,
      "logits/rejected": -0.880078136920929,
      "logps/chosen": -0.643750011920929,
      "logps/rejected": -1.7625000476837158,
      "loss": 0.7906,
      "nll_loss": 0.716796875,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06440429389476776,
      "rewards/margins": 0.11179199069738388,
      "rewards/rejected": -0.17612305283546448,
      "step": 6140
    },
    {
      "epoch": 0.4478100993919977,
      "grad_norm": 2.4985350307080227,
      "learning_rate": 1.0201227409013412e-06,
      "log_odds_chosen": 1.706445336341858,
      "log_odds_ratio": -0.3802246153354645,
      "logits/chosen": -1.0166015625,
      "logits/rejected": -0.9117187261581421,
      "logps/chosen": -0.69970703125,
      "logps/rejected": -1.9130859375,
      "loss": 0.7994,
      "nll_loss": 0.757617175579071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06993408501148224,
      "rewards/margins": 0.12126465141773224,
      "rewards/rejected": -0.1912841796875,
      "step": 6150
    },
    {
      "epoch": 0.4485382458950741,
      "grad_norm": 2.4204862965729728,
      "learning_rate": 1.0192943828752509e-06,
      "log_odds_chosen": 1.7175781726837158,
      "log_odds_ratio": -0.36186522245407104,
      "logits/chosen": -1.0193359851837158,
      "logits/rejected": -0.8939453363418579,
      "logps/chosen": -0.6610351800918579,
      "logps/rejected": -1.9265625476837158,
      "loss": 0.7963,
      "nll_loss": 0.7616211175918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06611327826976776,
      "rewards/margins": 0.12655028700828552,
      "rewards/rejected": -0.19267578423023224,
      "step": 6160
    },
    {
      "epoch": 0.4492663923981505,
      "grad_norm": 2.4703112298702226,
      "learning_rate": 1.0184680395023912e-06,
      "log_odds_chosen": 1.6238281726837158,
      "log_odds_ratio": -0.38237303495407104,
      "logits/chosen": -0.950976550579071,
      "logits/rejected": -0.82421875,
      "logps/chosen": -0.684765636920929,
      "logps/rejected": -1.861718773841858,
      "loss": 0.8019,
      "nll_loss": 0.765429675579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06851806491613388,
      "rewards/margins": 0.11774902045726776,
      "rewards/rejected": -0.18625488877296448,
      "step": 6170
    },
    {
      "epoch": 0.4499945389012269,
      "grad_norm": 2.5076159194949303,
      "learning_rate": 1.0176437026295688e-06,
      "log_odds_chosen": 1.463281273841858,
      "log_odds_ratio": -0.4150390625,
      "logits/chosen": -0.950390636920929,
      "logits/rejected": -0.8394531011581421,
      "logps/chosen": -0.6714843511581421,
      "logps/rejected": -1.697265625,
      "loss": 0.8252,
      "nll_loss": 0.8041015863418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.067138671875,
      "rewards/margins": 0.10260009765625,
      "rewards/rejected": -0.16975097358226776,
      "step": 6180
    },
    {
      "epoch": 0.4507226854043033,
      "grad_norm": 2.268214143169916,
      "learning_rate": 1.0168213641497094e-06,
      "log_odds_chosen": 1.390966773033142,
      "log_odds_ratio": -0.4046386778354645,
      "logits/chosen": -1.0148437023162842,
      "logits/rejected": -0.906054675579071,
      "logps/chosen": -0.711132824420929,
      "logps/rejected": -1.705664038658142,
      "loss": 0.8205,
      "nll_loss": 0.8441406488418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07111816108226776,
      "rewards/margins": 0.09953613579273224,
      "rewards/rejected": -0.17055663466453552,
      "step": 6190
    },
    {
      "epoch": 0.4514508319073798,
      "grad_norm": 2.692053168018983,
      "learning_rate": 1.016001016001524e-06,
      "log_odds_chosen": 1.3346679210662842,
      "log_odds_ratio": -0.40996092557907104,
      "logits/chosen": -0.9703124761581421,
      "logits/rejected": -0.849414050579071,
      "logps/chosen": -0.642773449420929,
      "logps/rejected": -1.5613281726837158,
      "loss": 0.8048,
      "nll_loss": 0.825878918170929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06427001953125,
      "rewards/margins": 0.091949462890625,
      "rewards/rejected": -0.15620116889476776,
      "step": 6200
    },
    {
      "epoch": 0.4521789784104562,
      "grad_norm": 2.8373641462855193,
      "learning_rate": 1.0151826501691747e-06,
      "log_odds_chosen": 1.5598633289337158,
      "log_odds_ratio": -0.38627928495407104,
      "logits/chosen": -1.050390601158142,
      "logits/rejected": -0.954882800579071,
      "logps/chosen": -0.627734363079071,
      "logps/rejected": -1.711328148841858,
      "loss": 0.8196,
      "nll_loss": 0.680371105670929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06284179538488388,
      "rewards/margins": 0.10833740234375,
      "rewards/rejected": -0.17124024033546448,
      "step": 6210
    },
    {
      "epoch": 0.4529071249135326,
      "grad_norm": 2.3840045783477364,
      "learning_rate": 1.0143662586819475e-06,
      "log_odds_chosen": 1.8654296398162842,
      "log_odds_ratio": -0.3145751953125,
      "logits/chosen": -0.9654296636581421,
      "logits/rejected": -0.8837890625,
      "logps/chosen": -0.654101550579071,
      "logps/rejected": -1.982031226158142,
      "loss": 0.8149,
      "nll_loss": 0.8021484613418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06540527194738388,
      "rewards/margins": 0.13291016221046448,
      "rewards/rejected": -0.19821777939796448,
      "step": 6220
    },
    {
      "epoch": 0.45363527141660903,
      "grad_norm": 3.3004241923323643,
      "learning_rate": 1.0135518336139257e-06,
      "log_odds_chosen": 1.442480444908142,
      "log_odds_ratio": -0.381103515625,
      "logits/chosen": -0.9683593511581421,
      "logits/rejected": -0.9056640863418579,
      "logps/chosen": -0.6043945550918579,
      "logps/rejected": -1.539453148841858,
      "loss": 0.8368,
      "nll_loss": 0.7793945074081421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06036376953125,
      "rewards/margins": 0.09347991645336151,
      "rewards/rejected": -0.15390625596046448,
      "step": 6230
    },
    {
      "epoch": 0.45436341791968543,
      "grad_norm": 2.750618897856238,
      "learning_rate": 1.0127393670836666e-06,
      "log_odds_chosen": 1.918115258216858,
      "log_odds_ratio": -0.34455567598342896,
      "logits/chosen": -1.0177733898162842,
      "logits/rejected": -0.918164074420929,
      "logps/chosen": -0.6548827886581421,
      "logps/rejected": -2.083789110183716,
      "loss": 0.8001,
      "nll_loss": 0.73876953125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.0655517578125,
      "rewards/margins": 0.142852783203125,
      "rewards/rejected": -0.20834961533546448,
      "step": 6240
    },
    {
      "epoch": 0.4550915644227619,
      "grad_norm": 2.7229558871411457,
      "learning_rate": 1.0119288512538813e-06,
      "log_odds_chosen": 1.7564208507537842,
      "log_odds_ratio": -0.35515135526657104,
      "logits/chosen": -0.9740234613418579,
      "logits/rejected": -0.8814452886581421,
      "logps/chosen": -0.670703113079071,
      "logps/rejected": -1.902929663658142,
      "loss": 0.7881,
      "nll_loss": 0.8236328363418579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06707763671875,
      "rewards/margins": 0.12322387844324112,
      "rewards/rejected": -0.19033202528953552,
      "step": 6250
    },
    {
      "epoch": 0.4558197109258383,
      "grad_norm": 2.5251247297173345,
      "learning_rate": 1.0111202783311173e-06,
      "log_odds_chosen": 1.242578148841858,
      "log_odds_ratio": -0.45263671875,
      "logits/chosen": -0.959765613079071,
      "logits/rejected": -0.883593738079071,
      "logps/chosen": -0.6937500238418579,
      "logps/rejected": -1.5460937023162842,
      "loss": 0.8184,
      "nll_loss": 0.816210925579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06936035305261612,
      "rewards/margins": 0.08524169772863388,
      "rewards/rejected": -0.15463867783546448,
      "step": 6260
    },
    {
      "epoch": 0.4565478574289147,
      "grad_norm": 2.2159286644771443,
      "learning_rate": 1.010313640565443e-06,
      "log_odds_chosen": 1.8515625,
      "log_odds_ratio": -0.3023437559604645,
      "logits/chosen": -1.0154297351837158,
      "logits/rejected": -0.9208984375,
      "logps/chosen": -0.5858398675918579,
      "logps/rejected": -1.814453125,
      "loss": 0.7999,
      "nll_loss": 0.7149413824081421,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.05855713039636612,
      "rewards/margins": 0.12290038913488388,
      "rewards/rejected": -0.18154296278953552,
      "step": 6270
    },
    {
      "epoch": 0.45727600393199114,
      "grad_norm": 2.5225961149602636,
      "learning_rate": 1.0095089302501373e-06,
      "log_odds_chosen": 1.592382788658142,
      "log_odds_ratio": -0.37548828125,
      "logits/chosen": -0.987500011920929,
      "logits/rejected": -0.8960937261581421,
      "logps/chosen": -0.7098633050918579,
      "logps/rejected": -1.8585937023162842,
      "loss": 0.8161,
      "nll_loss": 0.79248046875,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07093505561351776,
      "rewards/margins": 0.11495361477136612,
      "rewards/rejected": -0.18583984673023224,
      "step": 6280
    },
    {
      "epoch": 0.45800415043506754,
      "grad_norm": 3.018478677036301,
      "learning_rate": 1.0087061397213787e-06,
      "log_odds_chosen": 1.775781273841858,
      "log_odds_ratio": -0.362548828125,
      "logits/chosen": -0.982421875,
      "logits/rejected": -0.916210949420929,
      "logps/chosen": -0.627148449420929,
      "logps/rejected": -1.861328125,
      "loss": 0.7845,
      "nll_loss": 0.72412109375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06269530951976776,
      "rewards/margins": 0.12355957180261612,
      "rewards/rejected": -0.18613281846046448,
      "step": 6290
    },
    {
      "epoch": 0.45873229693814394,
      "grad_norm": 2.3485159287477604,
      "learning_rate": 1.007905261357939e-06,
      "log_odds_chosen": 1.359960913658142,
      "log_odds_ratio": -0.43779295682907104,
      "logits/chosen": -0.980273425579071,
      "logits/rejected": -0.886523425579071,
      "logps/chosen": -0.670214831829071,
      "logps/rejected": -1.644921898841858,
      "loss": 0.8037,
      "nll_loss": 0.8138672113418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06700439751148224,
      "rewards/margins": 0.09743042290210724,
      "rewards/rejected": -0.16440430283546448,
      "step": 6300
    },
    {
      "epoch": 0.4594604434412204,
      "grad_norm": 2.1990435977161726,
      "learning_rate": 1.0071062875808811e-06,
      "log_odds_chosen": 1.6437499523162842,
      "log_odds_ratio": -0.37541502714157104,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.918164074420929,
      "logps/chosen": -0.663378894329071,
      "logps/rejected": -1.846093773841858,
      "loss": 0.7799,
      "nll_loss": 0.713671863079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06630859524011612,
      "rewards/margins": 0.1182861328125,
      "rewards/rejected": -0.18447265028953552,
      "step": 6310
    },
    {
      "epoch": 0.4601885899442968,
      "grad_norm": 3.1357715146790333,
      "learning_rate": 1.0063092108532552e-06,
      "log_odds_chosen": 1.688623070716858,
      "log_odds_ratio": -0.37211912870407104,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.8892577886581421,
      "logps/chosen": -0.64990234375,
      "logps/rejected": -1.8386719226837158,
      "loss": 0.8304,
      "nll_loss": 0.818554699420929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06496582180261612,
      "rewards/margins": 0.11884842067956924,
      "rewards/rejected": -0.18376465141773224,
      "step": 6320
    },
    {
      "epoch": 0.4609167364473732,
      "grad_norm": 2.283925808102014,
      "learning_rate": 1.005514023679802e-06,
      "log_odds_chosen": 1.3818359375,
      "log_odds_ratio": -0.4169921875,
      "logits/chosen": -1.0125000476837158,
      "logits/rejected": -0.938281238079071,
      "logps/chosen": -0.65576171875,
      "logps/rejected": -1.625,
      "loss": 0.8184,
      "nll_loss": 0.807812511920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06558837741613388,
      "rewards/margins": 0.096893310546875,
      "rewards/rejected": -0.16257324814796448,
      "step": 6330
    },
    {
      "epoch": 0.46164488295044964,
      "grad_norm": 2.4375000606033526,
      "learning_rate": 1.0047207186066567e-06,
      "log_odds_chosen": 1.6865234375,
      "log_odds_ratio": -0.3727050721645355,
      "logits/chosen": -1.0029296875,
      "logits/rejected": -0.902148425579071,
      "logps/chosen": -0.693359375,
      "logps/rejected": -1.9343750476837158,
      "loss": 0.8027,
      "nll_loss": 0.7894531488418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.0693359375,
      "rewards/margins": 0.12403564155101776,
      "rewards/rejected": -0.193359375,
      "step": 6340
    },
    {
      "epoch": 0.46237302945352604,
      "grad_norm": 2.7966514024999687,
      "learning_rate": 1.0039292882210538e-06,
      "log_odds_chosen": 1.4171874523162842,
      "log_odds_ratio": -0.4225097596645355,
      "logits/chosen": -0.9810546636581421,
      "logits/rejected": -0.888671875,
      "logps/chosen": -0.6888672113418579,
      "logps/rejected": -1.718359351158142,
      "loss": 0.819,
      "nll_loss": 0.7681640386581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06889648735523224,
      "rewards/margins": 0.10291137546300888,
      "rewards/rejected": -0.1719970703125,
      "step": 6350
    },
    {
      "epoch": 0.4631011759566025,
      "grad_norm": 2.496092195906707,
      "learning_rate": 1.0031397251510382e-06,
      "log_odds_chosen": 1.5348632335662842,
      "log_odds_ratio": -0.38203126192092896,
      "logits/chosen": -1.0517578125,
      "logits/rejected": -0.9302734136581421,
      "logps/chosen": -0.625,
      "logps/rejected": -1.6599609851837158,
      "loss": 0.8047,
      "nll_loss": 0.7401367425918579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06251220405101776,
      "rewards/margins": 0.1033935546875,
      "rewards/rejected": -0.16594238579273224,
      "step": 6360
    },
    {
      "epoch": 0.4638293224596789,
      "grad_norm": 2.600323623795615,
      "learning_rate": 1.0023520220651762e-06,
      "log_odds_chosen": 1.485449194908142,
      "log_odds_ratio": -0.39299315214157104,
      "logits/chosen": -1.0369141101837158,
      "logits/rejected": -0.9091796875,
      "logps/chosen": -0.689257800579071,
      "logps/rejected": -1.7205078601837158,
      "loss": 0.8306,
      "nll_loss": 0.7613281011581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06888427585363388,
      "rewards/margins": 0.10314331203699112,
      "rewards/rejected": -0.17197266221046448,
      "step": 6370
    },
    {
      "epoch": 0.4645574689627553,
      "grad_norm": 2.251740300583199,
      "learning_rate": 1.0015661716722687e-06,
      "log_odds_chosen": 1.717382788658142,
      "log_odds_ratio": -0.3705078065395355,
      "logits/chosen": -1.0068359375,
      "logits/rejected": -0.8773437738418579,
      "logps/chosen": -0.6415039300918579,
      "logps/rejected": -1.8796875476837158,
      "loss": 0.7982,
      "nll_loss": 0.7470703125,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06418456882238388,
      "rewards/margins": 0.123779296875,
      "rewards/rejected": -0.18796387314796448,
      "step": 6380
    },
    {
      "epoch": 0.46528561546583175,
      "grad_norm": 2.6047992237863733,
      "learning_rate": 1.0007821667210687e-06,
      "log_odds_chosen": 1.530859351158142,
      "log_odds_ratio": -0.3729492127895355,
      "logits/chosen": -1.0576171875,
      "logits/rejected": -0.900585949420929,
      "logps/chosen": -0.62939453125,
      "logps/rejected": -1.7138671875,
      "loss": 0.7951,
      "nll_loss": 0.723828136920929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06298828125,
      "rewards/margins": 0.10858154296875,
      "rewards/rejected": -0.17155762016773224,
      "step": 6390
    },
    {
      "epoch": 0.46601376196890815,
      "grad_norm": 2.635994181964765,
      "learning_rate": 1e-06,
      "log_odds_chosen": 1.693750023841858,
      "log_odds_ratio": -0.34404295682907104,
      "logits/chosen": -1.0291016101837158,
      "logits/rejected": -0.900195300579071,
      "logps/chosen": -0.6036132574081421,
      "logps/rejected": -1.7859375476837158,
      "loss": 0.7903,
      "nll_loss": 0.6834961175918579,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06038818508386612,
      "rewards/margins": 0.11820068210363388,
      "rewards/rejected": -0.1785888671875,
      "step": 6400
    },
    {
      "epoch": 0.46674190847198455,
      "grad_norm": 2.3155391352159618,
      "learning_rate": 9.992196643368784e-07,
      "log_odds_chosen": 1.674414038658142,
      "log_odds_ratio": -0.3731445372104645,
      "logits/chosen": -1.033789038658142,
      "logits/rejected": -0.9205077886581421,
      "logps/chosen": -0.6712890863418579,
      "logps/rejected": -1.8439452648162842,
      "loss": 0.8129,
      "nll_loss": 0.7103515863418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06715087592601776,
      "rewards/margins": 0.11739502102136612,
      "rewards/rejected": -0.18447265028953552,
      "step": 6410
    },
    {
      "epoch": 0.467470054975061,
      "grad_norm": 2.3874985601990883,
      "learning_rate": 9.984411525986355e-07,
      "log_odds_chosen": 1.5038573741912842,
      "log_odds_ratio": -0.3951171934604645,
      "logits/chosen": -0.9765625,
      "logits/rejected": -0.8714843988418579,
      "logps/chosen": -0.684863269329071,
      "logps/rejected": -1.723046898841858,
      "loss": 0.7966,
      "nll_loss": 0.8013671636581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06843261420726776,
      "rewards/margins": 0.10357666015625,
      "rewards/rejected": -0.17214354872703552,
      "step": 6420
    },
    {
      "epoch": 0.4681982014781374,
      "grad_norm": 2.2903784604641158,
      "learning_rate": 9.97664457691046e-07,
      "log_odds_chosen": 1.7146484851837158,
      "log_odds_ratio": -0.36064451932907104,
      "logits/chosen": -0.9951171875,
      "logits/rejected": -0.85546875,
      "logps/chosen": -0.676562488079071,
      "logps/rejected": -1.9367187023162842,
      "loss": 0.8061,
      "nll_loss": 0.8101562261581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06757812201976776,
      "rewards/margins": 0.12596435844898224,
      "rewards/rejected": -0.193603515625,
      "step": 6430
    },
    {
      "epoch": 0.4689263479812138,
      "grad_norm": 4.635380747259522,
      "learning_rate": 9.968895725584535e-07,
      "log_odds_chosen": 1.607080101966858,
      "log_odds_ratio": -0.4034179747104645,
      "logits/chosen": -1.015039086341858,
      "logits/rejected": -0.884765625,
      "logps/chosen": -0.660449206829071,
      "logps/rejected": -1.8371093273162842,
      "loss": 0.7915,
      "nll_loss": 0.822265625,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06607665866613388,
      "rewards/margins": 0.11760177463293076,
      "rewards/rejected": -0.1834716796875,
      "step": 6440
    },
    {
      "epoch": 0.46965449448429025,
      "grad_norm": 2.761215986668235,
      "learning_rate": 9.961164901835046e-07,
      "log_odds_chosen": 1.294335961341858,
      "log_odds_ratio": -0.43891602754592896,
      "logits/chosen": -0.966015636920929,
      "logits/rejected": -0.8636718988418579,
      "logps/chosen": -0.6749023199081421,
      "logps/rejected": -1.595312476158142,
      "loss": 0.7992,
      "nll_loss": 0.7655273675918579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06757812201976776,
      "rewards/margins": 0.09205321967601776,
      "rewards/rejected": -0.15957030653953552,
      "step": 6450
    },
    {
      "epoch": 0.47038264098736665,
      "grad_norm": 2.7916506315295933,
      "learning_rate": 9.95345203586879e-07,
      "log_odds_chosen": 1.6357421875,
      "log_odds_ratio": -0.39360350370407104,
      "logits/chosen": -1.0066406726837158,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.664355456829071,
      "logps/rejected": -1.8332030773162842,
      "loss": 0.7912,
      "nll_loss": 0.730175793170929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06641845405101776,
      "rewards/margins": 0.11690673977136612,
      "rewards/rejected": -0.18339844048023224,
      "step": 6460
    },
    {
      "epoch": 0.47111078749044305,
      "grad_norm": 2.724843155115013,
      "learning_rate": 9.94575705827027e-07,
      "log_odds_chosen": 1.624414086341858,
      "log_odds_ratio": -0.375244140625,
      "logits/chosen": -0.947460949420929,
      "logits/rejected": -0.831250011920929,
      "logps/chosen": -0.627636730670929,
      "logps/rejected": -1.7705078125,
      "loss": 0.7916,
      "nll_loss": 0.764453113079071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.062744140625,
      "rewards/margins": 0.11434326320886612,
      "rewards/rejected": -0.177001953125,
      "step": 6470
    },
    {
      "epoch": 0.4718389339935195,
      "grad_norm": 2.2458324262602587,
      "learning_rate": 9.938079899999065e-07,
      "log_odds_chosen": 1.5758850574493408,
      "log_odds_ratio": -0.3849121034145355,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.830859363079071,
      "logps/chosen": -0.645703136920929,
      "logps/rejected": -1.7742187976837158,
      "loss": 0.7776,
      "nll_loss": 0.7328125238418579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.0645751953125,
      "rewards/margins": 0.11293335258960724,
      "rewards/rejected": -0.17756347358226776,
      "step": 6480
    },
    {
      "epoch": 0.4725670804965959,
      "grad_norm": 2.5666515295721557,
      "learning_rate": 9.930420492387219e-07,
      "log_odds_chosen": 1.630273461341858,
      "log_odds_ratio": -0.3658691346645355,
      "logits/chosen": -0.9544922113418579,
      "logits/rejected": -0.8794921636581421,
      "logps/chosen": -0.590624988079071,
      "logps/rejected": -1.7042968273162842,
      "loss": 0.7849,
      "nll_loss": 0.7427734136581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05902099609375,
      "rewards/margins": 0.11130370944738388,
      "rewards/rejected": -0.17031249403953552,
      "step": 6490
    },
    {
      "epoch": 0.47329522699967236,
      "grad_norm": 2.9196738269356466,
      "learning_rate": 9.922778767136676e-07,
      "log_odds_chosen": 1.625390648841858,
      "log_odds_ratio": -0.38427734375,
      "logits/chosen": -0.975390613079071,
      "logits/rejected": -0.865039050579071,
      "logps/chosen": -0.696093738079071,
      "logps/rejected": -1.890625,
      "loss": 0.8023,
      "nll_loss": 0.867382824420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06964111328125,
      "rewards/margins": 0.1195068359375,
      "rewards/rejected": -0.18901367485523224,
      "step": 6500
    },
    {
      "epoch": 0.47402337350274876,
      "grad_norm": 2.3552044002755723,
      "learning_rate": 9.915154656316713e-07,
      "log_odds_chosen": 1.8419921398162842,
      "log_odds_ratio": -0.320556640625,
      "logits/chosen": -0.9966796636581421,
      "logits/rejected": -0.8724609613418579,
      "logps/chosen": -0.670605480670929,
      "logps/rejected": -2.0054688453674316,
      "loss": 0.8159,
      "nll_loss": 0.8287109136581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06707763671875,
      "rewards/margins": 0.13338622450828552,
      "rewards/rejected": -0.200439453125,
      "step": 6510
    },
    {
      "epoch": 0.47475152000582516,
      "grad_norm": 2.220657537542711,
      "learning_rate": 9.907548092361398e-07,
      "log_odds_chosen": 1.856774926185608,
      "log_odds_ratio": -0.3631591796875,
      "logits/chosen": -0.9974609613418579,
      "logits/rejected": -0.8814452886581421,
      "logps/chosen": -0.601757824420929,
      "logps/rejected": -1.9587891101837158,
      "loss": 0.7924,
      "nll_loss": 0.7533203363418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06014404445886612,
      "rewards/margins": 0.13594360649585724,
      "rewards/rejected": -0.19594725966453552,
      "step": 6520
    },
    {
      "epoch": 0.4754796665089016,
      "grad_norm": 2.5661039747571595,
      "learning_rate": 9.899959008067097e-07,
      "log_odds_chosen": 1.7179687023162842,
      "log_odds_ratio": -0.388671875,
      "logits/chosen": -1.001953125,
      "logits/rejected": -0.8607422113418579,
      "logps/chosen": -0.63525390625,
      "logps/rejected": -1.869531273841858,
      "loss": 0.7866,
      "nll_loss": 0.733691394329071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06358642876148224,
      "rewards/margins": 0.12323608249425888,
      "rewards/rejected": -0.18681640923023224,
      "step": 6530
    },
    {
      "epoch": 0.476207813011978,
      "grad_norm": 3.509621771792403,
      "learning_rate": 9.892387336589959e-07,
      "log_odds_chosen": 1.4018065929412842,
      "log_odds_ratio": -0.427734375,
      "logits/chosen": -0.9859374761581421,
      "logits/rejected": -0.913281261920929,
      "logps/chosen": -0.6534179449081421,
      "logps/rejected": -1.590234398841858,
      "loss": 0.7892,
      "nll_loss": 0.766406238079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06533203274011612,
      "rewards/margins": 0.0936737060546875,
      "rewards/rejected": -0.1590576171875,
      "step": 6540
    },
    {
      "epoch": 0.4769359595150544,
      "grad_norm": 2.7226602201573957,
      "learning_rate": 9.884833011443446e-07,
      "log_odds_chosen": 1.8542969226837158,
      "log_odds_ratio": -0.3163818418979645,
      "logits/chosen": -0.9957031011581421,
      "logits/rejected": -0.9037109613418579,
      "logps/chosen": -0.636035144329071,
      "logps/rejected": -1.959375023841858,
      "loss": 0.806,
      "nll_loss": 0.751953125,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06363525241613388,
      "rewards/margins": 0.13239745795726776,
      "rewards/rejected": -0.19602051377296448,
      "step": 6550
    },
    {
      "epoch": 0.47766410601813086,
      "grad_norm": 2.703604996329485,
      "learning_rate": 9.877295966495897e-07,
      "log_odds_chosen": 1.421875,
      "log_odds_ratio": -0.40546876192092896,
      "logits/chosen": -1.011132836341858,
      "logits/rejected": -0.868847668170929,
      "logps/chosen": -0.647167980670929,
      "logps/rejected": -1.583593726158142,
      "loss": 0.8106,
      "nll_loss": 0.767285168170929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06474609673023224,
      "rewards/margins": 0.09353943169116974,
      "rewards/rejected": -0.15827636420726776,
      "step": 6560
    },
    {
      "epoch": 0.47839225252120726,
      "grad_norm": 2.56699604791327,
      "learning_rate": 9.86977613596807e-07,
      "log_odds_chosen": 1.7522461414337158,
      "log_odds_ratio": -0.3863525390625,
      "logits/chosen": -0.986523449420929,
      "logits/rejected": -0.8974609375,
      "logps/chosen": -0.6666015386581421,
      "logps/rejected": -1.896093726158142,
      "loss": 0.8081,
      "nll_loss": 0.787890613079071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.066650390625,
      "rewards/margins": 0.1228485107421875,
      "rewards/rejected": -0.18947753310203552,
      "step": 6570
    },
    {
      "epoch": 0.47912039902428366,
      "grad_norm": 2.7756452008650823,
      "learning_rate": 9.862273454430757e-07,
      "log_odds_chosen": 1.8166015148162842,
      "log_odds_ratio": -0.340576171875,
      "logits/chosen": -1.0109374523162842,
      "logits/rejected": -0.876757800579071,
      "logps/chosen": -0.6371093988418579,
      "logps/rejected": -1.9140625,
      "loss": 0.7874,
      "nll_loss": 0.7406250238418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06376953423023224,
      "rewards/margins": 0.12764891982078552,
      "rewards/rejected": -0.19133301079273224,
      "step": 6580
    },
    {
      "epoch": 0.4798485455273601,
      "grad_norm": 3.3056924999112356,
      "learning_rate": 9.85478785680238e-07,
      "log_odds_chosen": 1.6593749523162842,
      "log_odds_ratio": -0.4114746153354645,
      "logits/chosen": -1.0373046398162842,
      "logits/rejected": -0.918749988079071,
      "logps/chosen": -0.675585925579071,
      "logps/rejected": -1.8855469226837158,
      "loss": 0.7963,
      "nll_loss": 0.742968738079071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06756591796875,
      "rewards/margins": 0.12092284858226776,
      "rewards/rejected": -0.18833008408546448,
      "step": 6590
    },
    {
      "epoch": 0.4805766920304365,
      "grad_norm": 2.230079703120734,
      "learning_rate": 9.847319278346618e-07,
      "log_odds_chosen": 1.7019531726837158,
      "log_odds_ratio": -0.3668456971645355,
      "logits/chosen": -1.032617211341858,
      "logits/rejected": -0.8765624761581421,
      "logps/chosen": -0.632519543170929,
      "logps/rejected": -1.816992163658142,
      "loss": 0.7996,
      "nll_loss": 0.7623046636581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06324462592601776,
      "rewards/margins": 0.11843261867761612,
      "rewards/rejected": -0.18161621689796448,
      "step": 6600
    },
    {
      "epoch": 0.48130483853351297,
      "grad_norm": 2.421937099009381,
      "learning_rate": 9.839867654670063e-07,
      "log_odds_chosen": 1.433496117591858,
      "log_odds_ratio": -0.4129394590854645,
      "logits/chosen": -1.0017578601837158,
      "logits/rejected": -0.8916015625,
      "logps/chosen": -0.6539062261581421,
      "logps/rejected": -1.6669921875,
      "loss": 0.7847,
      "nll_loss": 0.790820300579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06529541313648224,
      "rewards/margins": 0.10108642280101776,
      "rewards/rejected": -0.1663818359375,
      "step": 6610
    },
    {
      "epoch": 0.48203298503658937,
      "grad_norm": 2.4366832929611015,
      "learning_rate": 9.832432921719876e-07,
      "log_odds_chosen": 1.396484375,
      "log_odds_ratio": -0.44135743379592896,
      "logits/chosen": -1.060937523841858,
      "logits/rejected": -0.909960925579071,
      "logps/chosen": -0.6861327886581421,
      "logps/rejected": -1.6955077648162842,
      "loss": 0.8085,
      "nll_loss": 0.735644519329071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06856689602136612,
      "rewards/margins": 0.10087890923023224,
      "rewards/rejected": -0.16958007216453552,
      "step": 6620
    },
    {
      "epoch": 0.48276113153966577,
      "grad_norm": 2.5900738746466225,
      "learning_rate": 9.825015015781493e-07,
      "log_odds_chosen": 1.447851538658142,
      "log_odds_ratio": -0.37333983182907104,
      "logits/chosen": -1.0031249523162842,
      "logits/rejected": -0.8916015625,
      "logps/chosen": -0.6484375,
      "logps/rejected": -1.6453125476837158,
      "loss": 0.789,
      "nll_loss": 0.701855480670929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06492920219898224,
      "rewards/margins": 0.09960327297449112,
      "rewards/rejected": -0.16457518935203552,
      "step": 6630
    },
    {
      "epoch": 0.4834892780427422,
      "grad_norm": 2.559152527560151,
      "learning_rate": 9.81761387347632e-07,
      "log_odds_chosen": 1.383203148841858,
      "log_odds_ratio": -0.39262694120407104,
      "logits/chosen": -1.0314452648162842,
      "logits/rejected": -0.9107421636581421,
      "logps/chosen": -0.641894519329071,
      "logps/rejected": -1.611718773841858,
      "loss": 0.7804,
      "nll_loss": 0.7676757574081421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06419678032398224,
      "rewards/margins": 0.09688720852136612,
      "rewards/rejected": -0.1610107421875,
      "step": 6640
    },
    {
      "epoch": 0.4842174245458186,
      "grad_norm": 2.789615323952939,
      "learning_rate": 9.810229431759452e-07,
      "log_odds_chosen": 1.634765625,
      "log_odds_ratio": -0.412353515625,
      "logits/chosen": -0.955859363079071,
      "logits/rejected": -0.8902343511581421,
      "logps/chosen": -0.642382800579071,
      "logps/rejected": -1.782812476158142,
      "loss": 0.8018,
      "nll_loss": 0.7144531011581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06425781548023224,
      "rewards/margins": 0.11411132663488388,
      "rewards/rejected": -0.17836913466453552,
      "step": 6650
    },
    {
      "epoch": 0.484945571048895,
      "grad_norm": 2.592497093992247,
      "learning_rate": 9.802861627917437e-07,
      "log_odds_chosen": 1.539453148841858,
      "log_odds_ratio": -0.40966796875,
      "logits/chosen": -1.0595703125,
      "logits/rejected": -0.9365234375,
      "logps/chosen": -0.6834961175918579,
      "logps/rejected": -1.76953125,
      "loss": 0.7895,
      "nll_loss": 0.7710937261581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06831054389476776,
      "rewards/margins": 0.10859374701976776,
      "rewards/rejected": -0.17685547471046448,
      "step": 6660
    },
    {
      "epoch": 0.4856737175519715,
      "grad_norm": 3.152536638434244,
      "learning_rate": 9.795510399566016e-07,
      "log_odds_chosen": 1.5526854991912842,
      "log_odds_ratio": -0.39892578125,
      "logits/chosen": -0.9800781011581421,
      "logits/rejected": -0.871289074420929,
      "logps/chosen": -0.7049804925918579,
      "logps/rejected": -1.7921874523162842,
      "loss": 0.8289,
      "nll_loss": 0.81787109375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07050780951976776,
      "rewards/margins": 0.10859985649585724,
      "rewards/rejected": -0.17919921875,
      "step": 6670
    },
    {
      "epoch": 0.4864018640550479,
      "grad_norm": 3.1203045760687993,
      "learning_rate": 9.788175684647926e-07,
      "log_odds_chosen": 1.6710937023162842,
      "log_odds_ratio": -0.367919921875,
      "logits/chosen": -1.0119140148162842,
      "logits/rejected": -0.877148449420929,
      "logps/chosen": -0.6304687261581421,
      "logps/rejected": -1.80859375,
      "loss": 0.7707,
      "nll_loss": 0.726367175579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.0631103515625,
      "rewards/margins": 0.11762695014476776,
      "rewards/rejected": -0.18085937201976776,
      "step": 6680
    },
    {
      "epoch": 0.48713001055812427,
      "grad_norm": 2.833800753935083,
      "learning_rate": 9.780857421430687e-07,
      "log_odds_chosen": 1.65234375,
      "log_odds_ratio": -0.37773436307907104,
      "logits/chosen": -1.053125023841858,
      "logits/rejected": -0.9019531011581421,
      "logps/chosen": -0.631152331829071,
      "logps/rejected": -1.8029296398162842,
      "loss": 0.7995,
      "nll_loss": 0.779296875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0631103515625,
      "rewards/margins": 0.11737060546875,
      "rewards/rejected": -0.1802978515625,
      "step": 6690
    },
    {
      "epoch": 0.4878581570612007,
      "grad_norm": 3.0599836171143853,
      "learning_rate": 9.773555548504417e-07,
      "log_odds_chosen": 1.7712891101837158,
      "log_odds_ratio": -0.34868162870407104,
      "logits/chosen": -1.0314452648162842,
      "logits/rejected": -0.8871093988418579,
      "logps/chosen": -0.5916992425918579,
      "logps/rejected": -1.8310546875,
      "loss": 0.7653,
      "nll_loss": 0.6805664300918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05921630933880806,
      "rewards/margins": 0.12398681789636612,
      "rewards/rejected": -0.18315429985523224,
      "step": 6700
    },
    {
      "epoch": 0.4885863035642771,
      "grad_norm": 3.026069273008677,
      "learning_rate": 9.76627000477968e-07,
      "log_odds_chosen": 1.6765625476837158,
      "log_odds_ratio": -0.37158203125,
      "logits/chosen": -1.009374976158142,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -0.6322265863418579,
      "logps/rejected": -1.8210937976837158,
      "loss": 0.7827,
      "nll_loss": 0.7740234136581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06324462592601776,
      "rewards/margins": 0.11887206882238388,
      "rewards/rejected": -0.18210449814796448,
      "step": 6710
    },
    {
      "epoch": 0.4893144500673536,
      "grad_norm": 2.814178297975446,
      "learning_rate": 9.75900072948533e-07,
      "log_odds_chosen": 1.4871337413787842,
      "log_odds_ratio": -0.37568360567092896,
      "logits/chosen": -1.024999976158142,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.647753894329071,
      "logps/rejected": -1.6884765625,
      "loss": 0.7804,
      "nll_loss": 0.69384765625,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06475830078125,
      "rewards/margins": 0.10388793796300888,
      "rewards/rejected": -0.16875000298023224,
      "step": 6720
    },
    {
      "epoch": 0.49004259657043,
      "grad_norm": 2.765887341253828,
      "learning_rate": 9.751747662166388e-07,
      "log_odds_chosen": 1.6193358898162842,
      "log_odds_ratio": -0.35356444120407104,
      "logits/chosen": -0.9658203125,
      "logits/rejected": -0.8492187261581421,
      "logps/chosen": -0.623828113079071,
      "logps/rejected": -1.7332031726837158,
      "loss": 0.7804,
      "nll_loss": 0.756640613079071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06240234524011612,
      "rewards/margins": 0.11092529445886612,
      "rewards/rejected": -0.17338867485523224,
      "step": 6730
    },
    {
      "epoch": 0.4907707430735064,
      "grad_norm": 2.46537015835012,
      "learning_rate": 9.744510742681917e-07,
      "log_odds_chosen": 2.0406250953674316,
      "log_odds_ratio": -0.298095703125,
      "logits/chosen": -1.008203148841858,
      "logits/rejected": -0.908398449420929,
      "logps/chosen": -0.599902331829071,
      "logps/rejected": -2.0296874046325684,
      "loss": 0.774,
      "nll_loss": 0.7108398675918579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06000976637005806,
      "rewards/margins": 0.14296874403953552,
      "rewards/rejected": -0.20292969048023224,
      "step": 6740
    },
    {
      "epoch": 0.49149888957658283,
      "grad_norm": 2.562503089956989,
      "learning_rate": 9.737289911202953e-07,
      "log_odds_chosen": 1.925195336341858,
      "log_odds_ratio": -0.3298095762729645,
      "logits/chosen": -0.987109363079071,
      "logits/rejected": -0.889453113079071,
      "logps/chosen": -0.6219726800918579,
      "logps/rejected": -2.002734422683716,
      "loss": 0.7865,
      "nll_loss": 0.724902331829071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06217040866613388,
      "rewards/margins": 0.13803711533546448,
      "rewards/rejected": -0.2001953125,
      "step": 6750
    },
    {
      "epoch": 0.49222703607965923,
      "grad_norm": 2.502742766079363,
      "learning_rate": 9.730085108210398e-07,
      "log_odds_chosen": 1.905859351158142,
      "log_odds_ratio": -0.3260498046875,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.8666015863418579,
      "logps/chosen": -0.6529296636581421,
      "logps/rejected": -2.0328125953674316,
      "loss": 0.7686,
      "nll_loss": 0.7342773675918579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06527099758386612,
      "rewards/margins": 0.13795165717601776,
      "rewards/rejected": -0.20322266221046448,
      "step": 6760
    },
    {
      "epoch": 0.49295518258273563,
      "grad_norm": 2.970141278133298,
      "learning_rate": 9.72289627449298e-07,
      "log_odds_chosen": 1.905175805091858,
      "log_odds_ratio": -0.36186522245407104,
      "logits/chosen": -1.028906226158142,
      "logits/rejected": -0.9310547113418579,
      "logps/chosen": -0.670703113079071,
      "logps/rejected": -2.0830078125,
      "loss": 0.7815,
      "nll_loss": 0.7491210699081421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06699218600988388,
      "rewards/margins": 0.14135131239891052,
      "rewards/rejected": -0.20830078423023224,
      "step": 6770
    },
    {
      "epoch": 0.4936833290858121,
      "grad_norm": 2.721144098689221,
      "learning_rate": 9.715723351145206e-07,
      "log_odds_chosen": 1.5060546398162842,
      "log_odds_ratio": -0.4406982362270355,
      "logits/chosen": -0.9896484613418579,
      "logits/rejected": -0.8998047113418579,
      "logps/chosen": -0.6761718988418579,
      "logps/rejected": -1.759765625,
      "loss": 0.7932,
      "nll_loss": 0.7572265863418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06760253757238388,
      "rewards/margins": 0.10834350436925888,
      "rewards/rejected": -0.1759033203125,
      "step": 6780
    },
    {
      "epoch": 0.4944114755888885,
      "grad_norm": 2.601429929689961,
      "learning_rate": 9.70856627956532e-07,
      "log_odds_chosen": 1.571679711341858,
      "log_odds_ratio": -0.39531248807907104,
      "logits/chosen": -1.005859375,
      "logits/rejected": -0.886914074420929,
      "logps/chosen": -0.652636706829071,
      "logps/rejected": -1.770117163658142,
      "loss": 0.7688,
      "nll_loss": 0.7958984375,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06528320163488388,
      "rewards/margins": 0.11171875149011612,
      "rewards/rejected": -0.17695312201976776,
      "step": 6790
    },
    {
      "epoch": 0.4951396220919649,
      "grad_norm": 2.702368418903479,
      "learning_rate": 9.701425001453318e-07,
      "log_odds_chosen": 1.920312523841858,
      "log_odds_ratio": -0.37812501192092896,
      "logits/chosen": -0.9886718988418579,
      "logits/rejected": -0.885937511920929,
      "logps/chosen": -0.65576171875,
      "logps/rejected": -2.0882811546325684,
      "loss": 0.8034,
      "nll_loss": 0.7251952886581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06557617336511612,
      "rewards/margins": 0.14323730766773224,
      "rewards/rejected": -0.20866699516773224,
      "step": 6800
    },
    {
      "epoch": 0.49586776859504134,
      "grad_norm": 2.6144663201308975,
      "learning_rate": 9.694299458808932e-07,
      "log_odds_chosen": 1.2844727039337158,
      "log_odds_ratio": -0.5077148675918579,
      "logits/chosen": -0.96875,
      "logits/rejected": -0.8814452886581421,
      "logps/chosen": -0.7149413824081421,
      "logps/rejected": -1.633203148841858,
      "loss": 0.7841,
      "nll_loss": 0.7583984136581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07147216796875,
      "rewards/margins": 0.09185485541820526,
      "rewards/rejected": -0.16328124701976776,
      "step": 6810
    },
    {
      "epoch": 0.49659591509811774,
      "grad_norm": 2.416490428560342,
      "learning_rate": 9.687189593929655e-07,
      "log_odds_chosen": 1.5353515148162842,
      "log_odds_ratio": -0.4383300840854645,
      "logits/chosen": -0.97265625,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.740039050579071,
      "logps/rejected": -1.8640625476837158,
      "loss": 0.7783,
      "nll_loss": 0.8525390625,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07398681342601776,
      "rewards/margins": 0.11235351860523224,
      "rewards/rejected": -0.18623046576976776,
      "step": 6820
    },
    {
      "epoch": 0.49732406160119413,
      "grad_norm": 2.316377823803131,
      "learning_rate": 9.680095349408789e-07,
      "log_odds_chosen": 1.530175805091858,
      "log_odds_ratio": -0.39311522245407104,
      "logits/chosen": -1.005468726158142,
      "logits/rejected": -0.9388672113418579,
      "logps/chosen": -0.6688476800918579,
      "logps/rejected": -1.719140648841858,
      "loss": 0.8055,
      "nll_loss": 0.819531261920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06685791164636612,
      "rewards/margins": 0.10508422553539276,
      "rewards/rejected": -0.17185059189796448,
      "step": 6830
    },
    {
      "epoch": 0.4980522081042706,
      "grad_norm": 19.79696481785,
      "learning_rate": 9.673016668133487e-07,
      "log_odds_chosen": 1.6560547351837158,
      "log_odds_ratio": -0.37114256620407104,
      "logits/chosen": -1.042382836341858,
      "logits/rejected": -0.896289050579071,
      "logps/chosen": -0.6224609613418579,
      "logps/rejected": -1.743749976158142,
      "loss": 0.7919,
      "nll_loss": 0.750683605670929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06228027492761612,
      "rewards/margins": 0.11217041313648224,
      "rewards/rejected": -0.17424316704273224,
      "step": 6840
    },
    {
      "epoch": 0.498780354607347,
      "grad_norm": 2.702906649248856,
      "learning_rate": 9.66595349328283e-07,
      "log_odds_chosen": 1.7101562023162842,
      "log_odds_ratio": -0.3731445372104645,
      "logits/chosen": -1.058203101158142,
      "logits/rejected": -0.8896484375,
      "logps/chosen": -0.6405273675918579,
      "logps/rejected": -1.83203125,
      "loss": 0.7874,
      "nll_loss": 0.735156238079071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.0640869140625,
      "rewards/margins": 0.11903686821460724,
      "rewards/rejected": -0.18320313096046448,
      "step": 6850
    },
    {
      "epoch": 0.49950850111042344,
      "grad_norm": 2.6792317275027506,
      "learning_rate": 9.658905768325902e-07,
      "log_odds_chosen": 1.5730469226837158,
      "log_odds_ratio": -0.394287109375,
      "logits/chosen": -1.0167968273162842,
      "logits/rejected": -0.9488281011581421,
      "logps/chosen": -0.647167980670929,
      "logps/rejected": -1.701171875,
      "loss": 0.7612,
      "nll_loss": 0.7040039300918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06472168117761612,
      "rewards/margins": 0.10546875,
      "rewards/rejected": -0.170166015625,
      "step": 6860
    },
    {
      "epoch": 0.5002366476134998,
      "grad_norm": 2.7161875993383338,
      "learning_rate": 9.651873437019902e-07,
      "log_odds_chosen": 1.7080078125,
      "log_odds_ratio": -0.3677734434604645,
      "logits/chosen": -1.0119140148162842,
      "logits/rejected": -0.887499988079071,
      "logps/chosen": -0.62744140625,
      "logps/rejected": -1.8273437023162842,
      "loss": 0.7791,
      "nll_loss": 0.7425781488418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06275634467601776,
      "rewards/margins": 0.12006835639476776,
      "rewards/rejected": -0.18271484971046448,
      "step": 6870
    },
    {
      "epoch": 0.5009647941165762,
      "grad_norm": 2.94746563464523,
      "learning_rate": 9.644856443408243e-07,
      "log_odds_chosen": 1.5188477039337158,
      "log_odds_ratio": -0.402099609375,
      "logits/chosen": -1.01953125,
      "logits/rejected": -0.882617175579071,
      "logps/chosen": -0.6597656011581421,
      "logps/rejected": -1.725976586341858,
      "loss": 0.7758,
      "nll_loss": 0.7953125238418579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06596679985523224,
      "rewards/margins": 0.10648498684167862,
      "rewards/rejected": -0.17270508408546448,
      "step": 6880
    },
    {
      "epoch": 0.5016929406196526,
      "grad_norm": 2.712158879024652,
      "learning_rate": 9.637854731818697e-07,
      "log_odds_chosen": 1.5043456554412842,
      "log_odds_ratio": -0.3942627012729645,
      "logits/chosen": -1.013281226158142,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -0.6124023199081421,
      "logps/rejected": -1.611328125,
      "loss": 0.8149,
      "nll_loss": 0.7798827886581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06121826171875,
      "rewards/margins": 0.09984131157398224,
      "rewards/rejected": -0.16103515028953552,
      "step": 6890
    },
    {
      "epoch": 0.502421087122729,
      "grad_norm": 2.6394306650577533,
      "learning_rate": 9.630868246861536e-07,
      "log_odds_chosen": 1.577734351158142,
      "log_odds_ratio": -0.38203126192092896,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.8890625238418579,
      "logps/chosen": -0.593457043170929,
      "logps/rejected": -1.649804711341858,
      "loss": 0.7666,
      "nll_loss": 0.703125,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05941162258386612,
      "rewards/margins": 0.105712890625,
      "rewards/rejected": -0.16496582329273224,
      "step": 6900
    },
    {
      "epoch": 0.5031492336258055,
      "grad_norm": 2.174885599829653,
      "learning_rate": 9.623896933427685e-07,
      "log_odds_chosen": 1.7498047351837158,
      "log_odds_ratio": -0.34941405057907104,
      "logits/chosen": -0.9931640625,
      "logits/rejected": -0.922656238079071,
      "logps/chosen": -0.6338866949081421,
      "logps/rejected": -1.83984375,
      "loss": 0.7667,
      "nll_loss": 0.7342773675918579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06341552734375,
      "rewards/margins": 0.12049560248851776,
      "rewards/rejected": -0.18398436903953552,
      "step": 6910
    },
    {
      "epoch": 0.503877380128882,
      "grad_norm": 2.347797717866109,
      "learning_rate": 9.6169407366869e-07,
      "log_odds_chosen": 1.5636718273162842,
      "log_odds_ratio": -0.37736815214157104,
      "logits/chosen": -0.9156249761581421,
      "logits/rejected": -0.8677734136581421,
      "logps/chosen": -0.6426757574081421,
      "logps/rejected": -1.7531249523162842,
      "loss": 0.7908,
      "nll_loss": 0.7886718511581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06424560397863388,
      "rewards/margins": 0.111083984375,
      "rewards/rejected": -0.17534179985523224,
      "step": 6920
    },
    {
      "epoch": 0.5046055266319583,
      "grad_norm": 2.5261443762952047,
      "learning_rate": 9.609999602085963e-07,
      "log_odds_chosen": 1.817285180091858,
      "log_odds_ratio": -0.3597168028354645,
      "logits/chosen": -1.009179711341858,
      "logits/rejected": -0.908398449420929,
      "logps/chosen": -0.6470702886581421,
      "logps/rejected": -1.9421875476837158,
      "loss": 0.798,
      "nll_loss": 0.833789050579071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06472168117761612,
      "rewards/margins": 0.12961426377296448,
      "rewards/rejected": -0.19426269829273224,
      "step": 6930
    },
    {
      "epoch": 0.5053336731350347,
      "grad_norm": 3.062622774475941,
      "learning_rate": 9.603073475346872e-07,
      "log_odds_chosen": 1.747460961341858,
      "log_odds_ratio": -0.3663085997104645,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.888476550579071,
      "logps/chosen": -0.616015613079071,
      "logps/rejected": -1.8058593273162842,
      "loss": 0.7493,
      "nll_loss": 0.756054699420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06165771558880806,
      "rewards/margins": 0.11882934719324112,
      "rewards/rejected": -0.18049316108226776,
      "step": 6940
    },
    {
      "epoch": 0.5060618196381111,
      "grad_norm": 2.47877709943063,
      "learning_rate": 9.596162302465074e-07,
      "log_odds_chosen": 1.7902343273162842,
      "log_odds_ratio": -0.32758790254592896,
      "logits/chosen": -0.990234375,
      "logits/rejected": -0.8685547113418579,
      "logps/chosen": -0.572070300579071,
      "logps/rejected": -1.8093750476837158,
      "loss": 0.7821,
      "nll_loss": 0.720410168170929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05722656100988388,
      "rewards/margins": 0.12370605766773224,
      "rewards/rejected": -0.18105468153953552,
      "step": 6950
    },
    {
      "epoch": 0.5067899661411877,
      "grad_norm": 2.485096638653835,
      "learning_rate": 9.589266029707683e-07,
      "log_odds_chosen": 1.879003882408142,
      "log_odds_ratio": -0.36188966035842896,
      "logits/chosen": -1.032617211341858,
      "logits/rejected": -0.904296875,
      "logps/chosen": -0.657519519329071,
      "logps/rejected": -2.014453172683716,
      "loss": 0.783,
      "nll_loss": 0.747851550579071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06574706733226776,
      "rewards/margins": 0.13569335639476776,
      "rewards/rejected": -0.20144042372703552,
      "step": 6960
    },
    {
      "epoch": 0.507518112644264,
      "grad_norm": 3.0260853927996623,
      "learning_rate": 9.582384603611731e-07,
      "log_odds_chosen": 1.7419922351837158,
      "log_odds_ratio": -0.36298829317092896,
      "logits/chosen": -0.9642578363418579,
      "logits/rejected": -0.8695312738418579,
      "logps/chosen": -0.691210925579071,
      "logps/rejected": -1.927343726158142,
      "loss": 0.7857,
      "nll_loss": 0.7837890386581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06912841647863388,
      "rewards/margins": 0.12355957180261612,
      "rewards/rejected": -0.19272461533546448,
      "step": 6970
    },
    {
      "epoch": 0.5082462591473405,
      "grad_norm": 3.2845152600434844,
      "learning_rate": 9.575517970982428e-07,
      "log_odds_chosen": 1.7921874523162842,
      "log_odds_ratio": -0.359375,
      "logits/chosen": -0.9671875238418579,
      "logits/rejected": -0.8373047113418579,
      "logps/chosen": -0.6583007574081421,
      "logps/rejected": -1.960546851158142,
      "loss": 0.7916,
      "nll_loss": 0.784960925579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06586913764476776,
      "rewards/margins": 0.13021239638328552,
      "rewards/rejected": -0.196044921875,
      "step": 6980
    },
    {
      "epoch": 0.5089744056504169,
      "grad_norm": 3.1348359346610235,
      "learning_rate": 9.568666078891436e-07,
      "log_odds_chosen": 1.494116187095642,
      "log_odds_ratio": -0.39228516817092896,
      "logits/chosen": -1.0007812976837158,
      "logits/rejected": -0.8681640625,
      "logps/chosen": -0.6751953363418579,
      "logps/rejected": -1.735937476158142,
      "loss": 0.7896,
      "nll_loss": 0.856249988079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0675048828125,
      "rewards/margins": 0.10592193901538849,
      "rewards/rejected": -0.17351074516773224,
      "step": 6990
    },
    {
      "epoch": 0.5097025521534932,
      "grad_norm": 2.76447619315592,
      "learning_rate": 9.561828874675149e-07,
      "log_odds_chosen": 1.521484375,
      "log_odds_ratio": -0.39433592557907104,
      "logits/chosen": -1.008203148841858,
      "logits/rejected": -0.8701171875,
      "logps/chosen": -0.6304687261581421,
      "logps/rejected": -1.633398413658142,
      "loss": 0.7678,
      "nll_loss": 0.7490234375,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06306152045726776,
      "rewards/margins": 0.100189208984375,
      "rewards/rejected": -0.16328124701976776,
      "step": 7000
    },
    {
      "epoch": 0.5104306986565696,
      "grad_norm": 2.787969636043256,
      "learning_rate": 9.555006305933e-07,
      "log_odds_chosen": 1.789648413658142,
      "log_odds_ratio": -0.37126463651657104,
      "logits/chosen": -0.967578113079071,
      "logits/rejected": -0.8736327886581421,
      "logps/chosen": -0.628125011920929,
      "logps/rejected": -1.912109375,
      "loss": 0.79,
      "nll_loss": 0.7623046636581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06287841498851776,
      "rewards/margins": 0.12841796875,
      "rewards/rejected": -0.19121094048023224,
      "step": 7010
    },
    {
      "epoch": 0.5111588451596462,
      "grad_norm": 2.6890610375868853,
      "learning_rate": 9.548198320525771e-07,
      "log_odds_chosen": 1.433496117591858,
      "log_odds_ratio": -0.44999998807907104,
      "logits/chosen": -1.018945336341858,
      "logits/rejected": -0.884570300579071,
      "logps/chosen": -0.677929699420929,
      "logps/rejected": -1.7150390148162842,
      "loss": 0.7798,
      "nll_loss": 0.7347656488418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06787109375,
      "rewards/margins": 0.10375366359949112,
      "rewards/rejected": -0.17148438096046448,
      "step": 7020
    },
    {
      "epoch": 0.5118869916627226,
      "grad_norm": 3.0645612120258585,
      "learning_rate": 9.54140486657392e-07,
      "log_odds_chosen": 1.5546875,
      "log_odds_ratio": -0.38720703125,
      "logits/chosen": -1.0144531726837158,
      "logits/rejected": -0.889453113079071,
      "logps/chosen": -0.652539074420929,
      "logps/rejected": -1.765625,
      "loss": 0.7683,
      "nll_loss": 0.7093750238418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06528320163488388,
      "rewards/margins": 0.11124267429113388,
      "rewards/rejected": -0.17658691108226776,
      "step": 7030
    },
    {
      "epoch": 0.512615138165799,
      "grad_norm": 2.7594799084353596,
      "learning_rate": 9.534625892455922e-07,
      "log_odds_chosen": 1.6123046875,
      "log_odds_ratio": -0.4106689393520355,
      "logits/chosen": -0.986523449420929,
      "logits/rejected": -0.9267578125,
      "logps/chosen": -0.66650390625,
      "logps/rejected": -1.814062476158142,
      "loss": 0.7832,
      "nll_loss": 0.8041015863418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06655273586511612,
      "rewards/margins": 0.11490478366613388,
      "rewards/rejected": -0.1815185546875,
      "step": 7040
    },
    {
      "epoch": 0.5133432846688754,
      "grad_norm": 2.6227700550536395,
      "learning_rate": 9.527861346806618e-07,
      "log_odds_chosen": 1.611328125,
      "log_odds_ratio": -0.37788087129592896,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.881054699420929,
      "logps/chosen": -0.6996093988418579,
      "logps/rejected": -1.8259766101837158,
      "loss": 0.7865,
      "nll_loss": 0.7939453125,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06997070461511612,
      "rewards/margins": 0.11255493015050888,
      "rewards/rejected": -0.18232421576976776,
      "step": 7050
    },
    {
      "epoch": 0.5140714311719518,
      "grad_norm": 2.6444206990048555,
      "learning_rate": 9.521111178515582e-07,
      "log_odds_chosen": 1.605078101158142,
      "log_odds_ratio": -0.3951171934604645,
      "logits/chosen": -0.9775390625,
      "logits/rejected": -0.8516601324081421,
      "logps/chosen": -0.714160144329071,
      "logps/rejected": -1.881250023841858,
      "loss": 0.8055,
      "nll_loss": 0.8359375,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07147216796875,
      "rewards/margins": 0.11663818359375,
      "rewards/rejected": -0.18798828125,
      "step": 7060
    },
    {
      "epoch": 0.5147995776750283,
      "grad_norm": 2.4346564324371287,
      "learning_rate": 9.514375336725502e-07,
      "log_odds_chosen": 1.715917944908142,
      "log_odds_ratio": -0.39384764432907104,
      "logits/chosen": -0.947460949420929,
      "logits/rejected": -0.8335937261581421,
      "logps/chosen": -0.657421886920929,
      "logps/rejected": -1.9015624523162842,
      "loss": 0.7684,
      "nll_loss": 0.721484363079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06574706733226776,
      "rewards/margins": 0.12456054985523224,
      "rewards/rejected": -0.1903076171875,
      "step": 7070
    },
    {
      "epoch": 0.5155277241781047,
      "grad_norm": 3.6399756361581717,
      "learning_rate": 9.507653770830566e-07,
      "log_odds_chosen": 1.6277344226837158,
      "log_odds_ratio": -0.3661132752895355,
      "logits/chosen": -0.998828113079071,
      "logits/rejected": -0.878125011920929,
      "logps/chosen": -0.6363281011581421,
      "logps/rejected": -1.7527344226837158,
      "loss": 0.7786,
      "nll_loss": 0.701367199420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06361083686351776,
      "rewards/margins": 0.11152343451976776,
      "rewards/rejected": -0.1751708984375,
      "step": 7080
    },
    {
      "epoch": 0.5162558706811811,
      "grad_norm": 3.012748533233533,
      "learning_rate": 9.500946430474869e-07,
      "log_odds_chosen": 1.667578101158142,
      "log_odds_ratio": -0.3998046815395355,
      "logits/chosen": -0.948437511920929,
      "logits/rejected": -0.890820324420929,
      "logps/chosen": -0.6026366949081421,
      "logps/rejected": -1.763085961341858,
      "loss": 0.7542,
      "nll_loss": 0.6666015386581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06026611477136612,
      "rewards/margins": 0.1162109375,
      "rewards/rejected": -0.1763916015625,
      "step": 7090
    },
    {
      "epoch": 0.5169840171842575,
      "grad_norm": 2.6434767090844296,
      "learning_rate": 9.494253265550825e-07,
      "log_odds_chosen": 2.03369140625,
      "log_odds_ratio": -0.3174804747104645,
      "logits/chosen": -0.9359375238418579,
      "logits/rejected": -0.824023425579071,
      "logps/chosen": -0.592578113079071,
      "logps/rejected": -2.030468702316284,
      "loss": 0.7797,
      "nll_loss": 0.735644519329071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05930175632238388,
      "rewards/margins": 0.14384765923023224,
      "rewards/rejected": -0.20305176079273224,
      "step": 7100
    },
    {
      "epoch": 0.5177121636873339,
      "grad_norm": 2.695050500910221,
      "learning_rate": 9.4875742261976e-07,
      "log_odds_chosen": 1.8271484375,
      "log_odds_ratio": -0.345947265625,
      "logits/chosen": -0.966992199420929,
      "logits/rejected": -0.827343761920929,
      "logps/chosen": -0.630566418170929,
      "logps/rejected": -1.939843773841858,
      "loss": 0.773,
      "nll_loss": 0.72265625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06307373195886612,
      "rewards/margins": 0.13103027641773224,
      "rewards/rejected": -0.19404296576976776,
      "step": 7110
    },
    {
      "epoch": 0.5184403101904103,
      "grad_norm": 3.0924837399908025,
      "learning_rate": 9.480909262799544e-07,
      "log_odds_chosen": 1.6745116710662842,
      "log_odds_ratio": -0.37397462129592896,
      "logits/chosen": -0.9740234613418579,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.7049804925918579,
      "logps/rejected": -1.8605468273162842,
      "loss": 0.8061,
      "nll_loss": 0.806835949420929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.07044677436351776,
      "rewards/margins": 0.115447998046875,
      "rewards/rejected": -0.18593749403953552,
      "step": 7120
    },
    {
      "epoch": 0.5191684566934868,
      "grad_norm": 2.4761898251233756,
      "learning_rate": 9.47425832598465e-07,
      "log_odds_chosen": 1.6911132335662842,
      "log_odds_ratio": -0.35273438692092896,
      "logits/chosen": -1.0359375476837158,
      "logits/rejected": -0.8744140863418579,
      "logps/chosen": -0.632617175579071,
      "logps/rejected": -1.8376953601837158,
      "loss": 0.7558,
      "nll_loss": 0.6934570074081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06320800632238388,
      "rewards/margins": 0.12045898288488388,
      "rewards/rejected": -0.18361815810203552,
      "step": 7130
    },
    {
      "epoch": 0.5198966031965632,
      "grad_norm": 2.5547772363254793,
      "learning_rate": 9.467621366623017e-07,
      "log_odds_chosen": 1.6383056640625,
      "log_odds_ratio": -0.40375977754592896,
      "logits/chosen": -0.9603515863418579,
      "logits/rejected": -0.8619140386581421,
      "logps/chosen": -0.6239258050918579,
      "logps/rejected": -1.766210913658142,
      "loss": 0.7746,
      "nll_loss": 0.747265636920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06242675706744194,
      "rewards/margins": 0.11422272026538849,
      "rewards/rejected": -0.17673340439796448,
      "step": 7140
    },
    {
      "epoch": 0.5206247496996396,
      "grad_norm": 2.6249236757719943,
      "learning_rate": 9.46099833582532e-07,
      "log_odds_chosen": 1.7257812023162842,
      "log_odds_ratio": -0.37646484375,
      "logits/chosen": -0.9931640625,
      "logits/rejected": -0.842578113079071,
      "logps/chosen": -0.65625,
      "logps/rejected": -1.855859398841858,
      "loss": 0.7888,
      "nll_loss": 0.739550769329071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06561279296875,
      "rewards/margins": 0.11979980766773224,
      "rewards/rejected": -0.1854248046875,
      "step": 7150
    },
    {
      "epoch": 0.521352896202716,
      "grad_norm": 2.616797707056822,
      "learning_rate": 9.45438918494131e-07,
      "log_odds_chosen": 1.619140625,
      "log_odds_ratio": -0.37578123807907104,
      "logits/chosen": -0.934765636920929,
      "logits/rejected": -0.8394531011581421,
      "logps/chosen": -0.6253906488418579,
      "logps/rejected": -1.7433593273162842,
      "loss": 0.7745,
      "nll_loss": 0.772656261920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06256103515625,
      "rewards/margins": 0.11176757514476776,
      "rewards/rejected": -0.17434081435203552,
      "step": 7160
    },
    {
      "epoch": 0.5220810427057924,
      "grad_norm": 2.248304663000216,
      "learning_rate": 9.447793865558291e-07,
      "log_odds_chosen": 1.623437523841858,
      "log_odds_ratio": -0.3822998106479645,
      "logits/chosen": -0.9730468988418579,
      "logits/rejected": -0.9117187261581421,
      "logps/chosen": -0.6107422113418579,
      "logps/rejected": -1.729882836341858,
      "loss": 0.788,
      "nll_loss": 0.747265636920929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06113281100988388,
      "rewards/margins": 0.11187744140625,
      "rewards/rejected": -0.17304687201976776,
      "step": 7170
    },
    {
      "epoch": 0.5228091892088689,
      "grad_norm": 2.6686718938391882,
      "learning_rate": 9.441212329499659e-07,
      "log_odds_chosen": 1.4019043445587158,
      "log_odds_ratio": -0.39873045682907104,
      "logits/chosen": -1.0119140148162842,
      "logits/rejected": -0.873046875,
      "logps/chosen": -0.657421886920929,
      "logps/rejected": -1.6072266101837158,
      "loss": 0.792,
      "nll_loss": 0.706738293170929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06575927883386612,
      "rewards/margins": 0.0949806198477745,
      "rewards/rejected": -0.16074219346046448,
      "step": 7180
    },
    {
      "epoch": 0.5235373357119453,
      "grad_norm": 2.3611393686836464,
      "learning_rate": 9.434644528823399e-07,
      "log_odds_chosen": 1.5719726085662842,
      "log_odds_ratio": -0.39726561307907104,
      "logits/chosen": -0.9810546636581421,
      "logits/rejected": -0.8589843511581421,
      "logps/chosen": -0.653613269329071,
      "logps/rejected": -1.7351562976837158,
      "loss": 0.7767,
      "nll_loss": 0.7349609136581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06534423679113388,
      "rewards/margins": 0.10825500637292862,
      "rewards/rejected": -0.173583984375,
      "step": 7190
    },
    {
      "epoch": 0.5242654822150217,
      "grad_norm": 2.9129728548642517,
      "learning_rate": 9.428090415820634e-07,
      "log_odds_chosen": 1.503808617591858,
      "log_odds_ratio": -0.4127441346645355,
      "logits/chosen": -1.017968773841858,
      "logits/rejected": -0.9136718511581421,
      "logps/chosen": -0.6578124761581421,
      "logps/rejected": -1.725195288658142,
      "loss": 0.7859,
      "nll_loss": 0.6968749761581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06574706733226776,
      "rewards/margins": 0.10673828423023224,
      "rewards/rejected": -0.17250975966453552,
      "step": 7200
    },
    {
      "epoch": 0.5249936287180981,
      "grad_norm": 2.8737287228889725,
      "learning_rate": 9.42154994301416e-07,
      "log_odds_chosen": 1.6320312023162842,
      "log_odds_ratio": -0.326416015625,
      "logits/chosen": -0.9849609136581421,
      "logits/rejected": -0.866406261920929,
      "logps/chosen": -0.619335949420929,
      "logps/rejected": -1.7314453125,
      "loss": 0.7735,
      "nll_loss": 0.751269519329071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06193847581744194,
      "rewards/margins": 0.11115722358226776,
      "rewards/rejected": -0.17319336533546448,
      "step": 7210
    },
    {
      "epoch": 0.5257217752211745,
      "grad_norm": 2.609387374777836,
      "learning_rate": 9.415023063157008e-07,
      "log_odds_chosen": 1.385156273841858,
      "log_odds_ratio": -0.4214843809604645,
      "logits/chosen": -0.9818359613418579,
      "logits/rejected": -0.888867199420929,
      "logps/chosen": -0.6656249761581421,
      "logps/rejected": -1.602148413658142,
      "loss": 0.7609,
      "nll_loss": 0.7417968511581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06655273586511612,
      "rewards/margins": 0.09357605129480362,
      "rewards/rejected": -0.16013184189796448,
      "step": 7220
    },
    {
      "epoch": 0.5264499217242509,
      "grad_norm": 2.5944689662545843,
      "learning_rate": 9.408509729231009e-07,
      "log_odds_chosen": 1.5574219226837158,
      "log_odds_ratio": -0.36181640625,
      "logits/chosen": -0.964062511920929,
      "logits/rejected": -0.8734375238418579,
      "logps/chosen": -0.663769543170929,
      "logps/rejected": -1.7410156726837158,
      "loss": 0.7802,
      "nll_loss": 0.713085949420929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06635741889476776,
      "rewards/margins": 0.10769043117761612,
      "rewards/rejected": -0.17414550483226776,
      "step": 7230
    },
    {
      "epoch": 0.5271780682273274,
      "grad_norm": 2.4878136964921267,
      "learning_rate": 9.402009894445369e-07,
      "log_odds_chosen": 1.4489257335662842,
      "log_odds_ratio": -0.4105468690395355,
      "logits/chosen": -0.9632812738418579,
      "logits/rejected": -0.862109363079071,
      "logps/chosen": -0.6304687261581421,
      "logps/rejected": -1.642968773841858,
      "loss": 0.7728,
      "nll_loss": 0.7826172113418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06307373195886612,
      "rewards/margins": 0.10131225734949112,
      "rewards/rejected": -0.16425780951976776,
      "step": 7240
    },
    {
      "epoch": 0.5279062147304038,
      "grad_norm": 2.3268206502662334,
      "learning_rate": 9.395523512235255e-07,
      "log_odds_chosen": 1.5632812976837158,
      "log_odds_ratio": -0.39619141817092896,
      "logits/chosen": -0.9341796636581421,
      "logits/rejected": -0.899609386920929,
      "logps/chosen": -0.6446288824081421,
      "logps/rejected": -1.776757836341858,
      "loss": 0.7738,
      "nll_loss": 0.730761706829071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06447754055261612,
      "rewards/margins": 0.11307983100414276,
      "rewards/rejected": -0.177490234375,
      "step": 7250
    },
    {
      "epoch": 0.5286343612334802,
      "grad_norm": 2.5407858089038364,
      "learning_rate": 9.389050536260404e-07,
      "log_odds_chosen": 1.6875,
      "log_odds_ratio": -0.3399414122104645,
      "logits/chosen": -0.965624988079071,
      "logits/rejected": -0.868359386920929,
      "logps/chosen": -0.6412109136581421,
      "logps/rejected": -1.7634766101837158,
      "loss": 0.754,
      "nll_loss": 0.669921875,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06405029445886612,
      "rewards/margins": 0.11226196587085724,
      "rewards/rejected": -0.17646484076976776,
      "step": 7260
    },
    {
      "epoch": 0.5293625077365566,
      "grad_norm": 2.9685344037358914,
      "learning_rate": 9.382590920403722e-07,
      "log_odds_chosen": 1.8103516101837158,
      "log_odds_ratio": -0.3704589903354645,
      "logits/chosen": -0.959765613079071,
      "logits/rejected": -0.8558593988418579,
      "logps/chosen": -0.615039050579071,
      "logps/rejected": -1.900781273841858,
      "loss": 0.7725,
      "nll_loss": 0.7295898199081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0615234375,
      "rewards/margins": 0.128498837351799,
      "rewards/rejected": -0.19016113877296448,
      "step": 7270
    },
    {
      "epoch": 0.530090654239633,
      "grad_norm": 6.043582036745641,
      "learning_rate": 9.376144618769908e-07,
      "log_odds_chosen": 1.56982421875,
      "log_odds_ratio": -0.4139160215854645,
      "logits/chosen": -0.9810546636581421,
      "logits/rejected": -0.8544921875,
      "logps/chosen": -0.61572265625,
      "logps/rejected": -1.720117211341858,
      "loss": 0.7517,
      "nll_loss": 0.68505859375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06151122972369194,
      "rewards/margins": 0.11052856594324112,
      "rewards/rejected": -0.17221680283546448,
      "step": 7280
    },
    {
      "epoch": 0.5308188007427095,
      "grad_norm": 2.678585493522053,
      "learning_rate": 9.369711585684086e-07,
      "log_odds_chosen": 1.7418944835662842,
      "log_odds_ratio": -0.33491212129592896,
      "logits/chosen": -1.0046875476837158,
      "logits/rejected": -0.8812500238418579,
      "logps/chosen": -0.6446288824081421,
      "logps/rejected": -1.8937499523162842,
      "loss": 0.7777,
      "nll_loss": 0.78271484375,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06450195610523224,
      "rewards/margins": 0.125,
      "rewards/rejected": -0.18959960341453552,
      "step": 7290
    },
    {
      "epoch": 0.5315469472457859,
      "grad_norm": 3.0904781296668427,
      "learning_rate": 9.363291775690445e-07,
      "log_odds_chosen": 1.5200684070587158,
      "log_odds_ratio": -0.4256347715854645,
      "logits/chosen": -0.9814453125,
      "logits/rejected": -0.868359386920929,
      "logps/chosen": -0.670214831829071,
      "logps/rejected": -1.7119140625,
      "loss": 0.7489,
      "nll_loss": 0.758984386920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06700439751148224,
      "rewards/margins": 0.10410461574792862,
      "rewards/rejected": -0.171142578125,
      "step": 7300
    },
    {
      "epoch": 0.5322750937488623,
      "grad_norm": 2.9523673265575687,
      "learning_rate": 9.356885143550886e-07,
      "log_odds_chosen": 1.6042969226837158,
      "log_odds_ratio": -0.37700194120407104,
      "logits/chosen": -1.004296898841858,
      "logits/rejected": -0.922656238079071,
      "logps/chosen": -0.601757824420929,
      "logps/rejected": -1.689453125,
      "loss": 0.7665,
      "nll_loss": 0.749218761920929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06014404445886612,
      "rewards/margins": 0.10865478217601776,
      "rewards/rejected": -0.16889648139476776,
      "step": 7310
    },
    {
      "epoch": 0.5330032402519387,
      "grad_norm": 2.8150989495689163,
      "learning_rate": 9.350491644243688e-07,
      "log_odds_chosen": 1.6564452648162842,
      "log_odds_ratio": -0.3947509825229645,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.8792968988418579,
      "logps/chosen": -0.6826171875,
      "logps/rejected": -1.882226586341858,
      "loss": 0.7774,
      "nll_loss": 0.7220703363418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06826172024011612,
      "rewards/margins": 0.119873046875,
      "rewards/rejected": -0.18808594346046448,
      "step": 7320
    },
    {
      "epoch": 0.5337313867550151,
      "grad_norm": 2.776709743953002,
      "learning_rate": 9.344111232962179e-07,
      "log_odds_chosen": 1.4716796875,
      "log_odds_ratio": -0.412353515625,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.863476574420929,
      "logps/chosen": -0.713085949420929,
      "logps/rejected": -1.745703101158142,
      "loss": 0.7496,
      "nll_loss": 0.6832031011581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07136230170726776,
      "rewards/margins": 0.10333862155675888,
      "rewards/rejected": -0.17473144829273224,
      "step": 7330
    },
    {
      "epoch": 0.5344595332580915,
      "grad_norm": 2.5481846645600608,
      "learning_rate": 9.337743865113415e-07,
      "log_odds_chosen": 1.398828148841858,
      "log_odds_ratio": -0.417236328125,
      "logits/chosen": -1.0021483898162842,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.65625,
      "logps/rejected": -1.6242187023162842,
      "loss": 0.7674,
      "nll_loss": 0.744921863079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.065643310546875,
      "rewards/margins": 0.09676513820886612,
      "rewards/rejected": -0.16252441704273224,
      "step": 7340
    },
    {
      "epoch": 0.535187679761168,
      "grad_norm": 2.7615639636085914,
      "learning_rate": 9.331389496316868e-07,
      "log_odds_chosen": 1.7820312976837158,
      "log_odds_ratio": -0.35332030057907104,
      "logits/chosen": -0.9759765863418579,
      "logits/rejected": -0.868359386920929,
      "logps/chosen": -0.6236327886581421,
      "logps/rejected": -1.899999976158142,
      "loss": 0.7495,
      "nll_loss": 0.716113269329071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06236572191119194,
      "rewards/margins": 0.12758788466453552,
      "rewards/rejected": -0.19003906846046448,
      "step": 7350
    },
    {
      "epoch": 0.5359158262642444,
      "grad_norm": 2.7452415497518143,
      "learning_rate": 9.325048082403138e-07,
      "log_odds_chosen": 1.5705077648162842,
      "log_odds_ratio": -0.39545899629592896,
      "logits/chosen": -0.978320300579071,
      "logits/rejected": -0.858593761920929,
      "logps/chosen": -0.665234386920929,
      "logps/rejected": -1.796875,
      "loss": 0.7507,
      "nll_loss": 0.71240234375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06651611626148224,
      "rewards/margins": 0.11311034858226776,
      "rewards/rejected": -0.17971190810203552,
      "step": 7360
    },
    {
      "epoch": 0.5366439727673208,
      "grad_norm": 2.725235746939542,
      "learning_rate": 9.318719579412648e-07,
      "log_odds_chosen": 1.8035156726837158,
      "log_odds_ratio": -0.33955079317092896,
      "logits/chosen": -1.0119140148162842,
      "logits/rejected": -0.8759765625,
      "logps/chosen": -0.60205078125,
      "logps/rejected": -1.8914062976837158,
      "loss": 0.7726,
      "nll_loss": 0.66455078125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06022949144244194,
      "rewards/margins": 0.12902221083641052,
      "rewards/rejected": -0.189208984375,
      "step": 7370
    },
    {
      "epoch": 0.5373721192703972,
      "grad_norm": 3.006964226858241,
      "learning_rate": 9.312403943594374e-07,
      "log_odds_chosen": 1.6570312976837158,
      "log_odds_ratio": -0.3499511778354645,
      "logits/chosen": -1.0021483898162842,
      "logits/rejected": -0.8617187738418579,
      "logps/chosen": -0.6385742425918579,
      "logps/rejected": -1.7937500476837158,
      "loss": 0.7633,
      "nll_loss": 0.7515624761581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06392822414636612,
      "rewards/margins": 0.11552734673023224,
      "rewards/rejected": -0.17954102158546448,
      "step": 7380
    },
    {
      "epoch": 0.5381002657734736,
      "grad_norm": 2.5700074635237784,
      "learning_rate": 9.306101131404582e-07,
      "log_odds_chosen": 1.3251953125,
      "log_odds_ratio": -0.40571290254592896,
      "logits/chosen": -1.000585913658142,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.634570300579071,
      "logps/rejected": -1.5324218273162842,
      "loss": 0.739,
      "nll_loss": 0.660351574420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.0634765625,
      "rewards/margins": 0.08984069526195526,
      "rewards/rejected": -0.1533203125,
      "step": 7390
    },
    {
      "epoch": 0.5388284122765501,
      "grad_norm": 2.796018741190052,
      "learning_rate": 9.299811099505542e-07,
      "log_odds_chosen": 1.637109398841858,
      "log_odds_ratio": -0.3953857421875,
      "logits/chosen": -0.9947265386581421,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.658496081829071,
      "logps/rejected": -1.831445336341858,
      "loss": 0.7651,
      "nll_loss": 0.7056640386581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06586913764476776,
      "rewards/margins": 0.11719970405101776,
      "rewards/rejected": -0.18325194716453552,
      "step": 7400
    },
    {
      "epoch": 0.5395565587796265,
      "grad_norm": 3.1080837019517986,
      "learning_rate": 9.293533804764305e-07,
      "log_odds_chosen": 1.5099608898162842,
      "log_odds_ratio": -0.3875488340854645,
      "logits/chosen": -0.997265636920929,
      "logits/rejected": -0.8716796636581421,
      "logps/chosen": -0.660449206829071,
      "logps/rejected": -1.7097656726837158,
      "loss": 0.7651,
      "nll_loss": 0.757617175579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06606445461511612,
      "rewards/margins": 0.10483398288488388,
      "rewards/rejected": -0.1708984375,
      "step": 7410
    },
    {
      "epoch": 0.5402847052827029,
      "grad_norm": 2.6257277851090595,
      "learning_rate": 9.28726920425144e-07,
      "log_odds_chosen": 1.6935546398162842,
      "log_odds_ratio": -0.363525390625,
      "logits/chosen": -0.9613281488418579,
      "logits/rejected": -0.8744140863418579,
      "logps/chosen": -0.6587890386581421,
      "logps/rejected": -1.8484375476837158,
      "loss": 0.7693,
      "nll_loss": 0.7398437261581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06588134914636612,
      "rewards/margins": 0.11900635063648224,
      "rewards/rejected": -0.1849365234375,
      "step": 7420
    },
    {
      "epoch": 0.5410128517857793,
      "grad_norm": 2.3699868871934737,
      "learning_rate": 9.281017255239815e-07,
      "log_odds_chosen": 1.3624999523162842,
      "log_odds_ratio": -0.455322265625,
      "logits/chosen": -0.962109386920929,
      "logits/rejected": -0.894726574420929,
      "logps/chosen": -0.6673828363418579,
      "logps/rejected": -1.6287109851837158,
      "loss": 0.7654,
      "nll_loss": 0.7855468988418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06673584133386612,
      "rewards/margins": 0.09618224948644638,
      "rewards/rejected": -0.162841796875,
      "step": 7430
    },
    {
      "epoch": 0.5417409982888557,
      "grad_norm": 2.49870492608428,
      "learning_rate": 9.274777915203365e-07,
      "log_odds_chosen": 1.662500023841858,
      "log_odds_ratio": -0.3683105409145355,
      "logits/chosen": -0.9917968511581421,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.6351562738418579,
      "logps/rejected": -1.766015648841858,
      "loss": 0.7491,
      "nll_loss": 0.702832043170929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06348876655101776,
      "rewards/margins": 0.11318359524011612,
      "rewards/rejected": -0.17666015028953552,
      "step": 7440
    },
    {
      "epoch": 0.5424691447919321,
      "grad_norm": 2.5974047194398158,
      "learning_rate": 9.268551141815875e-07,
      "log_odds_chosen": 1.5364258289337158,
      "log_odds_ratio": -0.40043944120407104,
      "logits/chosen": -0.9769531488418579,
      "logits/rejected": -0.8695312738418579,
      "logps/chosen": -0.6312500238418579,
      "logps/rejected": -1.7248046398162842,
      "loss": 0.7519,
      "nll_loss": 0.740234375,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06314697116613388,
      "rewards/margins": 0.10942383110523224,
      "rewards/rejected": -0.1724853515625,
      "step": 7450
    },
    {
      "epoch": 0.5431972912950086,
      "grad_norm": 4.5453249817954235,
      "learning_rate": 9.262336892949784e-07,
      "log_odds_chosen": 1.715234398841858,
      "log_odds_ratio": -0.3482910096645355,
      "logits/chosen": -1.0177733898162842,
      "logits/rejected": -0.912109375,
      "logps/chosen": -0.6324218511581421,
      "logps/rejected": -1.875390648841858,
      "loss": 0.7589,
      "nll_loss": 0.686816394329071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.063232421875,
      "rewards/margins": 0.1243896484375,
      "rewards/rejected": -0.1875,
      "step": 7460
    },
    {
      "epoch": 0.543925437798085,
      "grad_norm": 4.902347507986452,
      "learning_rate": 9.256135126674977e-07,
      "log_odds_chosen": 1.741601586341858,
      "log_odds_ratio": -0.3426269590854645,
      "logits/chosen": -1.0041015148162842,
      "logits/rejected": -0.880859375,
      "logps/chosen": -0.648242175579071,
      "logps/rejected": -1.877343773841858,
      "loss": 0.7702,
      "nll_loss": 0.737500011920929,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06485595554113388,
      "rewards/margins": 0.12290649116039276,
      "rewards/rejected": -0.18781737983226776,
      "step": 7470
    },
    {
      "epoch": 0.5446535843011614,
      "grad_norm": 2.953517637037253,
      "learning_rate": 9.249945801257605e-07,
      "log_odds_chosen": 1.5587890148162842,
      "log_odds_ratio": -0.3671875,
      "logits/chosen": -1.0333983898162842,
      "logits/rejected": -0.9173828363418579,
      "logps/chosen": -0.652539074420929,
      "logps/rejected": -1.727148413658142,
      "loss": 0.7614,
      "nll_loss": 0.68798828125,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06522216647863388,
      "rewards/margins": 0.1075439453125,
      "rewards/rejected": -0.17270508408546448,
      "step": 7480
    },
    {
      "epoch": 0.5453817308042378,
      "grad_norm": 4.384610181064515,
      "learning_rate": 9.243768875158902e-07,
      "log_odds_chosen": 1.6980469226837158,
      "log_odds_ratio": -0.35302734375,
      "logits/chosen": -1.057031273841858,
      "logits/rejected": -0.922656238079071,
      "logps/chosen": -0.6412109136581421,
      "logps/rejected": -1.836328148841858,
      "loss": 0.7586,
      "nll_loss": 0.723828136920929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06403808295726776,
      "rewards/margins": 0.11947021633386612,
      "rewards/rejected": -0.18352051079273224,
      "step": 7490
    },
    {
      "epoch": 0.5461098773073142,
      "grad_norm": 2.6625057814992386,
      "learning_rate": 9.23760430703401e-07,
      "log_odds_chosen": 1.4200439453125,
      "log_odds_ratio": -0.4434570372104645,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.8658202886581421,
      "logps/chosen": -0.6205078363418579,
      "logps/rejected": -1.5578124523162842,
      "loss": 0.7505,
      "nll_loss": 0.6304687261581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06206054612994194,
      "rewards/margins": 0.09378967434167862,
      "rewards/rejected": -0.15581054985523224,
      "step": 7500
    },
    {
      "epoch": 0.5468380238103907,
      "grad_norm": 2.44135267075062,
      "learning_rate": 9.231452055730832e-07,
      "log_odds_chosen": 1.8212890625,
      "log_odds_ratio": -0.3304687440395355,
      "logits/chosen": -0.998828113079071,
      "logits/rejected": -0.903124988079071,
      "logps/chosen": -0.58740234375,
      "logps/rejected": -1.865625023841858,
      "loss": 0.7473,
      "nll_loss": 0.6812499761581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05878906324505806,
      "rewards/margins": 0.12791748344898224,
      "rewards/rejected": -0.18666991591453552,
      "step": 7510
    },
    {
      "epoch": 0.5475661703134671,
      "grad_norm": 2.7931340510240594,
      "learning_rate": 9.225312080288851e-07,
      "log_odds_chosen": 1.5939452648162842,
      "log_odds_ratio": -0.3650146424770355,
      "logits/chosen": -0.9970703125,
      "logits/rejected": -0.829882800579071,
      "logps/chosen": -0.658886730670929,
      "logps/rejected": -1.803125023841858,
      "loss": 0.7579,
      "nll_loss": 0.7281249761581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06593017280101776,
      "rewards/margins": 0.11458740383386612,
      "rewards/rejected": -0.180419921875,
      "step": 7520
    },
    {
      "epoch": 0.5482943168165435,
      "grad_norm": 2.9391532672398837,
      "learning_rate": 9.219184339938013e-07,
      "log_odds_chosen": 1.662207007408142,
      "log_odds_ratio": -0.37482911348342896,
      "logits/chosen": -1.0185546875,
      "logits/rejected": -0.8970702886581421,
      "logps/chosen": -0.6751953363418579,
      "logps/rejected": -1.8671875,
      "loss": 0.7839,
      "nll_loss": 0.7328125238418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06752929836511612,
      "rewards/margins": 0.11907958984375,
      "rewards/rejected": -0.1866455078125,
      "step": 7530
    },
    {
      "epoch": 0.5490224633196199,
      "grad_norm": 2.533565429326835,
      "learning_rate": 9.213068794097574e-07,
      "log_odds_chosen": 1.5480468273162842,
      "log_odds_ratio": -0.3841308653354645,
      "logits/chosen": -1.01171875,
      "logits/rejected": -0.887499988079071,
      "logps/chosen": -0.66357421875,
      "logps/rejected": -1.7449219226837158,
      "loss": 0.735,
      "nll_loss": 0.6910156011581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06639404594898224,
      "rewards/margins": 0.10806884616613388,
      "rewards/rejected": -0.17441406846046448,
      "step": 7540
    },
    {
      "epoch": 0.5497506098226963,
      "grad_norm": 2.5570837883933013,
      "learning_rate": 9.206965402374975e-07,
      "log_odds_chosen": 1.8232421875,
      "log_odds_ratio": -0.3169189393520355,
      "logits/chosen": -0.966796875,
      "logits/rejected": -0.849804699420929,
      "logps/chosen": -0.574999988079071,
      "logps/rejected": -1.798437476158142,
      "loss": 0.7704,
      "nll_loss": 0.7872070074081421,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.05744628980755806,
      "rewards/margins": 0.122314453125,
      "rewards/rejected": -0.17973633110523224,
      "step": 7550
    },
    {
      "epoch": 0.5504787563257727,
      "grad_norm": 2.2966108998008554,
      "learning_rate": 9.200874124564723e-07,
      "log_odds_chosen": 1.8439452648162842,
      "log_odds_ratio": -0.32441407442092896,
      "logits/chosen": -1.0388672351837158,
      "logits/rejected": -0.8939453363418579,
      "logps/chosen": -0.651660144329071,
      "logps/rejected": -1.9386718273162842,
      "loss": 0.7762,
      "nll_loss": 0.73388671875,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.065185546875,
      "rewards/margins": 0.1287841796875,
      "rewards/rejected": -0.1939697265625,
      "step": 7560
    },
    {
      "epoch": 0.5512069028288492,
      "grad_norm": 2.888097209020331,
      "learning_rate": 9.194794920647274e-07,
      "log_odds_chosen": 1.5390625,
      "log_odds_ratio": -0.3916992247104645,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.897265613079071,
      "logps/chosen": -0.6465820074081421,
      "logps/rejected": -1.739843726158142,
      "loss": 0.7546,
      "nll_loss": 0.7115234136581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06468506157398224,
      "rewards/margins": 0.10917969048023224,
      "rewards/rejected": -0.17385253310203552,
      "step": 7570
    },
    {
      "epoch": 0.5519350493319256,
      "grad_norm": 3.0410140142109245,
      "learning_rate": 9.188727750787932e-07,
      "log_odds_chosen": 1.734960913658142,
      "log_odds_ratio": -0.3530029356479645,
      "logits/chosen": -1.030859351158142,
      "logits/rejected": -0.9154297113418579,
      "logps/chosen": -0.7124999761581421,
      "logps/rejected": -1.9621093273162842,
      "loss": 0.7403,
      "nll_loss": 0.7123047113418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07127685844898224,
      "rewards/margins": 0.12492676079273224,
      "rewards/rejected": -0.19624023139476776,
      "step": 7580
    },
    {
      "epoch": 0.552663195835002,
      "grad_norm": 2.417712269571774,
      "learning_rate": 9.182672575335757e-07,
      "log_odds_chosen": 1.713281273841858,
      "log_odds_ratio": -0.35888671875,
      "logits/chosen": -0.982617199420929,
      "logits/rejected": -0.8822265863418579,
      "logps/chosen": -0.6727539300918579,
      "logps/rejected": -1.897851586341858,
      "loss": 0.7542,
      "nll_loss": 0.751757800579071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06732177734375,
      "rewards/margins": 0.1224365234375,
      "rewards/rejected": -0.18967285752296448,
      "step": 7590
    },
    {
      "epoch": 0.5533913423380784,
      "grad_norm": 2.6138375151654696,
      "learning_rate": 9.176629354822469e-07,
      "log_odds_chosen": 1.4845702648162842,
      "log_odds_ratio": -0.4111328125,
      "logits/chosen": -0.9986327886581421,
      "logits/rejected": -0.8902343511581421,
      "logps/chosen": -0.678417980670929,
      "logps/rejected": -1.727148413658142,
      "loss": 0.7866,
      "nll_loss": 0.7640625238418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06778564304113388,
      "rewards/margins": 0.10484619438648224,
      "rewards/rejected": -0.17277832329273224,
      "step": 7600
    },
    {
      "epoch": 0.5541194888411548,
      "grad_norm": 2.869757180445666,
      "learning_rate": 9.170598049961371e-07,
      "log_odds_chosen": 1.637109398841858,
      "log_odds_ratio": -0.33320313692092896,
      "logits/chosen": -1.0486328601837158,
      "logits/rejected": -0.916796863079071,
      "logps/chosen": -0.6302734613418579,
      "logps/rejected": -1.7273437976837158,
      "loss": 0.7686,
      "nll_loss": 0.744433581829071,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06300048530101776,
      "rewards/margins": 0.10981445014476776,
      "rewards/rejected": -0.17287597060203552,
      "step": 7610
    },
    {
      "epoch": 0.5548476353442312,
      "grad_norm": 2.520457237049284,
      "learning_rate": 9.164578621646276e-07,
      "log_odds_chosen": 1.779882788658142,
      "log_odds_ratio": -0.3412841856479645,
      "logits/chosen": -1.0470702648162842,
      "logits/rejected": -0.902148425579071,
      "logps/chosen": -0.6729491949081421,
      "logps/rejected": -1.952734351158142,
      "loss": 0.7665,
      "nll_loss": 0.7308593988418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06728515774011612,
      "rewards/margins": 0.12808838486671448,
      "rewards/rejected": -0.1953125,
      "step": 7620
    },
    {
      "epoch": 0.5555757818473077,
      "grad_norm": 2.603666470022784,
      "learning_rate": 9.15857103095044e-07,
      "log_odds_chosen": 1.6472656726837158,
      "log_odds_ratio": -0.3467773497104645,
      "logits/chosen": -1.0304687023162842,
      "logits/rejected": -0.9205077886581421,
      "logps/chosen": -0.6161133050918579,
      "logps/rejected": -1.783203125,
      "loss": 0.7769,
      "nll_loss": 0.722460925579071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06153564527630806,
      "rewards/margins": 0.11667480319738388,
      "rewards/rejected": -0.17819824814796448,
      "step": 7630
    },
    {
      "epoch": 0.5563039283503841,
      "grad_norm": 2.952748005683899,
      "learning_rate": 9.15257523912551e-07,
      "log_odds_chosen": 1.6823241710662842,
      "log_odds_ratio": -0.3843994140625,
      "logits/chosen": -1.0261719226837158,
      "logits/rejected": -0.896679699420929,
      "logps/chosen": -0.683886706829071,
      "logps/rejected": -1.8566405773162842,
      "loss": 0.7586,
      "nll_loss": 0.7464843988418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.068359375,
      "rewards/margins": 0.11727295070886612,
      "rewards/rejected": -0.1856689453125,
      "step": 7640
    },
    {
      "epoch": 0.5570320748534605,
      "grad_norm": 3.191693246698705,
      "learning_rate": 9.146591207600472e-07,
      "log_odds_chosen": 1.710546851158142,
      "log_odds_ratio": -0.36625975370407104,
      "logits/chosen": -1.0185546875,
      "logits/rejected": -0.9164062738418579,
      "logps/chosen": -0.646484375,
      "logps/rejected": -1.8767578601837158,
      "loss": 0.7451,
      "nll_loss": 0.6639648675918579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06461181491613388,
      "rewards/margins": 0.12310180813074112,
      "rewards/rejected": -0.18771973252296448,
      "step": 7650
    },
    {
      "epoch": 0.5577602213565369,
      "grad_norm": 3.2740499796952727,
      "learning_rate": 9.140618897980601e-07,
      "log_odds_chosen": 1.572265625,
      "log_odds_ratio": -0.3704589903354645,
      "logits/chosen": -1.044335961341858,
      "logits/rejected": -0.9134765863418579,
      "logps/chosen": -0.630078136920929,
      "logps/rejected": -1.7355468273162842,
      "loss": 0.7739,
      "nll_loss": 0.70751953125,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06303711235523224,
      "rewards/margins": 0.11052856594324112,
      "rewards/rejected": -0.17353515326976776,
      "step": 7660
    },
    {
      "epoch": 0.5584883678596133,
      "grad_norm": 3.400274594872579,
      "learning_rate": 9.134658272046442e-07,
      "log_odds_chosen": 1.8337891101837158,
      "log_odds_ratio": -0.3326660096645355,
      "logits/chosen": -0.9750000238418579,
      "logits/rejected": -0.8746093511581421,
      "logps/chosen": -0.595996081829071,
      "logps/rejected": -1.8347656726837158,
      "loss": 0.7968,
      "nll_loss": 0.7333984375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05959472805261612,
      "rewards/margins": 0.12392578274011612,
      "rewards/rejected": -0.18361815810203552,
      "step": 7670
    },
    {
      "epoch": 0.5592165143626898,
      "grad_norm": 2.7835046904406995,
      "learning_rate": 9.128709291752768e-07,
      "log_odds_chosen": 1.876953125,
      "log_odds_ratio": -0.3768554627895355,
      "logits/chosen": -0.9681640863418579,
      "logits/rejected": -0.8880859613418579,
      "logps/chosen": -0.6302734613418579,
      "logps/rejected": -1.986718773841858,
      "loss": 0.7644,
      "nll_loss": 0.7421875,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06308593600988388,
      "rewards/margins": 0.13563232123851776,
      "rewards/rejected": -0.19882813096046448,
      "step": 7680
    },
    {
      "epoch": 0.5599446608657662,
      "grad_norm": 2.7941960774889543,
      "learning_rate": 9.122771919227568e-07,
      "log_odds_chosen": 2.003124952316284,
      "log_odds_ratio": -0.30925291776657104,
      "logits/chosen": -1.056249976158142,
      "logits/rejected": -0.924609363079071,
      "logps/chosen": -0.6187499761581421,
      "logps/rejected": -2.081249952316284,
      "loss": 0.7622,
      "nll_loss": 0.652539074420929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06187744066119194,
      "rewards/margins": 0.14616699516773224,
      "rewards/rejected": -0.2078857421875,
      "step": 7690
    },
    {
      "epoch": 0.5606728073688426,
      "grad_norm": 2.894570640452244,
      "learning_rate": 9.116846116771035e-07,
      "log_odds_chosen": 1.4879882335662842,
      "log_odds_ratio": -0.45063477754592896,
      "logits/chosen": -1.02734375,
      "logits/rejected": -0.91015625,
      "logps/chosen": -0.64111328125,
      "logps/rejected": -1.677343726158142,
      "loss": 0.7552,
      "nll_loss": 0.701367199420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06411132961511612,
      "rewards/margins": 0.10366211086511612,
      "rewards/rejected": -0.1678466796875,
      "step": 7700
    },
    {
      "epoch": 0.561400953871919,
      "grad_norm": 3.4656677767253927,
      "learning_rate": 9.110931846854553e-07,
      "log_odds_chosen": 1.781347632408142,
      "log_odds_ratio": -0.3113037049770355,
      "logits/chosen": -1.034570336341858,
      "logits/rejected": -0.912109375,
      "logps/chosen": -0.6138671636581421,
      "logps/rejected": -1.8542969226837158,
      "loss": 0.7472,
      "nll_loss": 0.691699206829071,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06138915941119194,
      "rewards/margins": 0.12414245307445526,
      "rewards/rejected": -0.18549804389476776,
      "step": 7710
    },
    {
      "epoch": 0.5621291003749954,
      "grad_norm": 2.9797074643455503,
      "learning_rate": 9.105029072119708e-07,
      "log_odds_chosen": 1.60888671875,
      "log_odds_ratio": -0.358154296875,
      "logits/chosen": -0.980664074420929,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.644824206829071,
      "logps/rejected": -1.734960913658142,
      "loss": 0.7497,
      "nll_loss": 0.710742175579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06450195610523224,
      "rewards/margins": 0.10904540866613388,
      "rewards/rejected": -0.1734619140625,
      "step": 7720
    },
    {
      "epoch": 0.5628572468780718,
      "grad_norm": 2.723801173417352,
      "learning_rate": 9.099137755377291e-07,
      "log_odds_chosen": 1.71484375,
      "log_odds_ratio": -0.3432373106479645,
      "logits/chosen": -0.9732421636581421,
      "logits/rejected": -0.8705078363418579,
      "logps/chosen": -0.587890625,
      "logps/rejected": -1.751953125,
      "loss": 0.7771,
      "nll_loss": 0.740527331829071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05880127102136612,
      "rewards/margins": 0.116424560546875,
      "rewards/rejected": -0.17526856064796448,
      "step": 7730
    },
    {
      "epoch": 0.5635853933811483,
      "grad_norm": 2.539753729936125,
      "learning_rate": 9.093257859606311e-07,
      "log_odds_chosen": 1.720703125,
      "log_odds_ratio": -0.3291992247104645,
      "logits/chosen": -0.9716796875,
      "logits/rejected": -0.8763672113418579,
      "logps/chosen": -0.59765625,
      "logps/rejected": -1.7507812976837158,
      "loss": 0.7766,
      "nll_loss": 0.6919921636581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05975341796875,
      "rewards/margins": 0.11531982570886612,
      "rewards/rejected": -0.17502442002296448,
      "step": 7740
    },
    {
      "epoch": 0.5643135398842247,
      "grad_norm": 2.994330845395331,
      "learning_rate": 9.087389347953037e-07,
      "log_odds_chosen": 1.604248046875,
      "log_odds_ratio": -0.38715821504592896,
      "logits/chosen": -0.9853515625,
      "logits/rejected": -0.882031261920929,
      "logps/chosen": -0.647167980670929,
      "logps/rejected": -1.814062476158142,
      "loss": 0.7418,
      "nll_loss": 0.7411133050918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06472168117761612,
      "rewards/margins": 0.11683960258960724,
      "rewards/rejected": -0.18137207627296448,
      "step": 7750
    },
    {
      "epoch": 0.5650416863873011,
      "grad_norm": 3.3255595160046245,
      "learning_rate": 9.081532183729995e-07,
      "log_odds_chosen": 1.716796875,
      "log_odds_ratio": -0.38374024629592896,
      "logits/chosen": -0.9800781011581421,
      "logits/rejected": -0.90625,
      "logps/chosen": -0.6143554449081421,
      "logps/rejected": -1.8359375,
      "loss": 0.7342,
      "nll_loss": 0.662792980670929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06141357496380806,
      "rewards/margins": 0.12211914360523224,
      "rewards/rejected": -0.18339844048023224,
      "step": 7760
    },
    {
      "epoch": 0.5657698328903775,
      "grad_norm": 4.266926384363548,
      "learning_rate": 9.075686330415037e-07,
      "log_odds_chosen": 1.676367163658142,
      "log_odds_ratio": -0.3365478515625,
      "logits/chosen": -1.0070312023162842,
      "logits/rejected": -0.8568359613418579,
      "logps/chosen": -0.6397460699081421,
      "logps/rejected": -1.802734375,
      "loss": 0.7446,
      "nll_loss": 0.6512695550918579,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06400146335363388,
      "rewards/margins": 0.11652831733226776,
      "rewards/rejected": -0.1802978515625,
      "step": 7770
    },
    {
      "epoch": 0.5664979793934539,
      "grad_norm": 2.521866158471409,
      "learning_rate": 9.069851751650364e-07,
      "log_odds_chosen": 1.7119140625,
      "log_odds_ratio": -0.3543457090854645,
      "logits/chosen": -1.014257788658142,
      "logits/rejected": -0.8871093988418579,
      "logps/chosen": -0.6299804449081421,
      "logps/rejected": -1.802148461341858,
      "loss": 0.7336,
      "nll_loss": 0.6747070550918579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06302490085363388,
      "rewards/margins": 0.11733398586511612,
      "rewards/rejected": -0.18024902045726776,
      "step": 7780
    },
    {
      "epoch": 0.5672261258965304,
      "grad_norm": 2.8934724701376435,
      "learning_rate": 9.064028411241582e-07,
      "log_odds_chosen": 1.3977539539337158,
      "log_odds_ratio": -0.44145506620407104,
      "logits/chosen": -1.0041015148162842,
      "logits/rejected": -0.8812500238418579,
      "logps/chosen": -0.66015625,
      "logps/rejected": -1.6580078601837158,
      "loss": 0.7526,
      "nll_loss": 0.7466796636581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06602783501148224,
      "rewards/margins": 0.09980468451976776,
      "rewards/rejected": -0.1658935546875,
      "step": 7790
    },
    {
      "epoch": 0.5679542723996068,
      "grad_norm": 4.547374749002742,
      "learning_rate": 9.058216273156764e-07,
      "log_odds_chosen": 1.6826171875,
      "log_odds_ratio": -0.36865234375,
      "logits/chosen": -0.990234375,
      "logits/rejected": -0.843945324420929,
      "logps/chosen": -0.6502929925918579,
      "logps/rejected": -1.844140648841858,
      "loss": 0.7565,
      "nll_loss": 0.727734386920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06494140625,
      "rewards/margins": 0.11948242038488388,
      "rewards/rejected": -0.1845703125,
      "step": 7800
    },
    {
      "epoch": 0.5686824189026832,
      "grad_norm": 3.0619608925597506,
      "learning_rate": 9.052415301525511e-07,
      "log_odds_chosen": 1.908789038658142,
      "log_odds_ratio": -0.3045410215854645,
      "logits/chosen": -0.996874988079071,
      "logits/rejected": -0.873242199420929,
      "logps/chosen": -0.6220703125,
      "logps/rejected": -1.955468773841858,
      "loss": 0.7275,
      "nll_loss": 0.685351550579071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06219482421875,
      "rewards/margins": 0.13323974609375,
      "rewards/rejected": -0.19545897841453552,
      "step": 7810
    },
    {
      "epoch": 0.5694105654057596,
      "grad_norm": 3.1941187945537286,
      "learning_rate": 9.046625460638012e-07,
      "log_odds_chosen": 1.731835961341858,
      "log_odds_ratio": -0.3487304747104645,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.8802734613418579,
      "logps/chosen": -0.6421874761581421,
      "logps/rejected": -1.8380858898162842,
      "loss": 0.7301,
      "nll_loss": 0.728515625,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06423339992761612,
      "rewards/margins": 0.11958618462085724,
      "rewards/rejected": -0.18366698920726776,
      "step": 7820
    },
    {
      "epoch": 0.570138711908836,
      "grad_norm": 2.851710609830766,
      "learning_rate": 9.040846714944138e-07,
      "log_odds_chosen": 1.5380859375,
      "log_odds_ratio": -0.4166503846645355,
      "logits/chosen": -0.9857422113418579,
      "logits/rejected": -0.9019531011581421,
      "logps/chosen": -0.6556640863418579,
      "logps/rejected": -1.7355468273162842,
      "loss": 0.7567,
      "nll_loss": 0.7796875238418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06557617336511612,
      "rewards/margins": 0.10799255222082138,
      "rewards/rejected": -0.17351074516773224,
      "step": 7830
    },
    {
      "epoch": 0.5708668584119124,
      "grad_norm": 3.0843962103156413,
      "learning_rate": 9.035079029052513e-07,
      "log_odds_chosen": 1.931054711341858,
      "log_odds_ratio": -0.302734375,
      "logits/chosen": -1.016992211341858,
      "logits/rejected": -0.8783203363418579,
      "logps/chosen": -0.6377929449081421,
      "logps/rejected": -2.004687547683716,
      "loss": 0.7565,
      "nll_loss": 0.7310546636581421,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06369628757238388,
      "rewards/margins": 0.13673095405101776,
      "rewards/rejected": -0.20039062201976776,
      "step": 7840
    },
    {
      "epoch": 0.5715950049149889,
      "grad_norm": 3.3046770669931136,
      "learning_rate": 9.029322367729605e-07,
      "log_odds_chosen": 1.6710937023162842,
      "log_odds_ratio": -0.34526365995407104,
      "logits/chosen": -0.996289074420929,
      "logits/rejected": -0.8740234375,
      "logps/chosen": -0.582324206829071,
      "logps/rejected": -1.736328125,
      "loss": 0.723,
      "nll_loss": 0.6421874761581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.05826415866613388,
      "rewards/margins": 0.11538086086511612,
      "rewards/rejected": -0.173583984375,
      "step": 7850
    },
    {
      "epoch": 0.5723231514180653,
      "grad_norm": 2.4361921115850738,
      "learning_rate": 9.02357669589883e-07,
      "log_odds_chosen": 1.516503930091858,
      "log_odds_ratio": -0.38227540254592896,
      "logits/chosen": -1.015039086341858,
      "logits/rejected": -0.901562511920929,
      "logps/chosen": -0.606152355670929,
      "logps/rejected": -1.68359375,
      "loss": 0.7627,
      "nll_loss": 0.698535144329071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06063232570886612,
      "rewards/margins": 0.1077880859375,
      "rewards/rejected": -0.16835936903953552,
      "step": 7860
    },
    {
      "epoch": 0.5730512979211417,
      "grad_norm": 2.3762785457127142,
      "learning_rate": 9.017841978639643e-07,
      "log_odds_chosen": 1.5766112804412842,
      "log_odds_ratio": -0.3990722596645355,
      "logits/chosen": -0.9976562261581421,
      "logits/rejected": -0.9087890386581421,
      "logps/chosen": -0.63720703125,
      "logps/rejected": -1.7458984851837158,
      "loss": 0.7572,
      "nll_loss": 0.7162109613418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06376953423023224,
      "rewards/margins": 0.11075134575366974,
      "rewards/rejected": -0.17451171576976776,
      "step": 7870
    },
    {
      "epoch": 0.5737794444242181,
      "grad_norm": 3.3101451058829108,
      "learning_rate": 9.012118181186658e-07,
      "log_odds_chosen": 1.623046875,
      "log_odds_ratio": -0.33642578125,
      "logits/chosen": -1.0031249523162842,
      "logits/rejected": -0.916015625,
      "logps/chosen": -0.6392577886581421,
      "logps/rejected": -1.744531273841858,
      "loss": 0.7569,
      "nll_loss": 0.683886706829071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06395263969898224,
      "rewards/margins": 0.11058349907398224,
      "rewards/rejected": -0.17451171576976776,
      "step": 7880
    },
    {
      "epoch": 0.5745075909272945,
      "grad_norm": 2.727561156730717,
      "learning_rate": 9.00640526892875e-07,
      "log_odds_chosen": 1.4894530773162842,
      "log_odds_ratio": -0.40483397245407104,
      "logits/chosen": -0.9892578125,
      "logits/rejected": -0.8902343511581421,
      "logps/chosen": -0.6136718988418579,
      "logps/rejected": -1.596093773841858,
      "loss": 0.7489,
      "nll_loss": 0.7523437738418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06131591647863388,
      "rewards/margins": 0.09833984076976776,
      "rewards/rejected": -0.15966796875,
      "step": 7890
    },
    {
      "epoch": 0.575235737430371,
      "grad_norm": 2.744820927134792,
      "learning_rate": 9.000703207408191e-07,
      "log_odds_chosen": 1.301367163658142,
      "log_odds_ratio": -0.412841796875,
      "logits/chosen": -0.997851550579071,
      "logits/rejected": -0.909960925579071,
      "logps/chosen": -0.6826171875,
      "logps/rejected": -1.560937523841858,
      "loss": 0.7481,
      "nll_loss": 0.6986328363418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06826172024011612,
      "rewards/margins": 0.08783569186925888,
      "rewards/rejected": -0.15620116889476776,
      "step": 7900
    },
    {
      "epoch": 0.5759638839334474,
      "grad_norm": 3.593700908577961,
      "learning_rate": 8.995011962319761e-07,
      "log_odds_chosen": 1.641845703125,
      "log_odds_ratio": -0.3814941346645355,
      "logits/chosen": -0.946484386920929,
      "logits/rejected": -0.880078136920929,
      "logps/chosen": -0.658203125,
      "logps/rejected": -1.7937500476837158,
      "loss": 0.7711,
      "nll_loss": 0.681835949420929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06582031399011612,
      "rewards/margins": 0.11363372951745987,
      "rewards/rejected": -0.17937012016773224,
      "step": 7910
    },
    {
      "epoch": 0.5766920304365238,
      "grad_norm": 3.3411652114306927,
      "learning_rate": 8.989331499509894e-07,
      "log_odds_chosen": 1.707421898841858,
      "log_odds_ratio": -0.3526855409145355,
      "logits/chosen": -0.953906238079071,
      "logits/rejected": -0.8521484136581421,
      "logps/chosen": -0.6341797113418579,
      "logps/rejected": -1.7908203601837158,
      "loss": 0.7305,
      "nll_loss": 0.73681640625,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06340332329273224,
      "rewards/margins": 0.11555175483226776,
      "rewards/rejected": -0.17897948622703552,
      "step": 7920
    },
    {
      "epoch": 0.5774201769396002,
      "grad_norm": 3.042577724970246,
      "learning_rate": 8.983661784975812e-07,
      "log_odds_chosen": 1.9187500476837158,
      "log_odds_ratio": -0.3406982421875,
      "logits/chosen": -1.046484351158142,
      "logits/rejected": -0.9166015386581421,
      "logps/chosen": -0.615527331829071,
      "logps/rejected": -1.9949219226837158,
      "loss": 0.7307,
      "nll_loss": 0.702832043170929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06147461012005806,
      "rewards/margins": 0.13809815049171448,
      "rewards/rejected": -0.19956055283546448,
      "step": 7930
    },
    {
      "epoch": 0.5781483234426766,
      "grad_norm": 3.0327579045694266,
      "learning_rate": 8.97800278486467e-07,
      "log_odds_chosen": 1.464257836341858,
      "log_odds_ratio": -0.41059571504592896,
      "logits/chosen": -0.994140625,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -0.68359375,
      "logps/rejected": -1.7000000476837158,
      "loss": 0.7609,
      "nll_loss": 0.743945300579071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06833495944738388,
      "rewards/margins": 0.10173340141773224,
      "rewards/rejected": -0.169921875,
      "step": 7940
    },
    {
      "epoch": 0.578876469945753,
      "grad_norm": 2.4736047364722538,
      "learning_rate": 8.972354465472708e-07,
      "log_odds_chosen": 1.6513671875,
      "log_odds_ratio": -0.35026854276657104,
      "logits/chosen": -0.9388672113418579,
      "logits/rejected": -0.854296863079071,
      "logps/chosen": -0.56689453125,
      "logps/rejected": -1.6611328125,
      "loss": 0.7297,
      "nll_loss": 0.6727539300918579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05667724460363388,
      "rewards/margins": 0.10955810546875,
      "rewards/rejected": -0.16635742783546448,
      "step": 7950
    },
    {
      "epoch": 0.5796046164488295,
      "grad_norm": 2.763201925397316,
      "learning_rate": 8.966716793244405e-07,
      "log_odds_chosen": 1.599218726158142,
      "log_odds_ratio": -0.37255859375,
      "logits/chosen": -1.002539038658142,
      "logits/rejected": -0.867382824420929,
      "logps/chosen": -0.639453113079071,
      "logps/rejected": -1.783203125,
      "loss": 0.7283,
      "nll_loss": 0.7162109613418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06397704780101776,
      "rewards/margins": 0.11431884765625,
      "rewards/rejected": -0.17832031846046448,
      "step": 7960
    },
    {
      "epoch": 0.580332762951906,
      "grad_norm": 2.5731300162913415,
      "learning_rate": 8.96108973477165e-07,
      "log_odds_chosen": 1.678369164466858,
      "log_odds_ratio": -0.37116700410842896,
      "logits/chosen": -0.9683593511581421,
      "logits/rejected": -0.859179675579071,
      "logps/chosen": -0.653613269329071,
      "logps/rejected": -1.848046898841858,
      "loss": 0.7402,
      "nll_loss": 0.730175793170929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06533203274011612,
      "rewards/margins": 0.11951293796300888,
      "rewards/rejected": -0.18513183295726776,
      "step": 7970
    },
    {
      "epoch": 0.5810609094549823,
      "grad_norm": 2.5082168558305904,
      "learning_rate": 8.955473256792899e-07,
      "log_odds_chosen": 1.686132788658142,
      "log_odds_ratio": -0.3375244140625,
      "logits/chosen": -0.96484375,
      "logits/rejected": -0.8277343511581421,
      "logps/chosen": -0.5804687738418579,
      "logps/rejected": -1.714453101158142,
      "loss": 0.7163,
      "nll_loss": 0.68896484375,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05800781399011612,
      "rewards/margins": 0.11337890475988388,
      "rewards/rejected": -0.17143554985523224,
      "step": 7980
    },
    {
      "epoch": 0.5817890559580587,
      "grad_norm": 3.1463933549183767,
      "learning_rate": 8.949867326192358e-07,
      "log_odds_chosen": 1.487695336341858,
      "log_odds_ratio": -0.3834472596645355,
      "logits/chosen": -1.0281250476837158,
      "logits/rejected": -0.890429675579071,
      "logps/chosen": -0.621777355670929,
      "logps/rejected": -1.6476562023162842,
      "loss": 0.7542,
      "nll_loss": 0.701953113079071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06218261644244194,
      "rewards/margins": 0.10255126655101776,
      "rewards/rejected": -0.164794921875,
      "step": 7990
    },
    {
      "epoch": 0.5825172024611351,
      "grad_norm": 2.7644194242982043,
      "learning_rate": 8.944271909999158e-07,
      "log_odds_chosen": 1.543554663658142,
      "log_odds_ratio": -0.4083007872104645,
      "logits/chosen": -0.944140613079071,
      "logits/rejected": -0.871289074420929,
      "logps/chosen": -0.683886706829071,
      "logps/rejected": -1.806249976158142,
      "loss": 0.7507,
      "nll_loss": 0.783398449420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06844482570886612,
      "rewards/margins": 0.11209716647863388,
      "rewards/rejected": -0.18049316108226776,
      "step": 8000
    },
    {
      "epoch": 0.5832453489642117,
      "grad_norm": 2.7345942637479435,
      "learning_rate": 8.938686975386545e-07,
      "log_odds_chosen": 1.627539038658142,
      "log_odds_ratio": -0.362548828125,
      "logits/chosen": -0.952343761920929,
      "logits/rejected": -0.8525390625,
      "logps/chosen": -0.683398425579071,
      "logps/rejected": -1.867578148841858,
      "loss": 0.7541,
      "nll_loss": 0.761523425579071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06833495944738388,
      "rewards/margins": 0.11845703423023224,
      "rewards/rejected": -0.18696288764476776,
      "step": 8010
    },
    {
      "epoch": 0.583973495467288,
      "grad_norm": 3.745222662537538,
      "learning_rate": 8.933112489671067e-07,
      "log_odds_chosen": 1.8073241710662842,
      "log_odds_ratio": -0.3519287109375,
      "logits/chosen": -0.9439452886581421,
      "logits/rejected": -0.875781238079071,
      "logps/chosen": -0.623046875,
      "logps/rejected": -1.862695336341858,
      "loss": 0.7567,
      "nll_loss": 0.665722668170929,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06229247897863388,
      "rewards/margins": 0.124053955078125,
      "rewards/rejected": -0.1864013671875,
      "step": 8020
    },
    {
      "epoch": 0.5847016419703644,
      "grad_norm": 3.1985910955441135,
      "learning_rate": 8.927548420311771e-07,
      "log_odds_chosen": 1.7392578125,
      "log_odds_ratio": -0.31640625,
      "logits/chosen": -1.0509765148162842,
      "logits/rejected": -0.9146484136581421,
      "logps/chosen": -0.5732421875,
      "logps/rejected": -1.7421875,
      "loss": 0.7317,
      "nll_loss": 0.646191418170929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05732421949505806,
      "rewards/margins": 0.11679687350988388,
      "rewards/rejected": -0.17429199814796448,
      "step": 8030
    },
    {
      "epoch": 0.5854297884734408,
      "grad_norm": 2.599881212535056,
      "learning_rate": 8.921994734909409e-07,
      "log_odds_chosen": 1.6453125476837158,
      "log_odds_ratio": -0.37763673067092896,
      "logits/chosen": -0.998828113079071,
      "logits/rejected": -0.8882812261581421,
      "logps/chosen": -0.6625000238418579,
      "logps/rejected": -1.8552734851837158,
      "loss": 0.7432,
      "nll_loss": 0.6318359375,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06633301079273224,
      "rewards/margins": 0.11915282905101776,
      "rewards/rejected": -0.18557128310203552,
      "step": 8040
    },
    {
      "epoch": 0.5861579349765172,
      "grad_norm": 2.456607835446078,
      "learning_rate": 8.916451401205645e-07,
      "log_odds_chosen": 1.757421851158142,
      "log_odds_ratio": -0.38359373807907104,
      "logits/chosen": -1.052343726158142,
      "logits/rejected": -0.906445324420929,
      "logps/chosen": -0.647753894329071,
      "logps/rejected": -1.8982422351837158,
      "loss": 0.7431,
      "nll_loss": 0.7181640863418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06477050483226776,
      "rewards/margins": 0.12489013373851776,
      "rewards/rejected": -0.1898193359375,
      "step": 8050
    },
    {
      "epoch": 0.5868860814795936,
      "grad_norm": 2.840688891917763,
      "learning_rate": 8.91091838708226e-07,
      "log_odds_chosen": 1.659765601158142,
      "log_odds_ratio": -0.3893066346645355,
      "logits/chosen": -0.96875,
      "logits/rejected": -0.8740234375,
      "logps/chosen": -0.6787109375,
      "logps/rejected": -1.837890625,
      "loss": 0.7569,
      "nll_loss": 0.7748047113418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06785888969898224,
      "rewards/margins": 0.11595459282398224,
      "rewards/rejected": -0.18386229872703552,
      "step": 8060
    },
    {
      "epoch": 0.5876142279826702,
      "grad_norm": 3.436415764679697,
      "learning_rate": 8.905395660560378e-07,
      "log_odds_chosen": 1.632421851158142,
      "log_odds_ratio": -0.3890624940395355,
      "logits/chosen": -1.0222656726837158,
      "logits/rejected": -0.896679699420929,
      "logps/chosen": -0.623046875,
      "logps/rejected": -1.7628905773162842,
      "loss": 0.7557,
      "nll_loss": 0.69970703125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06231689453125,
      "rewards/margins": 0.11384277045726776,
      "rewards/rejected": -0.1761474609375,
      "step": 8070
    },
    {
      "epoch": 0.5883423744857466,
      "grad_norm": 2.5229661779371075,
      "learning_rate": 8.899883189799695e-07,
      "log_odds_chosen": 1.59326171875,
      "log_odds_ratio": -0.3965820372104645,
      "logits/chosen": -0.9986327886581421,
      "logits/rejected": -0.8759765625,
      "logps/chosen": -0.655957043170929,
      "logps/rejected": -1.824609398841858,
      "loss": 0.7724,
      "nll_loss": 0.7422851324081421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06561279296875,
      "rewards/margins": 0.11694946140050888,
      "rewards/rejected": -0.18251952528953552,
      "step": 8080
    },
    {
      "epoch": 0.589070520988823,
      "grad_norm": 3.515440527710206,
      "learning_rate": 8.894380943097694e-07,
      "log_odds_chosen": 1.8439452648162842,
      "log_odds_ratio": -0.3108886778354645,
      "logits/chosen": -1.0324218273162842,
      "logits/rejected": -0.8955078125,
      "logps/chosen": -0.690136730670929,
      "logps/rejected": -2.002734422683716,
      "loss": 0.7375,
      "nll_loss": 0.76171875,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06901855766773224,
      "rewards/margins": 0.1312255859375,
      "rewards/rejected": -0.20029297471046448,
      "step": 8090
    },
    {
      "epoch": 0.5897986674918994,
      "grad_norm": 3.20937932495858,
      "learning_rate": 8.888888888888888e-07,
      "log_odds_chosen": 1.6279785633087158,
      "log_odds_ratio": -0.39750975370407104,
      "logits/chosen": -1.033203125,
      "logits/rejected": -0.891406238079071,
      "logps/chosen": -0.6546875238418579,
      "logps/rejected": -1.784570336341858,
      "loss": 0.7598,
      "nll_loss": 0.7496093511581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06547851860523224,
      "rewards/margins": 0.113128662109375,
      "rewards/rejected": -0.178466796875,
      "step": 8100
    },
    {
      "epoch": 0.5905268139949758,
      "grad_norm": 3.2885426514977176,
      "learning_rate": 8.883406995744061e-07,
      "log_odds_chosen": 1.65234375,
      "log_odds_ratio": -0.39775389432907104,
      "logits/chosen": -0.976757824420929,
      "logits/rejected": -0.864062488079071,
      "logps/chosen": -0.644335925579071,
      "logps/rejected": -1.7800781726837158,
      "loss": 0.7384,
      "nll_loss": 0.76025390625,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06440429389476776,
      "rewards/margins": 0.11375732719898224,
      "rewards/rejected": -0.17812499403953552,
      "step": 8110
    },
    {
      "epoch": 0.5912549604980523,
      "grad_norm": 2.3005982611392284,
      "learning_rate": 8.877935232369506e-07,
      "log_odds_chosen": 1.732031226158142,
      "log_odds_ratio": -0.3402343690395355,
      "logits/chosen": -1.03515625,
      "logits/rejected": -0.9212890863418579,
      "logps/chosen": -0.6322265863418579,
      "logps/rejected": -1.8322265148162842,
      "loss": 0.7386,
      "nll_loss": 0.67431640625,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06320800632238388,
      "rewards/margins": 0.12028808891773224,
      "rewards/rejected": -0.18330077826976776,
      "step": 8120
    },
    {
      "epoch": 0.5919831070011287,
      "grad_norm": 2.7099101008294326,
      "learning_rate": 8.872473567606276e-07,
      "log_odds_chosen": 1.6279296875,
      "log_odds_ratio": -0.3829589784145355,
      "logits/chosen": -1.0369141101837158,
      "logits/rejected": -0.9339843988418579,
      "logps/chosen": -0.6612304449081421,
      "logps/rejected": -1.845703125,
      "loss": 0.7539,
      "nll_loss": 0.745898425579071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06613769382238388,
      "rewards/margins": 0.11839599907398224,
      "rewards/rejected": -0.18452148139476776,
      "step": 8130
    },
    {
      "epoch": 0.5927112535042051,
      "grad_norm": 3.6296129321404997,
      "learning_rate": 8.867021970429453e-07,
      "log_odds_chosen": 1.6611328125,
      "log_odds_ratio": -0.35932618379592896,
      "logits/chosen": -1.0207030773162842,
      "logits/rejected": -0.8890625238418579,
      "logps/chosen": -0.6617187261581421,
      "logps/rejected": -1.826562523841858,
      "loss": 0.723,
      "nll_loss": 0.7040039300918579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06622314453125,
      "rewards/margins": 0.11635742336511612,
      "rewards/rejected": -0.1826171875,
      "step": 8140
    },
    {
      "epoch": 0.5934394000072815,
      "grad_norm": 2.6979576541710175,
      "learning_rate": 8.86158040994738e-07,
      "log_odds_chosen": 1.416894555091858,
      "log_odds_ratio": -0.42998045682907104,
      "logits/chosen": -0.9951171875,
      "logits/rejected": -0.882031261920929,
      "logps/chosen": -0.707324206829071,
      "logps/rejected": -1.7218749523162842,
      "loss": 0.7292,
      "nll_loss": 0.7269531488418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07073974609375,
      "rewards/margins": 0.10148926079273224,
      "rewards/rejected": -0.1722412109375,
      "step": 8150
    },
    {
      "epoch": 0.5941675465103579,
      "grad_norm": 3.754172415622287,
      "learning_rate": 8.856148855400954e-07,
      "log_odds_chosen": 1.6769530773162842,
      "log_odds_ratio": -0.3658691346645355,
      "logits/chosen": -0.9996093511581421,
      "logits/rejected": -0.890429675579071,
      "logps/chosen": -0.636914074420929,
      "logps/rejected": -1.812109351158142,
      "loss": 0.7315,
      "nll_loss": 0.73046875,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06364746391773224,
      "rewards/margins": 0.11765136569738388,
      "rewards/rejected": -0.18125000596046448,
      "step": 8160
    },
    {
      "epoch": 0.5948956930134343,
      "grad_norm": 2.838878170152878,
      "learning_rate": 8.850727276162873e-07,
      "log_odds_chosen": 1.460351586341858,
      "log_odds_ratio": -0.38310545682907104,
      "logits/chosen": -0.998242199420929,
      "logits/rejected": -0.8746093511581421,
      "logps/chosen": -0.6913086175918579,
      "logps/rejected": -1.703515648841858,
      "loss": 0.7465,
      "nll_loss": 0.7578125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06922607123851776,
      "rewards/margins": 0.10137939453125,
      "rewards/rejected": -0.17038574814796448,
      "step": 8170
    },
    {
      "epoch": 0.5956238395165108,
      "grad_norm": 2.8830576007290376,
      "learning_rate": 8.845315641736929e-07,
      "log_odds_chosen": 1.5007812976837158,
      "log_odds_ratio": -0.35181885957717896,
      "logits/chosen": -0.96484375,
      "logits/rejected": -0.8623046875,
      "logps/chosen": -0.663281261920929,
      "logps/rejected": -1.671484351158142,
      "loss": 0.742,
      "nll_loss": 0.7583984136581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06629638373851776,
      "rewards/margins": 0.10068359225988388,
      "rewards/rejected": -0.16691894829273224,
      "step": 8180
    },
    {
      "epoch": 0.5963519860195872,
      "grad_norm": 3.0712173168830468,
      "learning_rate": 8.839913921757278e-07,
      "log_odds_chosen": 1.5867187976837158,
      "log_odds_ratio": -0.4339355528354645,
      "logits/chosen": -0.9751952886581421,
      "logits/rejected": -0.884765625,
      "logps/chosen": -0.699999988079071,
      "logps/rejected": -1.8468749523162842,
      "loss": 0.7552,
      "nll_loss": 0.6953125,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06998290866613388,
      "rewards/margins": 0.11473999172449112,
      "rewards/rejected": -0.184814453125,
      "step": 8190
    },
    {
      "epoch": 0.5970801325226636,
      "grad_norm": 2.6812346128242175,
      "learning_rate": 8.834522085987722e-07,
      "log_odds_chosen": 1.5862548351287842,
      "log_odds_ratio": -0.3882812559604645,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.8472656011581421,
      "logps/chosen": -0.6456054449081421,
      "logps/rejected": -1.671484351158142,
      "loss": 0.7563,
      "nll_loss": 0.67138671875,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06450195610523224,
      "rewards/margins": 0.10265350341796875,
      "rewards/rejected": -0.16721192002296448,
      "step": 8200
    },
    {
      "epoch": 0.59780827902574,
      "grad_norm": 2.524221248540114,
      "learning_rate": 8.829140104321008e-07,
      "log_odds_chosen": 1.697265625,
      "log_odds_ratio": -0.346923828125,
      "logits/chosen": -1.0027344226837158,
      "logits/rejected": -0.9105468988418579,
      "logps/chosen": -0.627636730670929,
      "logps/rejected": -1.794531226158142,
      "loss": 0.7503,
      "nll_loss": 0.7396484613418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06270752102136612,
      "rewards/margins": 0.11690673977136612,
      "rewards/rejected": -0.17949219048023224,
      "step": 8210
    },
    {
      "epoch": 0.5985364255288164,
      "grad_norm": 2.7650551312288947,
      "learning_rate": 8.82376794677811e-07,
      "log_odds_chosen": 1.488671898841858,
      "log_odds_ratio": -0.380615234375,
      "logits/chosen": -1.00390625,
      "logits/rejected": -0.919140636920929,
      "logps/chosen": -0.6092773675918579,
      "logps/rejected": -1.6453125476837158,
      "loss": 0.7466,
      "nll_loss": 0.5962890386581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06097412109375,
      "rewards/margins": 0.10357666015625,
      "rewards/rejected": -0.16464844346046448,
      "step": 8220
    },
    {
      "epoch": 0.5992645720318928,
      "grad_norm": 2.9542490854379704,
      "learning_rate": 8.818405583507537e-07,
      "log_odds_chosen": 1.4060547351837158,
      "log_odds_ratio": -0.4031738340854645,
      "logits/chosen": -1.0574219226837158,
      "logits/rejected": -0.947070300579071,
      "logps/chosen": -0.642382800579071,
      "logps/rejected": -1.5857422351837158,
      "loss": 0.7258,
      "nll_loss": 0.6341797113418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06424560397863388,
      "rewards/margins": 0.0943603515625,
      "rewards/rejected": -0.15866699814796448,
      "step": 8230
    },
    {
      "epoch": 0.5999927185349693,
      "grad_norm": 3.0083388513165614,
      "learning_rate": 8.813052984784634e-07,
      "log_odds_chosen": 1.5631835460662842,
      "log_odds_ratio": -0.39042967557907104,
      "logits/chosen": -0.9697265625,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.637402355670929,
      "logps/rejected": -1.7390625476837158,
      "loss": 0.7453,
      "nll_loss": 0.7420898675918579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06376953423023224,
      "rewards/margins": 0.11024780571460724,
      "rewards/rejected": -0.1739501953125,
      "step": 8240
    },
    {
      "epoch": 0.6007208650380457,
      "grad_norm": 2.6965709037026695,
      "learning_rate": 8.807710121010885e-07,
      "log_odds_chosen": 1.780859351158142,
      "log_odds_ratio": -0.32939451932907104,
      "logits/chosen": -1.0261719226837158,
      "logits/rejected": -0.9375,
      "logps/chosen": -0.6504882574081421,
      "logps/rejected": -1.890625,
      "loss": 0.7318,
      "nll_loss": 0.7037109136581421,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.06507568061351776,
      "rewards/margins": 0.12395019829273224,
      "rewards/rejected": -0.18911132216453552,
      "step": 8250
    },
    {
      "epoch": 0.6014490115411221,
      "grad_norm": 2.5588193032281827,
      "learning_rate": 8.802376962713231e-07,
      "log_odds_chosen": 1.895117163658142,
      "log_odds_ratio": -0.34248048067092896,
      "logits/chosen": -1.033789038658142,
      "logits/rejected": -0.9273437261581421,
      "logps/chosen": -0.6444336175918579,
      "logps/rejected": -1.962890625,
      "loss": 0.7148,
      "nll_loss": 0.66650390625,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06439208984375,
      "rewards/margins": 0.13181152939796448,
      "rewards/rejected": -0.19631347060203552,
      "step": 8260
    },
    {
      "epoch": 0.6021771580441985,
      "grad_norm": 2.701725607640747,
      "learning_rate": 8.797053480543386e-07,
      "log_odds_chosen": 1.446386694908142,
      "log_odds_ratio": -0.39277344942092896,
      "logits/chosen": -0.983203113079071,
      "logits/rejected": -0.8980468511581421,
      "logps/chosen": -0.645312488079071,
      "logps/rejected": -1.616601586341858,
      "loss": 0.7314,
      "nll_loss": 0.6546875238418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06447754055261612,
      "rewards/margins": 0.0971527099609375,
      "rewards/rejected": -0.16169433295726776,
      "step": 8270
    },
    {
      "epoch": 0.6029053045472749,
      "grad_norm": 2.9035082219959962,
      "learning_rate": 8.79173964527716e-07,
      "log_odds_chosen": 1.703027367591858,
      "log_odds_ratio": -0.35493165254592896,
      "logits/chosen": -1.025976538658142,
      "logits/rejected": -0.9371093511581421,
      "logps/chosen": -0.6318359375,
      "logps/rejected": -1.818359375,
      "loss": 0.7482,
      "nll_loss": 0.6976562738418579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06317138671875,
      "rewards/margins": 0.11872558295726776,
      "rewards/rejected": -0.18195800483226776,
      "step": 8280
    },
    {
      "epoch": 0.6036334510503514,
      "grad_norm": 2.8145134277773667,
      "learning_rate": 8.78643542781378e-07,
      "log_odds_chosen": 1.620507836341858,
      "log_odds_ratio": -0.41191405057907104,
      "logits/chosen": -1.0255858898162842,
      "logits/rejected": -0.9283202886581421,
      "logps/chosen": -0.695117175579071,
      "logps/rejected": -1.874609351158142,
      "loss": 0.7406,
      "nll_loss": 0.74072265625,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06953124701976776,
      "rewards/margins": 0.11796875298023224,
      "rewards/rejected": -0.18764647841453552,
      "step": 8290
    },
    {
      "epoch": 0.6043615975534278,
      "grad_norm": 2.7199145777681446,
      "learning_rate": 8.781140799175228e-07,
      "log_odds_chosen": 1.583984375,
      "log_odds_ratio": -0.4493164122104645,
      "logits/chosen": -1.086523413658142,
      "logits/rejected": -0.9380859136581421,
      "logps/chosen": -0.6626952886581421,
      "logps/rejected": -1.8005859851837158,
      "loss": 0.7359,
      "nll_loss": 0.6050781011581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0662841796875,
      "rewards/margins": 0.11369629204273224,
      "rewards/rejected": -0.18000487983226776,
      "step": 8300
    },
    {
      "epoch": 0.6050897440565042,
      "grad_norm": 2.8070093814780503,
      "learning_rate": 8.775855730505568e-07,
      "log_odds_chosen": 1.457421898841858,
      "log_odds_ratio": -0.394287109375,
      "logits/chosen": -1.0246093273162842,
      "logits/rejected": -0.913281261920929,
      "logps/chosen": -0.6841796636581421,
      "logps/rejected": -1.6974608898162842,
      "loss": 0.7051,
      "nll_loss": 0.654101550579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06840820610523224,
      "rewards/margins": 0.10146789252758026,
      "rewards/rejected": -0.16994628310203552,
      "step": 8310
    },
    {
      "epoch": 0.6058178905595806,
      "grad_norm": 2.8504827543145868,
      "learning_rate": 8.770580193070291e-07,
      "log_odds_chosen": 1.736718773841858,
      "log_odds_ratio": -0.3272949159145355,
      "logits/chosen": -1.0498046875,
      "logits/rejected": -0.886914074420929,
      "logps/chosen": -0.609375,
      "logps/rejected": -1.795507788658142,
      "loss": 0.7309,
      "nll_loss": 0.671875,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06096191331744194,
      "rewards/margins": 0.11849365383386612,
      "rewards/rejected": -0.17941895127296448,
      "step": 8320
    },
    {
      "epoch": 0.606546037062657,
      "grad_norm": 2.5713483605910104,
      "learning_rate": 8.765314158255661e-07,
      "log_odds_chosen": 1.5714843273162842,
      "log_odds_ratio": -0.3888183534145355,
      "logits/chosen": -1.0710937976837158,
      "logits/rejected": -0.9683593511581421,
      "logps/chosen": -0.664257824420929,
      "logps/rejected": -1.771875023841858,
      "loss": 0.7207,
      "nll_loss": 0.667285144329071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06640625,
      "rewards/margins": 0.11072997748851776,
      "rewards/rejected": -0.17714843153953552,
      "step": 8330
    },
    {
      "epoch": 0.6072741835657334,
      "grad_norm": 3.1073046634375965,
      "learning_rate": 8.760057597568057e-07,
      "log_odds_chosen": 1.6642577648162842,
      "log_odds_ratio": -0.36994630098342896,
      "logits/chosen": -1.052343726158142,
      "logits/rejected": -0.9408203363418579,
      "logps/chosen": -0.6375976800918579,
      "logps/rejected": -1.8019530773162842,
      "loss": 0.7228,
      "nll_loss": 0.7040039300918579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06372680515050888,
      "rewards/margins": 0.116455078125,
      "rewards/rejected": -0.18012695014476776,
      "step": 8340
    },
    {
      "epoch": 0.6080023300688099,
      "grad_norm": 2.6792793557084305,
      "learning_rate": 8.754810482633324e-07,
      "log_odds_chosen": 1.64453125,
      "log_odds_ratio": -0.35332030057907104,
      "logits/chosen": -1.009179711341858,
      "logits/rejected": -0.8876953125,
      "logps/chosen": -0.6246093511581421,
      "logps/rejected": -1.7316405773162842,
      "loss": 0.7385,
      "nll_loss": 0.705078125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06246338039636612,
      "rewards/margins": 0.11072997748851776,
      "rewards/rejected": -0.17316894233226776,
      "step": 8350
    },
    {
      "epoch": 0.6087304765718863,
      "grad_norm": 3.810742754838614,
      "learning_rate": 8.749572785196142e-07,
      "log_odds_chosen": 1.6642577648162842,
      "log_odds_ratio": -0.34187012910842896,
      "logits/chosen": -1.0505859851837158,
      "logits/rejected": -0.938281238079071,
      "logps/chosen": -0.62939453125,
      "logps/rejected": -1.7703125476837158,
      "loss": 0.7274,
      "nll_loss": 0.6451171636581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06293945014476776,
      "rewards/margins": 0.11420898139476776,
      "rewards/rejected": -0.17709961533546448,
      "step": 8360
    },
    {
      "epoch": 0.6094586230749627,
      "grad_norm": 2.562781209787786,
      "learning_rate": 8.744344477119373e-07,
      "log_odds_chosen": 1.460546851158142,
      "log_odds_ratio": -0.39189451932907104,
      "logits/chosen": -1.016210913658142,
      "logits/rejected": -0.920117199420929,
      "logps/chosen": -0.5980468988418579,
      "logps/rejected": -1.5929687023162842,
      "loss": 0.7242,
      "nll_loss": 0.6595703363418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.059814453125,
      "rewards/margins": 0.09953002631664276,
      "rewards/rejected": -0.15939942002296448,
      "step": 8370
    },
    {
      "epoch": 0.6101867695780391,
      "grad_norm": 2.962136143002416,
      "learning_rate": 8.739125530383433e-07,
      "log_odds_chosen": 1.69970703125,
      "log_odds_ratio": -0.3591552674770355,
      "logits/chosen": -0.9912109375,
      "logits/rejected": -0.913867175579071,
      "logps/chosen": -0.64892578125,
      "logps/rejected": -1.837890625,
      "loss": 0.7191,
      "nll_loss": 0.6986328363418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06485595554113388,
      "rewards/margins": 0.1190185546875,
      "rewards/rejected": -0.18388672173023224,
      "step": 8380
    },
    {
      "epoch": 0.6109149160811155,
      "grad_norm": 2.7206804811791074,
      "learning_rate": 8.733915917085661e-07,
      "log_odds_chosen": 1.749414086341858,
      "log_odds_ratio": -0.35883790254592896,
      "logits/chosen": -1.043359398841858,
      "logits/rejected": -0.9029296636581421,
      "logps/chosen": -0.7021484375,
      "logps/rejected": -1.9675781726837158,
      "loss": 0.7403,
      "nll_loss": 0.7579101324081421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0701904296875,
      "rewards/margins": 0.12640380859375,
      "rewards/rejected": -0.196533203125,
      "step": 8390
    },
    {
      "epoch": 0.611643062584192,
      "grad_norm": 3.0287936848156325,
      "learning_rate": 8.728715609439695e-07,
      "log_odds_chosen": 1.392968773841858,
      "log_odds_ratio": -0.4473876953125,
      "logits/chosen": -0.9898437261581421,
      "logits/rejected": -0.932421863079071,
      "logps/chosen": -0.69189453125,
      "logps/rejected": -1.6574218273162842,
      "loss": 0.7517,
      "nll_loss": 0.75146484375,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06915283203125,
      "rewards/margins": 0.09659729152917862,
      "rewards/rejected": -0.165771484375,
      "step": 8400
    },
    {
      "epoch": 0.6123712090872684,
      "grad_norm": 2.2707084404880145,
      "learning_rate": 8.72352457977484e-07,
      "log_odds_chosen": 1.600195288658142,
      "log_odds_ratio": -0.3811279237270355,
      "logits/chosen": -1.0126953125,
      "logits/rejected": -0.9156249761581421,
      "logps/chosen": -0.6102539300918579,
      "logps/rejected": -1.714257836341858,
      "loss": 0.7347,
      "nll_loss": 0.6856445074081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06108398362994194,
      "rewards/margins": 0.11036376655101776,
      "rewards/rejected": -0.17143554985523224,
      "step": 8410
    },
    {
      "epoch": 0.6130993555903448,
      "grad_norm": 2.873832513952325,
      "learning_rate": 8.718342800535456e-07,
      "log_odds_chosen": 1.485937476158142,
      "log_odds_ratio": -0.3880371153354645,
      "logits/chosen": -1.0341796875,
      "logits/rejected": -0.916796863079071,
      "logps/chosen": -0.646679699420929,
      "logps/rejected": -1.712890625,
      "loss": 0.7242,
      "nll_loss": 0.650585949420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06463623046875,
      "rewards/margins": 0.10659179836511612,
      "rewards/rejected": -0.17133788764476776,
      "step": 8420
    },
    {
      "epoch": 0.6138275020934212,
      "grad_norm": 3.4180799837627833,
      "learning_rate": 8.713170244280353e-07,
      "log_odds_chosen": 1.4998047351837158,
      "log_odds_ratio": -0.38720703125,
      "logits/chosen": -0.9859374761581421,
      "logits/rejected": -0.917187511920929,
      "logps/chosen": -0.6429687738418579,
      "logps/rejected": -1.654296875,
      "loss": 0.7134,
      "nll_loss": 0.722949206829071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0643310546875,
      "rewards/margins": 0.101165771484375,
      "rewards/rejected": -0.16538086533546448,
      "step": 8430
    },
    {
      "epoch": 0.6145556485964976,
      "grad_norm": 3.0639599219105,
      "learning_rate": 8.708006883682162e-07,
      "log_odds_chosen": 1.7654297351837158,
      "log_odds_ratio": -0.379638671875,
      "logits/chosen": -1.0734374523162842,
      "logits/rejected": -0.987109363079071,
      "logps/chosen": -0.661328136920929,
      "logps/rejected": -1.9226562976837158,
      "loss": 0.721,
      "nll_loss": 0.6703125238418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06610107421875,
      "rewards/margins": 0.12623901665210724,
      "rewards/rejected": -0.19237060844898224,
      "step": 8440
    },
    {
      "epoch": 0.615283795099574,
      "grad_norm": 3.3879505804326207,
      "learning_rate": 8.702852691526739e-07,
      "log_odds_chosen": 1.567773461341858,
      "log_odds_ratio": -0.36962890625,
      "logits/chosen": -1.0705077648162842,
      "logits/rejected": -0.914843738079071,
      "logps/chosen": -0.6485351324081421,
      "logps/rejected": -1.754296898841858,
      "loss": 0.717,
      "nll_loss": 0.69677734375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06490478664636612,
      "rewards/margins": 0.11057128757238388,
      "rewards/rejected": -0.17539063096046448,
      "step": 8450
    },
    {
      "epoch": 0.6160119416026505,
      "grad_norm": 3.3467635114050127,
      "learning_rate": 8.697707640712562e-07,
      "log_odds_chosen": 1.6330077648162842,
      "log_odds_ratio": -0.3785156309604645,
      "logits/chosen": -1.0275390148162842,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.596484363079071,
      "logps/rejected": -1.695898413658142,
      "loss": 0.7426,
      "nll_loss": 0.696582019329071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05970459058880806,
      "rewards/margins": 0.10993652045726776,
      "rewards/rejected": -0.16962890326976776,
      "step": 8460
    },
    {
      "epoch": 0.6167400881057269,
      "grad_norm": 2.6803776745211865,
      "learning_rate": 8.692571704250135e-07,
      "log_odds_chosen": 1.421289086341858,
      "log_odds_ratio": -0.4140625,
      "logits/chosen": -1.079492211341858,
      "logits/rejected": -0.9332031011581421,
      "logps/chosen": -0.623046875,
      "logps/rejected": -1.5939452648162842,
      "loss": 0.7133,
      "nll_loss": 0.6460937261581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06235351413488388,
      "rewards/margins": 0.09708251804113388,
      "rewards/rejected": -0.15932616591453552,
      "step": 8470
    },
    {
      "epoch": 0.6174682346088033,
      "grad_norm": 2.60658832310545,
      "learning_rate": 8.687444855261388e-07,
      "log_odds_chosen": 1.7666015625,
      "log_odds_ratio": -0.33720701932907104,
      "logits/chosen": -1.036523461341858,
      "logits/rejected": -0.919140636920929,
      "logps/chosen": -0.611523449420929,
      "logps/rejected": -1.830468773841858,
      "loss": 0.7242,
      "nll_loss": 0.6513671875,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06108398362994194,
      "rewards/margins": 0.12187500298023224,
      "rewards/rejected": -0.1829833984375,
      "step": 8480
    },
    {
      "epoch": 0.6181963811118797,
      "grad_norm": 2.945371742976902,
      "learning_rate": 8.682327066979084e-07,
      "log_odds_chosen": 1.815039038658142,
      "log_odds_ratio": -0.3404785096645355,
      "logits/chosen": -0.983203113079071,
      "logits/rejected": -0.889453113079071,
      "logps/chosen": -0.644726574420929,
      "logps/rejected": -1.904296875,
      "loss": 0.7124,
      "nll_loss": 0.718554675579071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.064453125,
      "rewards/margins": 0.12587890028953552,
      "rewards/rejected": -0.19028320908546448,
      "step": 8490
    },
    {
      "epoch": 0.6189245276149561,
      "grad_norm": 3.1594011785817866,
      "learning_rate": 8.677218312746247e-07,
      "log_odds_chosen": 1.888671875,
      "log_odds_ratio": -0.3163085877895355,
      "logits/chosen": -1.0447266101837158,
      "logits/rejected": -0.912304699420929,
      "logps/chosen": -0.652539074420929,
      "logps/rejected": -2.0093750953674316,
      "loss": 0.7218,
      "nll_loss": 0.71240234375,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06523437798023224,
      "rewards/margins": 0.13580322265625,
      "rewards/rejected": -0.20112304389476776,
      "step": 8500
    },
    {
      "epoch": 0.6196526741180326,
      "grad_norm": 2.567175524427138,
      "learning_rate": 8.672118566015558e-07,
      "log_odds_chosen": 1.5828125476837158,
      "log_odds_ratio": -0.38105469942092896,
      "logits/chosen": -1.009179711341858,
      "logits/rejected": -0.8662109375,
      "logps/chosen": -0.688671886920929,
      "logps/rejected": -1.821874976158142,
      "loss": 0.7387,
      "nll_loss": 0.716796875,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06884765625,
      "rewards/margins": 0.11341552436351776,
      "rewards/rejected": -0.18227538466453552,
      "step": 8510
    },
    {
      "epoch": 0.620380820621109,
      "grad_norm": 2.67467217671831,
      "learning_rate": 8.667027800348789e-07,
      "log_odds_chosen": 1.613671898841858,
      "log_odds_ratio": -0.366943359375,
      "logits/chosen": -1.099023461341858,
      "logits/rejected": -0.9496093988418579,
      "logps/chosen": -0.6419922113418579,
      "logps/rejected": -1.796484351158142,
      "loss": 0.7041,
      "nll_loss": 0.6688476800918579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06409911811351776,
      "rewards/margins": 0.11542968451976776,
      "rewards/rejected": -0.17983397841453552,
      "step": 8520
    },
    {
      "epoch": 0.6211089671241854,
      "grad_norm": 2.9445492121325567,
      "learning_rate": 8.661945989416229e-07,
      "log_odds_chosen": 1.6330077648162842,
      "log_odds_ratio": -0.36083984375,
      "logits/chosen": -0.974609375,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.598437488079071,
      "logps/rejected": -1.7033202648162842,
      "loss": 0.7301,
      "nll_loss": 0.691113293170929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05987548828125,
      "rewards/margins": 0.11052246391773224,
      "rewards/rejected": -0.17041015625,
      "step": 8530
    },
    {
      "epoch": 0.6218371136272618,
      "grad_norm": 2.8456097273485583,
      "learning_rate": 8.6568731069961e-07,
      "log_odds_chosen": 1.708593726158142,
      "log_odds_ratio": -0.37749022245407104,
      "logits/chosen": -1.008203148841858,
      "logits/rejected": -0.8880859613418579,
      "logps/chosen": -0.659960925579071,
      "logps/rejected": -1.8640625476837158,
      "loss": 0.7094,
      "nll_loss": 0.6200195550918579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06599120795726776,
      "rewards/margins": 0.12054443359375,
      "rewards/rejected": -0.18637695908546448,
      "step": 8540
    },
    {
      "epoch": 0.6225652601303382,
      "grad_norm": 2.4054695984861407,
      "learning_rate": 8.651809126974002e-07,
      "log_odds_chosen": 1.655859351158142,
      "log_odds_ratio": -0.3834472596645355,
      "logits/chosen": -0.983593761920929,
      "logits/rejected": -0.882031261920929,
      "logps/chosen": -0.6626952886581421,
      "logps/rejected": -1.8660156726837158,
      "loss": 0.74,
      "nll_loss": 0.710253894329071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06624756008386612,
      "rewards/margins": 0.12049560248851776,
      "rewards/rejected": -0.18671874701976776,
      "step": 8550
    },
    {
      "epoch": 0.6232934066334146,
      "grad_norm": 2.98894973239979,
      "learning_rate": 8.646754023342339e-07,
      "log_odds_chosen": 1.556249976158142,
      "log_odds_ratio": -0.3735107481479645,
      "logits/chosen": -0.9876953363418579,
      "logits/rejected": -0.8785156011581421,
      "logps/chosen": -0.644335925579071,
      "logps/rejected": -1.7326171398162842,
      "loss": 0.7281,
      "nll_loss": 0.7123047113418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06440429389476776,
      "rewards/margins": 0.10886230319738388,
      "rewards/rejected": -0.17348632216453552,
      "step": 8560
    },
    {
      "epoch": 0.6240215531364911,
      "grad_norm": 3.087953492334864,
      "learning_rate": 8.64170777019976e-07,
      "log_odds_chosen": 1.717382788658142,
      "log_odds_ratio": -0.38203126192092896,
      "logits/chosen": -1.0125000476837158,
      "logits/rejected": -0.87109375,
      "logps/chosen": -0.680859386920929,
      "logps/rejected": -1.9128906726837158,
      "loss": 0.7244,
      "nll_loss": 0.7001953125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06804199516773224,
      "rewards/margins": 0.1231689453125,
      "rewards/rejected": -0.191162109375,
      "step": 8570
    },
    {
      "epoch": 0.6247496996395675,
      "grad_norm": 4.232381686902273,
      "learning_rate": 8.636670341750609e-07,
      "log_odds_chosen": 1.6462891101837158,
      "log_odds_ratio": -0.35566407442092896,
      "logits/chosen": -0.9681640863418579,
      "logits/rejected": -0.850390613079071,
      "logps/chosen": -0.61669921875,
      "logps/rejected": -1.7863280773162842,
      "loss": 0.7114,
      "nll_loss": 0.6981445550918579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06168212741613388,
      "rewards/margins": 0.11704101413488388,
      "rewards/rejected": -0.17868652939796448,
      "step": 8580
    },
    {
      "epoch": 0.6254778461426439,
      "grad_norm": 3.0345972298952377,
      "learning_rate": 8.631641712304359e-07,
      "log_odds_chosen": 1.310937523841858,
      "log_odds_ratio": -0.4342285096645355,
      "logits/chosen": -0.9703124761581421,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.6166015863418579,
      "logps/rejected": -1.4865233898162842,
      "loss": 0.7229,
      "nll_loss": 0.7470703125,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06170654296875,
      "rewards/margins": 0.08698425441980362,
      "rewards/rejected": -0.14860840141773224,
      "step": 8590
    },
    {
      "epoch": 0.6262059926457203,
      "grad_norm": 2.6441794558090357,
      "learning_rate": 8.626621856275073e-07,
      "log_odds_chosen": 1.914453148841858,
      "log_odds_ratio": -0.3064941465854645,
      "logits/chosen": -0.966796875,
      "logits/rejected": -0.844921886920929,
      "logps/chosen": -0.627148449420929,
      "logps/rejected": -1.9500000476837158,
      "loss": 0.7332,
      "nll_loss": 0.74169921875,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.06264648586511612,
      "rewards/margins": 0.1324462890625,
      "rewards/rejected": -0.195068359375,
      "step": 8600
    },
    {
      "epoch": 0.6269341391487967,
      "grad_norm": 3.0006317482794262,
      "learning_rate": 8.621610748180847e-07,
      "log_odds_chosen": 1.95166015625,
      "log_odds_ratio": -0.33037108182907104,
      "logits/chosen": -1.018164038658142,
      "logits/rejected": -0.908984363079071,
      "logps/chosen": -0.6092773675918579,
      "logps/rejected": -1.990234375,
      "loss": 0.7249,
      "nll_loss": 0.7120116949081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06096191331744194,
      "rewards/margins": 0.13807372748851776,
      "rewards/rejected": -0.19899901747703552,
      "step": 8610
    },
    {
      "epoch": 0.6276622856518732,
      "grad_norm": 2.881516654324948,
      "learning_rate": 8.616608362643274e-07,
      "log_odds_chosen": 1.6759765148162842,
      "log_odds_ratio": -0.36079102754592896,
      "logits/chosen": -0.9814453125,
      "logits/rejected": -0.90234375,
      "logps/chosen": -0.6444336175918579,
      "logps/rejected": -1.806054711341858,
      "loss": 0.7214,
      "nll_loss": 0.6973632574081421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06442870944738388,
      "rewards/margins": 0.11605224758386612,
      "rewards/rejected": -0.18059082329273224,
      "step": 8620
    },
    {
      "epoch": 0.6283904321549496,
      "grad_norm": 2.4495788911993053,
      "learning_rate": 8.611614674386904e-07,
      "log_odds_chosen": 1.771875023841858,
      "log_odds_ratio": -0.32954102754592896,
      "logits/chosen": -0.980273425579071,
      "logits/rejected": -0.9007812738418579,
      "logps/chosen": -0.6119140386581421,
      "logps/rejected": -1.8097655773162842,
      "loss": 0.7243,
      "nll_loss": 0.710156261920929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06114501878619194,
      "rewards/margins": 0.119873046875,
      "rewards/rejected": -0.18098144233226776,
      "step": 8630
    },
    {
      "epoch": 0.629118578658026,
      "grad_norm": 2.3178016512933963,
      "learning_rate": 8.606629658238703e-07,
      "log_odds_chosen": 1.504980444908142,
      "log_odds_ratio": -0.4437499940395355,
      "logits/chosen": -1.011132836341858,
      "logits/rejected": -0.8998047113418579,
      "logps/chosen": -0.6719726324081421,
      "logps/rejected": -1.7472655773162842,
      "loss": 0.7096,
      "nll_loss": 0.6961914300918579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06727294623851776,
      "rewards/margins": 0.1074981689453125,
      "rewards/rejected": -0.17453613877296448,
      "step": 8640
    },
    {
      "epoch": 0.6298467251611024,
      "grad_norm": 2.4447121230183577,
      "learning_rate": 8.601653289127525e-07,
      "log_odds_chosen": 1.5476562976837158,
      "log_odds_ratio": -0.371826171875,
      "logits/chosen": -0.9990234375,
      "logits/rejected": -0.927734375,
      "logps/chosen": -0.665332019329071,
      "logps/rejected": -1.741796851158142,
      "loss": 0.7415,
      "nll_loss": 0.6693359613418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06650390475988388,
      "rewards/margins": 0.10771484673023224,
      "rewards/rejected": -0.17416992783546448,
      "step": 8650
    },
    {
      "epoch": 0.6305748716641788,
      "grad_norm": 4.197818676298968,
      "learning_rate": 8.596685542083577e-07,
      "log_odds_chosen": 1.584326148033142,
      "log_odds_ratio": -0.3987793028354645,
      "logits/chosen": -1.030859351158142,
      "logits/rejected": -0.943554699420929,
      "logps/chosen": -0.6484375,
      "logps/rejected": -1.703125,
      "loss": 0.731,
      "nll_loss": 0.6519531011581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06492920219898224,
      "rewards/margins": 0.10557098686695099,
      "rewards/rejected": -0.1702880859375,
      "step": 8660
    },
    {
      "epoch": 0.6313030181672552,
      "grad_norm": 2.8608605498751136,
      "learning_rate": 8.591726392237899e-07,
      "log_odds_chosen": 1.487890601158142,
      "log_odds_ratio": -0.393798828125,
      "logits/chosen": -0.996874988079071,
      "logits/rejected": -0.8974609375,
      "logps/chosen": -0.64501953125,
      "logps/rejected": -1.6648437976837158,
      "loss": 0.7379,
      "nll_loss": 0.7025390863418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.064453125,
      "rewards/margins": 0.10190429538488388,
      "rewards/rejected": -0.16645507514476776,
      "step": 8670
    },
    {
      "epoch": 0.6320311646703317,
      "grad_norm": 3.0112182870942354,
      "learning_rate": 8.586775814821837e-07,
      "log_odds_chosen": 1.3122069835662842,
      "log_odds_ratio": -0.4146972596645355,
      "logits/chosen": -0.988476574420929,
      "logits/rejected": -0.9019531011581421,
      "logps/chosen": -0.6078125238418579,
      "logps/rejected": -1.432031273841858,
      "loss": 0.7294,
      "nll_loss": 0.6962890625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06081543117761612,
      "rewards/margins": 0.08237304538488388,
      "rewards/rejected": -0.14321288466453552,
      "step": 8680
    },
    {
      "epoch": 0.6327593111734081,
      "grad_norm": 3.49160592853151,
      "learning_rate": 8.58183378516652e-07,
      "log_odds_chosen": 1.6382324695587158,
      "log_odds_ratio": -0.37348634004592896,
      "logits/chosen": -0.996874988079071,
      "logits/rejected": -0.9150390625,
      "logps/chosen": -0.611523449420929,
      "logps/rejected": -1.704492211341858,
      "loss": 0.7308,
      "nll_loss": 0.765332043170929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06120605394244194,
      "rewards/margins": 0.10914459079504013,
      "rewards/rejected": -0.17041015625,
      "step": 8690
    },
    {
      "epoch": 0.6334874576764845,
      "grad_norm": 2.6950284574307135,
      "learning_rate": 8.576900278702358e-07,
      "log_odds_chosen": 1.3389892578125,
      "log_odds_ratio": -0.4654296934604645,
      "logits/chosen": -1.0400390625,
      "logits/rejected": -0.9388672113418579,
      "logps/chosen": -0.698437511920929,
      "logps/rejected": -1.625,
      "loss": 0.741,
      "nll_loss": 0.753613293170929,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06978759914636612,
      "rewards/margins": 0.09270630031824112,
      "rewards/rejected": -0.16254882514476776,
      "step": 8700
    },
    {
      "epoch": 0.6342156041795609,
      "grad_norm": 3.1093802545042104,
      "learning_rate": 8.57197527095851e-07,
      "log_odds_chosen": 1.736914038658142,
      "log_odds_ratio": -0.3435302674770355,
      "logits/chosen": -1.009765625,
      "logits/rejected": -0.908984363079071,
      "logps/chosen": -0.6495116949081421,
      "logps/rejected": -1.873046875,
      "loss": 0.7268,
      "nll_loss": 0.6832031011581421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06499023735523224,
      "rewards/margins": 0.12216796725988388,
      "rewards/rejected": -0.187255859375,
      "step": 8710
    },
    {
      "epoch": 0.6349437506826373,
      "grad_norm": 2.4598129168880183,
      "learning_rate": 8.567058737562385e-07,
      "log_odds_chosen": 1.492089867591858,
      "log_odds_ratio": -0.4078125059604645,
      "logits/chosen": -0.9457031488418579,
      "logits/rejected": -0.8529297113418579,
      "logps/chosen": -0.6878906488418579,
      "logps/rejected": -1.7244141101837158,
      "loss": 0.7226,
      "nll_loss": 0.742968738079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06873778998851776,
      "rewards/margins": 0.10378112643957138,
      "rewards/rejected": -0.172607421875,
      "step": 8720
    },
    {
      "epoch": 0.6356718971857138,
      "grad_norm": 2.5666696761001635,
      "learning_rate": 8.562150654239141e-07,
      "log_odds_chosen": 1.879296898841858,
      "log_odds_ratio": -0.32685548067092896,
      "logits/chosen": -0.9970703125,
      "logits/rejected": -0.8861328363418579,
      "logps/chosen": -0.6070312261581421,
      "logps/rejected": -1.91796875,
      "loss": 0.7209,
      "nll_loss": 0.7181640863418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06068115308880806,
      "rewards/margins": 0.13105468451976776,
      "rewards/rejected": -0.19172362983226776,
      "step": 8730
    },
    {
      "epoch": 0.6364000436887902,
      "grad_norm": 2.9324468253580167,
      "learning_rate": 8.55725099681116e-07,
      "log_odds_chosen": 1.4736328125,
      "log_odds_ratio": -0.3765625059604645,
      "logits/chosen": -1.015234351158142,
      "logits/rejected": -0.882617175579071,
      "logps/chosen": -0.6484375,
      "logps/rejected": -1.6492187976837158,
      "loss": 0.7354,
      "nll_loss": 0.758984386920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06480713188648224,
      "rewards/margins": 0.09993896633386612,
      "rewards/rejected": -0.16484375298023224,
      "step": 8740
    },
    {
      "epoch": 0.6371281901918666,
      "grad_norm": 5.788306996857363,
      "learning_rate": 8.552359741197579e-07,
      "log_odds_chosen": 1.491308569908142,
      "log_odds_ratio": -0.3841308653354645,
      "logits/chosen": -1.0187499523162842,
      "logits/rejected": -0.905078113079071,
      "logps/chosen": -0.6529296636581421,
      "logps/rejected": -1.6931641101837158,
      "loss": 0.7102,
      "nll_loss": 0.6712890863418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06527099758386612,
      "rewards/margins": 0.10399780422449112,
      "rewards/rejected": -0.16926269233226776,
      "step": 8750
    },
    {
      "epoch": 0.637856336694943,
      "grad_norm": 2.749684170344462,
      "learning_rate": 8.547476863413765e-07,
      "log_odds_chosen": 1.792382836341858,
      "log_odds_ratio": -0.337158203125,
      "logits/chosen": -0.9912109375,
      "logits/rejected": -0.8794921636581421,
      "logps/chosen": -0.66064453125,
      "logps/rejected": -1.9406249523162842,
      "loss": 0.7166,
      "nll_loss": 0.7093750238418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06608887016773224,
      "rewards/margins": 0.12800292670726776,
      "rewards/rejected": -0.1939697265625,
      "step": 8760
    },
    {
      "epoch": 0.6385844831980194,
      "grad_norm": 2.8597608485635835,
      "learning_rate": 8.54260233957083e-07,
      "log_odds_chosen": 1.629492163658142,
      "log_odds_ratio": -0.37397462129592896,
      "logits/chosen": -1.025976538658142,
      "logits/rejected": -0.9222656488418579,
      "logps/chosen": -0.596484363079071,
      "logps/rejected": -1.73046875,
      "loss": 0.7106,
      "nll_loss": 0.6566406488418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05961913987994194,
      "rewards/margins": 0.11329345405101776,
      "rewards/rejected": -0.17307129502296448,
      "step": 8770
    },
    {
      "epoch": 0.6393126297010958,
      "grad_norm": 3.122350421100307,
      "learning_rate": 8.537736145875154e-07,
      "log_odds_chosen": 1.9109375476837158,
      "log_odds_ratio": -0.3287353515625,
      "logits/chosen": -1.0349609851837158,
      "logits/rejected": -0.932812511920929,
      "logps/chosen": -0.587890625,
      "logps/rejected": -1.8992187976837158,
      "loss": 0.7496,
      "nll_loss": 0.6631835699081421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05880127102136612,
      "rewards/margins": 0.13121338188648224,
      "rewards/rejected": -0.18996581435203552,
      "step": 8780
    },
    {
      "epoch": 0.6400407762041723,
      "grad_norm": 4.020176406459666,
      "learning_rate": 8.532878258627874e-07,
      "log_odds_chosen": 1.645898461341858,
      "log_odds_ratio": -0.3631347715854645,
      "logits/chosen": -1.030664086341858,
      "logits/rejected": -0.908984363079071,
      "logps/chosen": -0.6714843511581421,
      "logps/rejected": -1.787109375,
      "loss": 0.7373,
      "nll_loss": 0.7393554449081421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06711425632238388,
      "rewards/margins": 0.11164550483226776,
      "rewards/rejected": -0.17866210639476776,
      "step": 8790
    },
    {
      "epoch": 0.6407689227072487,
      "grad_norm": 2.483148691537567,
      "learning_rate": 8.528028654224416e-07,
      "log_odds_chosen": 1.67578125,
      "log_odds_ratio": -0.3345703184604645,
      "logits/chosen": -1.0382812023162842,
      "logits/rejected": -0.8919922113418579,
      "logps/chosen": -0.634472668170929,
      "logps/rejected": -1.759374976158142,
      "loss": 0.7345,
      "nll_loss": 0.6722656488418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06337890774011612,
      "rewards/margins": 0.11256103217601776,
      "rewards/rejected": -0.17585448920726776,
      "step": 8800
    },
    {
      "epoch": 0.6414970692103251,
      "grad_norm": 3.7041883846169283,
      "learning_rate": 8.523187309154008e-07,
      "log_odds_chosen": 1.7841796875,
      "log_odds_ratio": -0.34794920682907104,
      "logits/chosen": -0.9830077886581421,
      "logits/rejected": -0.872851550579071,
      "logps/chosen": -0.6065429449081421,
      "logps/rejected": -1.8542969226837158,
      "loss": 0.7108,
      "nll_loss": 0.654101550579071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06068115308880806,
      "rewards/margins": 0.12478027492761612,
      "rewards/rejected": -0.18544921278953552,
      "step": 8810
    },
    {
      "epoch": 0.6422252157134015,
      "grad_norm": 3.3737769601431893,
      "learning_rate": 8.518354199999198e-07,
      "log_odds_chosen": 1.3810546398162842,
      "log_odds_ratio": -0.41752928495407104,
      "logits/chosen": -1.0246093273162842,
      "logits/rejected": -0.8941406011581421,
      "logps/chosen": -0.662304699420929,
      "logps/rejected": -1.6306641101837158,
      "loss": 0.7144,
      "nll_loss": 0.6717773675918579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06618652492761612,
      "rewards/margins": 0.09685669094324112,
      "rewards/rejected": -0.16311034560203552,
      "step": 8820
    },
    {
      "epoch": 0.6429533622164779,
      "grad_norm": 2.7441716324506324,
      "learning_rate": 8.513529303435386e-07,
      "log_odds_chosen": 1.8293945789337158,
      "log_odds_ratio": -0.34636229276657104,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.8814452886581421,
      "logps/chosen": -0.63525390625,
      "logps/rejected": -1.9070312976837158,
      "loss": 0.7065,
      "nll_loss": 0.7318359613418579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06351318210363388,
      "rewards/margins": 0.12723389267921448,
      "rewards/rejected": -0.19084472954273224,
      "step": 8830
    },
    {
      "epoch": 0.6436815087195544,
      "grad_norm": 2.8818735854551045,
      "learning_rate": 8.50871259623034e-07,
      "log_odds_chosen": 1.758203148841858,
      "log_odds_ratio": -0.3658691346645355,
      "logits/chosen": -0.9917968511581421,
      "logits/rejected": -0.913281261920929,
      "logps/chosen": -0.6407226324081421,
      "logps/rejected": -1.876367211341858,
      "loss": 0.7336,
      "nll_loss": 0.7109375,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06405029445886612,
      "rewards/margins": 0.12348632514476776,
      "rewards/rejected": -0.18754883110523224,
      "step": 8840
    },
    {
      "epoch": 0.6444096552226308,
      "grad_norm": 2.8243982884850882,
      "learning_rate": 8.503904055243742e-07,
      "log_odds_chosen": 1.7492187023162842,
      "log_odds_ratio": -0.35161131620407104,
      "logits/chosen": -1.027734398841858,
      "logits/rejected": -0.9244140386581421,
      "logps/chosen": -0.588183581829071,
      "logps/rejected": -1.7607421875,
      "loss": 0.7075,
      "nll_loss": 0.642578125,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.058837890625,
      "rewards/margins": 0.11713866889476776,
      "rewards/rejected": -0.1759033203125,
      "step": 8850
    },
    {
      "epoch": 0.6451378017257072,
      "grad_norm": 2.983322770672721,
      "learning_rate": 8.499103657426704e-07,
      "log_odds_chosen": 1.6691405773162842,
      "log_odds_ratio": -0.364501953125,
      "logits/chosen": -1.0302734375,
      "logits/rejected": -0.8955078125,
      "logps/chosen": -0.5840820074081421,
      "logps/rejected": -1.7531249523162842,
      "loss": 0.7177,
      "nll_loss": 0.66845703125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.05841064453125,
      "rewards/margins": 0.11696777492761612,
      "rewards/rejected": -0.17526856064796448,
      "step": 8860
    },
    {
      "epoch": 0.6458659482287836,
      "grad_norm": 2.9981936439618013,
      "learning_rate": 8.494311379821314e-07,
      "log_odds_chosen": 1.80859375,
      "log_odds_ratio": -0.34443360567092896,
      "logits/chosen": -0.960742175579071,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.664355456829071,
      "logps/rejected": -1.960546851158142,
      "loss": 0.7026,
      "nll_loss": 0.689257800579071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06646728515625,
      "rewards/margins": 0.12965087592601776,
      "rewards/rejected": -0.1959228515625,
      "step": 8870
    },
    {
      "epoch": 0.64659409473186,
      "grad_norm": 2.5390642354697057,
      "learning_rate": 8.489527199560178e-07,
      "log_odds_chosen": 1.74169921875,
      "log_odds_ratio": -0.34892576932907104,
      "logits/chosen": -0.986328125,
      "logits/rejected": -0.8521484136581421,
      "logps/chosen": -0.6181640625,
      "logps/rejected": -1.806249976158142,
      "loss": 0.7001,
      "nll_loss": 0.672656238079071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06181640550494194,
      "rewards/margins": 0.11883239448070526,
      "rewards/rejected": -0.18071289360523224,
      "step": 8880
    },
    {
      "epoch": 0.6473222412349364,
      "grad_norm": 2.668263821670515,
      "learning_rate": 8.484751093865948e-07,
      "log_odds_chosen": 1.6982421875,
      "log_odds_ratio": -0.3606201112270355,
      "logits/chosen": -1.040429711341858,
      "logits/rejected": -0.9267578125,
      "logps/chosen": -0.652148425579071,
      "logps/rejected": -1.883203148841858,
      "loss": 0.7109,
      "nll_loss": 0.7476562261581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06528320163488388,
      "rewards/margins": 0.12298583984375,
      "rewards/rejected": -0.188232421875,
      "step": 8890
    },
    {
      "epoch": 0.6480503877380129,
      "grad_norm": 2.9825648534532125,
      "learning_rate": 8.47998304005088e-07,
      "log_odds_chosen": 1.783789038658142,
      "log_odds_ratio": -0.3736816346645355,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.8824218511581421,
      "logps/chosen": -0.6688476800918579,
      "logps/rejected": -1.9539062976837158,
      "loss": 0.7169,
      "nll_loss": 0.672070324420929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06684570014476776,
      "rewards/margins": 0.12838134169578552,
      "rewards/rejected": -0.1951904296875,
      "step": 8900
    },
    {
      "epoch": 0.6487785342410893,
      "grad_norm": 2.954067402014089,
      "learning_rate": 8.475223015516377e-07,
      "log_odds_chosen": 1.905859351158142,
      "log_odds_ratio": -0.34211426973342896,
      "logits/chosen": -1.0392577648162842,
      "logits/rejected": -0.9130859375,
      "logps/chosen": -0.6063476800918579,
      "logps/rejected": -1.964453101158142,
      "loss": 0.7234,
      "nll_loss": 0.641894519329071,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.0606689453125,
      "rewards/margins": 0.13565674424171448,
      "rewards/rejected": -0.19636230170726776,
      "step": 8910
    },
    {
      "epoch": 0.6495066807441657,
      "grad_norm": 2.992527599581005,
      "learning_rate": 8.470470997752534e-07,
      "log_odds_chosen": 1.740820288658142,
      "log_odds_ratio": -0.3729492127895355,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.876953125,
      "logps/chosen": -0.6712890863418579,
      "logps/rejected": -1.9269530773162842,
      "loss": 0.7101,
      "nll_loss": 0.719531238079071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06708984076976776,
      "rewards/margins": 0.12564697861671448,
      "rewards/rejected": -0.19279785454273224,
      "step": 8920
    },
    {
      "epoch": 0.6502348272472421,
      "grad_norm": 2.837068810401583,
      "learning_rate": 8.465726964337702e-07,
      "log_odds_chosen": 1.754492163658142,
      "log_odds_ratio": -0.34423828125,
      "logits/chosen": -1.005468726158142,
      "logits/rejected": -0.8861328363418579,
      "logps/chosen": -0.645312488079071,
      "logps/rejected": -1.8914062976837158,
      "loss": 0.7424,
      "nll_loss": 0.739453136920929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06453857570886612,
      "rewards/margins": 0.124755859375,
      "rewards/rejected": -0.18937988579273224,
      "step": 8930
    },
    {
      "epoch": 0.6509629737503185,
      "grad_norm": 2.897896908200356,
      "learning_rate": 8.460990892938031e-07,
      "log_odds_chosen": 1.7654297351837158,
      "log_odds_ratio": -0.34599608182907104,
      "logits/chosen": -1.004296898841858,
      "logits/rejected": -0.8876953125,
      "logps/chosen": -0.6380859613418579,
      "logps/rejected": -1.859765648841858,
      "loss": 0.7055,
      "nll_loss": 0.665234386920929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06379394233226776,
      "rewards/margins": 0.12226562201976776,
      "rewards/rejected": -0.18593749403953552,
      "step": 8940
    },
    {
      "epoch": 0.6516911202533949,
      "grad_norm": 2.442097135186036,
      "learning_rate": 8.456262761307038e-07,
      "log_odds_chosen": 1.4421875476837158,
      "log_odds_ratio": -0.4010253846645355,
      "logits/chosen": -0.9585937261581421,
      "logits/rejected": -0.8746093511581421,
      "logps/chosen": -0.670117199420929,
      "logps/rejected": -1.641015648841858,
      "loss": 0.7372,
      "nll_loss": 0.705859363079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06697998195886612,
      "rewards/margins": 0.09714355319738388,
      "rewards/rejected": -0.16413573920726776,
      "step": 8950
    },
    {
      "epoch": 0.6524192667564714,
      "grad_norm": 2.592170957943941,
      "learning_rate": 8.451542547285166e-07,
      "log_odds_chosen": 1.373437523841858,
      "log_odds_ratio": -0.4225097596645355,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.896484375,
      "logps/chosen": -0.676562488079071,
      "logps/rejected": -1.6281249523162842,
      "loss": 0.7324,
      "nll_loss": 0.7587890625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06766357272863388,
      "rewards/margins": 0.09521789848804474,
      "rewards/rejected": -0.16289062798023224,
      "step": 8960
    },
    {
      "epoch": 0.6531474132595478,
      "grad_norm": 2.98775829839562,
      "learning_rate": 8.44683022879934e-07,
      "log_odds_chosen": 1.5964844226837158,
      "log_odds_ratio": -0.37199705839157104,
      "logits/chosen": -0.9593750238418579,
      "logits/rejected": -0.838183581829071,
      "logps/chosen": -0.621289074420929,
      "logps/rejected": -1.697265625,
      "loss": 0.7042,
      "nll_loss": 0.67041015625,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06219482421875,
      "rewards/margins": 0.1075439453125,
      "rewards/rejected": -0.16977539658546448,
      "step": 8970
    },
    {
      "epoch": 0.6538755597626242,
      "grad_norm": 5.826606972549238,
      "learning_rate": 8.442125783862544e-07,
      "log_odds_chosen": 1.7585937976837158,
      "log_odds_ratio": -0.39130860567092896,
      "logits/chosen": -0.951953113079071,
      "logits/rejected": -0.88671875,
      "logps/chosen": -0.6485351324081421,
      "logps/rejected": -1.887109398841858,
      "loss": 0.7057,
      "nll_loss": 0.6397460699081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06484375149011612,
      "rewards/margins": 0.123870849609375,
      "rewards/rejected": -0.18869629502296448,
      "step": 8980
    },
    {
      "epoch": 0.6546037062657006,
      "grad_norm": 4.317271370295915,
      "learning_rate": 8.437429190573388e-07,
      "log_odds_chosen": 1.768945336341858,
      "log_odds_ratio": -0.3340820372104645,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.878125011920929,
      "logps/chosen": -0.655078113079071,
      "logps/rejected": -1.9347655773162842,
      "loss": 0.732,
      "nll_loss": 0.774121105670929,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06547851860523224,
      "rewards/margins": 0.12784424424171448,
      "rewards/rejected": -0.19331054389476776,
      "step": 8990
    },
    {
      "epoch": 0.655331852768777,
      "grad_norm": 2.5015131326763105,
      "learning_rate": 8.432740427115678e-07,
      "log_odds_chosen": 1.5925781726837158,
      "log_odds_ratio": -0.39116209745407104,
      "logits/chosen": -0.985156238079071,
      "logits/rejected": -0.8509765863418579,
      "logps/chosen": -0.64111328125,
      "logps/rejected": -1.736718773841858,
      "loss": 0.7346,
      "nll_loss": 0.6512695550918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06409911811351776,
      "rewards/margins": 0.10950927436351776,
      "rewards/rejected": -0.1737060546875,
      "step": 9000
    },
    {
      "epoch": 0.6560599992718535,
      "grad_norm": 3.4331155893816794,
      "learning_rate": 8.428059471757984e-07,
      "log_odds_chosen": 1.57666015625,
      "log_odds_ratio": -0.38813477754592896,
      "logits/chosen": -0.978320300579071,
      "logits/rejected": -0.8568359613418579,
      "logps/chosen": -0.653613269329071,
      "logps/rejected": -1.7443358898162842,
      "loss": 0.7229,
      "nll_loss": 0.676074206829071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06539306789636612,
      "rewards/margins": 0.10916747897863388,
      "rewards/rejected": -0.17453613877296448,
      "step": 9010
    },
    {
      "epoch": 0.6567881457749299,
      "grad_norm": 2.5220231044268346,
      "learning_rate": 8.423386302853226e-07,
      "log_odds_chosen": 1.3791992664337158,
      "log_odds_ratio": -0.387939453125,
      "logits/chosen": -0.941601574420929,
      "logits/rejected": -0.8333984613418579,
      "logps/chosen": -0.62841796875,
      "logps/rejected": -1.511328101158142,
      "loss": 0.7149,
      "nll_loss": 0.7098633050918579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06285400688648224,
      "rewards/margins": 0.08829345554113388,
      "rewards/rejected": -0.15119628608226776,
      "step": 9020
    },
    {
      "epoch": 0.6575162922780063,
      "grad_norm": 2.9232519578416754,
      "learning_rate": 8.418720898838254e-07,
      "log_odds_chosen": 1.725000023841858,
      "log_odds_ratio": -0.36572265625,
      "logits/chosen": -0.9898437261581421,
      "logits/rejected": -0.8541015386581421,
      "logps/chosen": -0.616015613079071,
      "logps/rejected": -1.7898437976837158,
      "loss": 0.7367,
      "nll_loss": 0.7559570074081421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06162109225988388,
      "rewards/margins": 0.11744384467601776,
      "rewards/rejected": -0.17902831733226776,
      "step": 9030
    },
    {
      "epoch": 0.6582444387810827,
      "grad_norm": 3.169548259969985,
      "learning_rate": 8.414063238233425e-07,
      "log_odds_chosen": 1.6552734375,
      "log_odds_ratio": -0.3829101622104645,
      "logits/chosen": -1.0304687023162842,
      "logits/rejected": -0.8921874761581421,
      "logps/chosen": -0.643750011920929,
      "logps/rejected": -1.808984398841858,
      "loss": 0.7259,
      "nll_loss": 0.7079101800918579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06441650539636612,
      "rewards/margins": 0.11655273288488388,
      "rewards/rejected": -0.18076172471046448,
      "step": 9040
    },
    {
      "epoch": 0.6589725852841591,
      "grad_norm": 3.2505506131992807,
      "learning_rate": 8.409413299642188e-07,
      "log_odds_chosen": 1.778906226158142,
      "log_odds_ratio": -0.36298829317092896,
      "logits/chosen": -1.0486328601837158,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -0.6455078125,
      "logps/rejected": -1.9148437976837158,
      "loss": 0.6815,
      "nll_loss": 0.638476550579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06455077975988388,
      "rewards/margins": 0.12685546278953552,
      "rewards/rejected": -0.19133301079273224,
      "step": 9050
    },
    {
      "epoch": 0.6597007317872355,
      "grad_norm": 3.0929307975278926,
      "learning_rate": 8.404771061750672e-07,
      "log_odds_chosen": 1.7316405773162842,
      "log_odds_ratio": -0.38115233182907104,
      "logits/chosen": -1.022851586341858,
      "logits/rejected": -0.878125011920929,
      "logps/chosen": -0.6421874761581421,
      "logps/rejected": -1.8708984851837158,
      "loss": 0.7012,
      "nll_loss": 0.6729491949081421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06425781548023224,
      "rewards/margins": 0.12291260063648224,
      "rewards/rejected": -0.18710938096046448,
      "step": 9060
    },
    {
      "epoch": 0.660428878290312,
      "grad_norm": 3.1422303766275483,
      "learning_rate": 8.400136503327277e-07,
      "log_odds_chosen": 1.68212890625,
      "log_odds_ratio": -0.325439453125,
      "logits/chosen": -1.0021483898162842,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.6239258050918579,
      "logps/rejected": -1.7917969226837158,
      "loss": 0.7031,
      "nll_loss": 0.6548827886581421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06242675706744194,
      "rewards/margins": 0.11680908501148224,
      "rewards/rejected": -0.1790771484375,
      "step": 9070
    },
    {
      "epoch": 0.6611570247933884,
      "grad_norm": 2.782005652385191,
      "learning_rate": 8.395509603222271e-07,
      "log_odds_chosen": 1.701171875,
      "log_odds_ratio": -0.32573240995407104,
      "logits/chosen": -0.9330078363418579,
      "logits/rejected": -0.832226574420929,
      "logps/chosen": -0.610546886920929,
      "logps/rejected": -1.767968773841858,
      "loss": 0.7211,
      "nll_loss": 0.719042956829071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06102294847369194,
      "rewards/margins": 0.1158447265625,
      "rewards/rejected": -0.17702636122703552,
      "step": 9080
    },
    {
      "epoch": 0.6618851712964648,
      "grad_norm": 2.726635924845548,
      "learning_rate": 8.390890340367368e-07,
      "log_odds_chosen": 1.51220703125,
      "log_odds_ratio": -0.40043944120407104,
      "logits/chosen": -0.9994140863418579,
      "logits/rejected": -0.8744140863418579,
      "logps/chosen": -0.64599609375,
      "logps/rejected": -1.707421898841858,
      "loss": 0.7222,
      "nll_loss": 0.6640625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06452636420726776,
      "rewards/margins": 0.10614013671875,
      "rewards/rejected": -0.17070312798023224,
      "step": 9090
    },
    {
      "epoch": 0.6626133177995412,
      "grad_norm": 2.690171134281044,
      "learning_rate": 8.386278693775346e-07,
      "log_odds_chosen": 1.7841796875,
      "log_odds_ratio": -0.3144287168979645,
      "logits/chosen": -0.9869140386581421,
      "logits/rejected": -0.868945300579071,
      "logps/chosen": -0.6075195074081421,
      "logps/rejected": -1.799218773841858,
      "loss": 0.7091,
      "nll_loss": 0.7266601324081421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06076660007238388,
      "rewards/margins": 0.11928711086511612,
      "rewards/rejected": -0.17995604872703552,
      "step": 9100
    },
    {
      "epoch": 0.6633414643026176,
      "grad_norm": 2.6615208309932648,
      "learning_rate": 8.381674642539632e-07,
      "log_odds_chosen": 1.530126929283142,
      "log_odds_ratio": -0.39936524629592896,
      "logits/chosen": -1.016015648841858,
      "logits/rejected": -0.8921874761581421,
      "logps/chosen": -0.719921886920929,
      "logps/rejected": -1.8123047351837158,
      "loss": 0.7119,
      "nll_loss": 0.723828136920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07199706882238388,
      "rewards/margins": 0.10925903171300888,
      "rewards/rejected": -0.1812744140625,
      "step": 9110
    },
    {
      "epoch": 0.6640696108056942,
      "grad_norm": 2.964966557558039,
      "learning_rate": 8.37707816583391e-07,
      "log_odds_chosen": 1.527734398841858,
      "log_odds_ratio": -0.40434569120407104,
      "logits/chosen": -1.0185546875,
      "logits/rejected": -0.9287109375,
      "logps/chosen": -0.6244140863418579,
      "logps/rejected": -1.678125023841858,
      "loss": 0.706,
      "nll_loss": 0.7158203125,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06241454929113388,
      "rewards/margins": 0.10535583645105362,
      "rewards/rejected": -0.16779784858226776,
      "step": 9120
    },
    {
      "epoch": 0.6647977573087706,
      "grad_norm": 3.2854638572013624,
      "learning_rate": 8.372489242911724e-07,
      "log_odds_chosen": 1.6374022960662842,
      "log_odds_ratio": -0.40300291776657104,
      "logits/chosen": -0.9554687738418579,
      "logits/rejected": -0.8265625238418579,
      "logps/chosen": -0.713671863079071,
      "logps/rejected": -1.875,
      "loss": 0.7158,
      "nll_loss": 0.734375,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07139892876148224,
      "rewards/margins": 0.1160888671875,
      "rewards/rejected": -0.18745116889476776,
      "step": 9130
    },
    {
      "epoch": 0.665525903811847,
      "grad_norm": 2.7513964113474976,
      "learning_rate": 8.367907853106078e-07,
      "log_odds_chosen": 1.689062476158142,
      "log_odds_ratio": -0.33509522676467896,
      "logits/chosen": -1.015039086341858,
      "logits/rejected": -0.88671875,
      "logps/chosen": -0.6070312261581421,
      "logps/rejected": -1.7458984851837158,
      "loss": 0.7087,
      "nll_loss": 0.7841796875,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06068115308880806,
      "rewards/margins": 0.11386718600988388,
      "rewards/rejected": -0.17463378608226776,
      "step": 9140
    },
    {
      "epoch": 0.6662540503149234,
      "grad_norm": 3.1040796275107163,
      "learning_rate": 8.363333975829066e-07,
      "log_odds_chosen": 1.4833984375,
      "log_odds_ratio": -0.35454100370407104,
      "logits/chosen": -1.01171875,
      "logits/rejected": -0.884765625,
      "logps/chosen": -0.59716796875,
      "logps/rejected": -1.5587890148162842,
      "loss": 0.7226,
      "nll_loss": 0.6504882574081421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05972900241613388,
      "rewards/margins": 0.09602661430835724,
      "rewards/rejected": -0.15566405653953552,
      "step": 9150
    },
    {
      "epoch": 0.6669821968179998,
      "grad_norm": 3.3700602162113578,
      "learning_rate": 8.358767590571457e-07,
      "log_odds_chosen": 1.812890648841858,
      "log_odds_ratio": -0.331787109375,
      "logits/chosen": -1.020898461341858,
      "logits/rejected": -0.907031238079071,
      "logps/chosen": -0.6015625,
      "logps/rejected": -1.878515601158142,
      "loss": 0.7356,
      "nll_loss": 0.697460949420929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06016845628619194,
      "rewards/margins": 0.12773437798023224,
      "rewards/rejected": -0.18789061903953552,
      "step": 9160
    },
    {
      "epoch": 0.6677103433210761,
      "grad_norm": 2.9946396272053337,
      "learning_rate": 8.354208676902326e-07,
      "log_odds_chosen": 1.5712890625,
      "log_odds_ratio": -0.40581053495407104,
      "logits/chosen": -1.012304663658142,
      "logits/rejected": -0.9136718511581421,
      "logps/chosen": -0.666210949420929,
      "logps/rejected": -1.7820312976837158,
      "loss": 0.71,
      "nll_loss": 0.6924804449081421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06658935546875,
      "rewards/margins": 0.11163330078125,
      "rewards/rejected": -0.17819824814796448,
      "step": 9170
    },
    {
      "epoch": 0.6684384898241527,
      "grad_norm": 2.7237576895390827,
      "learning_rate": 8.349657214468659e-07,
      "log_odds_chosen": 1.6281249523162842,
      "log_odds_ratio": -0.3890136778354645,
      "logits/chosen": -1.008203148841858,
      "logits/rejected": -0.9029296636581421,
      "logps/chosen": -0.6055663824081421,
      "logps/rejected": -1.6962890625,
      "loss": 0.7191,
      "nll_loss": 0.67626953125,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06058349460363388,
      "rewards/margins": 0.10899658501148224,
      "rewards/rejected": -0.16943359375,
      "step": 9180
    },
    {
      "epoch": 0.6691666363272291,
      "grad_norm": 2.7654455917352068,
      "learning_rate": 8.345113182994988e-07,
      "log_odds_chosen": 1.4718749523162842,
      "log_odds_ratio": -0.4141601622104645,
      "logits/chosen": -1.0320312976837158,
      "logits/rejected": -0.8763672113418579,
      "logps/chosen": -0.645312488079071,
      "logps/rejected": -1.6554687023162842,
      "loss": 0.7054,
      "nll_loss": 0.6714843511581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06452636420726776,
      "rewards/margins": 0.10100097954273224,
      "rewards/rejected": -0.16550293564796448,
      "step": 9190
    },
    {
      "epoch": 0.6698947828303055,
      "grad_norm": 2.7095205311626605,
      "learning_rate": 8.34057656228299e-07,
      "log_odds_chosen": 1.546484351158142,
      "log_odds_ratio": -0.40668946504592896,
      "logits/chosen": -1.0322265625,
      "logits/rejected": -0.935351550579071,
      "logps/chosen": -0.64794921875,
      "logps/rejected": -1.70703125,
      "loss": 0.6987,
      "nll_loss": 0.67236328125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0648193359375,
      "rewards/margins": 0.10593261569738388,
      "rewards/rejected": -0.17075195908546448,
      "step": 9200
    },
    {
      "epoch": 0.6706229293333819,
      "grad_norm": 2.5760555572856294,
      "learning_rate": 8.336047332211128e-07,
      "log_odds_chosen": 1.558984398841858,
      "log_odds_ratio": -0.4190429747104645,
      "logits/chosen": -0.9730468988418579,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.670214831829071,
      "logps/rejected": -1.786718726158142,
      "loss": 0.7329,
      "nll_loss": 0.7281249761581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06702880561351776,
      "rewards/margins": 0.11182861030101776,
      "rewards/rejected": -0.17885741591453552,
      "step": 9210
    },
    {
      "epoch": 0.6713510758364583,
      "grad_norm": 3.1218227431003025,
      "learning_rate": 8.331525472734267e-07,
      "log_odds_chosen": 1.558447241783142,
      "log_odds_ratio": -0.3697753846645355,
      "logits/chosen": -1.0224609375,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.623339831829071,
      "logps/rejected": -1.709570288658142,
      "loss": 0.7221,
      "nll_loss": 0.765820324420929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06224365159869194,
      "rewards/margins": 0.1087188720703125,
      "rewards/rejected": -0.171142578125,
      "step": 9220
    },
    {
      "epoch": 0.6720792223395348,
      "grad_norm": 3.2225849447646926,
      "learning_rate": 8.327010963883302e-07,
      "log_odds_chosen": 1.83935546875,
      "log_odds_ratio": -0.36323243379592896,
      "logits/chosen": -1.0068359375,
      "logits/rejected": -0.8974609375,
      "logps/chosen": -0.646484375,
      "logps/rejected": -1.9333984851837158,
      "loss": 0.7051,
      "nll_loss": 0.6478515863418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06470946967601776,
      "rewards/margins": 0.12897948920726776,
      "rewards/rejected": -0.19350585341453552,
      "step": 9230
    },
    {
      "epoch": 0.6728073688426112,
      "grad_norm": 2.4050347615788374,
      "learning_rate": 8.322503785764789e-07,
      "log_odds_chosen": 1.7651855945587158,
      "log_odds_ratio": -0.3617187440395355,
      "logits/chosen": -0.978320300579071,
      "logits/rejected": -0.9037109613418579,
      "logps/chosen": -0.6153320074081421,
      "logps/rejected": -1.8445312976837158,
      "loss": 0.6804,
      "nll_loss": 0.655468761920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06158447265625,
      "rewards/margins": 0.12314452975988388,
      "rewards/rejected": -0.1846923828125,
      "step": 9240
    },
    {
      "epoch": 0.6735355153456876,
      "grad_norm": 2.9940372148878267,
      "learning_rate": 8.318003918560583e-07,
      "log_odds_chosen": 1.8566405773162842,
      "log_odds_ratio": -0.3237548768520355,
      "logits/chosen": -1.014257788658142,
      "logits/rejected": -0.8675781488418579,
      "logps/chosen": -0.5791015625,
      "logps/rejected": -1.842187523841858,
      "loss": 0.7173,
      "nll_loss": 0.6880859136581421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.05788574367761612,
      "rewards/margins": 0.12626953423023224,
      "rewards/rejected": -0.18403320014476776,
      "step": 9250
    },
    {
      "epoch": 0.674263661848764,
      "grad_norm": 3.5285000752405606,
      "learning_rate": 8.313511342527453e-07,
      "log_odds_chosen": 1.729882836341858,
      "log_odds_ratio": -0.35761719942092896,
      "logits/chosen": -1.0021483898162842,
      "logits/rejected": -0.8851562738418579,
      "logps/chosen": -0.63525390625,
      "logps/rejected": -1.854101538658142,
      "loss": 0.7292,
      "nll_loss": 0.668164074420929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06350097805261612,
      "rewards/margins": 0.12175293266773224,
      "rewards/rejected": -0.185302734375,
      "step": 9260
    },
    {
      "epoch": 0.6749918083518404,
      "grad_norm": 3.861025926964799,
      "learning_rate": 8.309026037996745e-07,
      "log_odds_chosen": 1.6337890625,
      "log_odds_ratio": -0.3812011778354645,
      "logits/chosen": -1.0283203125,
      "logits/rejected": -0.922656238079071,
      "logps/chosen": -0.621874988079071,
      "logps/rejected": -1.7599608898162842,
      "loss": 0.6939,
      "nll_loss": 0.677734375,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06220703199505806,
      "rewards/margins": 0.11384277045726776,
      "rewards/rejected": -0.17600098252296448,
      "step": 9270
    },
    {
      "epoch": 0.6757199548549168,
      "grad_norm": 2.873747911638871,
      "learning_rate": 8.304547985373996e-07,
      "log_odds_chosen": 1.6433594226837158,
      "log_odds_ratio": -0.36323243379592896,
      "logits/chosen": -0.9994140863418579,
      "logits/rejected": -0.919140636920929,
      "logps/chosen": -0.5987304449081421,
      "logps/rejected": -1.712499976158142,
      "loss": 0.7149,
      "nll_loss": 0.652050793170929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05992431566119194,
      "rewards/margins": 0.11142577975988388,
      "rewards/rejected": -0.17133788764476776,
      "step": 9280
    },
    {
      "epoch": 0.6764481013579933,
      "grad_norm": 2.7023506995235262,
      "learning_rate": 8.300077165138592e-07,
      "log_odds_chosen": 1.7566406726837158,
      "log_odds_ratio": -0.3354736268520355,
      "logits/chosen": -1.0587890148162842,
      "logits/rejected": -0.912890613079071,
      "logps/chosen": -0.6695312261581421,
      "logps/rejected": -1.9308593273162842,
      "loss": 0.724,
      "nll_loss": 0.6576172113418579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06694336235523224,
      "rewards/margins": 0.12629394233226776,
      "rewards/rejected": -0.19306640326976776,
      "step": 9290
    },
    {
      "epoch": 0.6771762478610697,
      "grad_norm": 4.215004107253507,
      "learning_rate": 8.295613557843402e-07,
      "log_odds_chosen": 1.5089843273162842,
      "log_odds_ratio": -0.37651365995407104,
      "logits/chosen": -1.0314452648162842,
      "logits/rejected": -0.868945300579071,
      "logps/chosen": -0.6426757574081421,
      "logps/rejected": -1.69140625,
      "loss": 0.7226,
      "nll_loss": 0.655078113079071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06425781548023224,
      "rewards/margins": 0.10482177883386612,
      "rewards/rejected": -0.16914062201976776,
      "step": 9300
    },
    {
      "epoch": 0.6779043943641461,
      "grad_norm": 3.1319869126606297,
      "learning_rate": 8.291157144114419e-07,
      "log_odds_chosen": 1.799414038658142,
      "log_odds_ratio": -0.3438720703125,
      "logits/chosen": -1.011132836341858,
      "logits/rejected": -0.921093761920929,
      "logps/chosen": -0.662109375,
      "logps/rejected": -1.8781249523162842,
      "loss": 0.7268,
      "nll_loss": 0.6734374761581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06622314453125,
      "rewards/margins": 0.12183837592601776,
      "rewards/rejected": -0.18789061903953552,
      "step": 9310
    },
    {
      "epoch": 0.6786325408672225,
      "grad_norm": 2.93607052001491,
      "learning_rate": 8.286707904650417e-07,
      "log_odds_chosen": 1.867578148841858,
      "log_odds_ratio": -0.34980469942092896,
      "logits/chosen": -1.0099608898162842,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.6429687738418579,
      "logps/rejected": -1.9128906726837158,
      "loss": 0.7069,
      "nll_loss": 0.7256835699081421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06430663913488388,
      "rewards/margins": 0.1270751953125,
      "rewards/rejected": -0.19125977158546448,
      "step": 9320
    },
    {
      "epoch": 0.6793606873702989,
      "grad_norm": 2.9800392503695217,
      "learning_rate": 8.282265820222593e-07,
      "log_odds_chosen": 1.5031249523162842,
      "log_odds_ratio": -0.38471680879592896,
      "logits/chosen": -1.01171875,
      "logits/rejected": -0.88671875,
      "logps/chosen": -0.619824230670929,
      "logps/rejected": -1.6632812023162842,
      "loss": 0.7044,
      "nll_loss": 0.6865234375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06201171875,
      "rewards/margins": 0.10441894829273224,
      "rewards/rejected": -0.16640624403953552,
      "step": 9330
    },
    {
      "epoch": 0.6800888338733754,
      "grad_norm": 3.45340920697323,
      "learning_rate": 8.277830871674222e-07,
      "log_odds_chosen": 1.663671851158142,
      "log_odds_ratio": -0.35249024629592896,
      "logits/chosen": -1.0607421398162842,
      "logits/rejected": -0.9107421636581421,
      "logps/chosen": -0.658007800579071,
      "logps/rejected": -1.8351562023162842,
      "loss": 0.7094,
      "nll_loss": 0.71533203125,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06582031399011612,
      "rewards/margins": 0.11778564751148224,
      "rewards/rejected": -0.18344727158546448,
      "step": 9340
    },
    {
      "epoch": 0.6808169803764518,
      "grad_norm": 2.591084756716443,
      "learning_rate": 8.273403039920306e-07,
      "log_odds_chosen": 1.939794898033142,
      "log_odds_ratio": -0.3537353575229645,
      "logits/chosen": -1.001367211341858,
      "logits/rejected": -0.9183593988418579,
      "logps/chosen": -0.6563476324081421,
      "logps/rejected": -2.0537109375,
      "loss": 0.6862,
      "nll_loss": 0.647656261920929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06562499701976776,
      "rewards/margins": 0.139862060546875,
      "rewards/rejected": -0.20554199814796448,
      "step": 9350
    },
    {
      "epoch": 0.6815451268795282,
      "grad_norm": 2.8728367297011927,
      "learning_rate": 8.268982305947231e-07,
      "log_odds_chosen": 2.14453125,
      "log_odds_ratio": -0.2612548768520355,
      "logits/chosen": -1.056640625,
      "logits/rejected": -0.9251953363418579,
      "logps/chosen": -0.585253894329071,
      "logps/rejected": -2.067187547683716,
      "loss": 0.7001,
      "nll_loss": 0.6162109375,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.05845947191119194,
      "rewards/margins": 0.14809569716453552,
      "rewards/rejected": -0.20654296875,
      "step": 9360
    },
    {
      "epoch": 0.6822732733826046,
      "grad_norm": 2.8441030277756996,
      "learning_rate": 8.264568650812423e-07,
      "log_odds_chosen": 2.1292967796325684,
      "log_odds_ratio": -0.2952880859375,
      "logits/chosen": -1.045312523841858,
      "logits/rejected": -0.904492199420929,
      "logps/chosen": -0.601855456829071,
      "logps/rejected": -2.1556639671325684,
      "loss": 0.709,
      "nll_loss": 0.619921863079071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06020507961511612,
      "rewards/margins": 0.155517578125,
      "rewards/rejected": -0.21547850966453552,
      "step": 9370
    },
    {
      "epoch": 0.683001419885681,
      "grad_norm": 2.916360366444195,
      "learning_rate": 8.26016205564401e-07,
      "log_odds_chosen": 1.516699194908142,
      "log_odds_ratio": -0.39472657442092896,
      "logits/chosen": -0.993945300579071,
      "logits/rejected": -0.853515625,
      "logps/chosen": -0.6454101800918579,
      "logps/rejected": -1.6951172351837158,
      "loss": 0.7248,
      "nll_loss": 0.702929675579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06452636420726776,
      "rewards/margins": 0.10498962551355362,
      "rewards/rejected": -0.16953125596046448,
      "step": 9380
    },
    {
      "epoch": 0.6837295663887574,
      "grad_norm": 2.644303441504737,
      "learning_rate": 8.25576250164048e-07,
      "log_odds_chosen": 1.495507836341858,
      "log_odds_ratio": -0.37617188692092896,
      "logits/chosen": -1.0099608898162842,
      "logits/rejected": -0.91015625,
      "logps/chosen": -0.6522461175918579,
      "logps/rejected": -1.7179687023162842,
      "loss": 0.7104,
      "nll_loss": 0.700390636920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06529541313648224,
      "rewards/margins": 0.10661621391773224,
      "rewards/rejected": -0.17185059189796448,
      "step": 9390
    },
    {
      "epoch": 0.6844577128918339,
      "grad_norm": 3.040135954992706,
      "learning_rate": 8.251369970070346e-07,
      "log_odds_chosen": 1.769921898841858,
      "log_odds_ratio": -0.3707031309604645,
      "logits/chosen": -1.0031249523162842,
      "logits/rejected": -0.908203125,
      "logps/chosen": -0.6578124761581421,
      "logps/rejected": -1.886328101158142,
      "loss": 0.6983,
      "nll_loss": 0.7083984613418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06577148288488388,
      "rewards/margins": 0.12294922024011612,
      "rewards/rejected": -0.18879394233226776,
      "step": 9400
    },
    {
      "epoch": 0.6851858593949103,
      "grad_norm": 3.0674648642334974,
      "learning_rate": 8.246984442271813e-07,
      "log_odds_chosen": 1.7744140625,
      "log_odds_ratio": -0.2940429747104645,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.876953125,
      "logps/chosen": -0.596386730670929,
      "logps/rejected": -1.771875023841858,
      "loss": 0.6965,
      "nll_loss": 0.68994140625,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.05959472805261612,
      "rewards/margins": 0.1177978515625,
      "rewards/rejected": -0.17729492485523224,
      "step": 9410
    },
    {
      "epoch": 0.6859140058979867,
      "grad_norm": 2.3686073671623475,
      "learning_rate": 8.242605899652435e-07,
      "log_odds_chosen": 1.445410132408142,
      "log_odds_ratio": -0.424072265625,
      "logits/chosen": -1.0080077648162842,
      "logits/rejected": -0.877148449420929,
      "logps/chosen": -0.673535168170929,
      "logps/rejected": -1.657617211341858,
      "loss": 0.6852,
      "nll_loss": 0.678027331829071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06739501655101776,
      "rewards/margins": 0.09858398139476776,
      "rewards/rejected": -0.16596679389476776,
      "step": 9420
    },
    {
      "epoch": 0.6866421524010631,
      "grad_norm": 9.595767688210557,
      "learning_rate": 8.238234323688798e-07,
      "log_odds_chosen": 1.890234351158142,
      "log_odds_ratio": -0.3437744081020355,
      "logits/chosen": -1.051367163658142,
      "logits/rejected": -0.9193359613418579,
      "logps/chosen": -0.674609363079071,
      "logps/rejected": -2.0425782203674316,
      "loss": 0.7111,
      "nll_loss": 0.6778320074081421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06744384765625,
      "rewards/margins": 0.13673095405101776,
      "rewards/rejected": -0.2041015625,
      "step": 9430
    },
    {
      "epoch": 0.6873702989041395,
      "grad_norm": 3.770134132150641,
      "learning_rate": 8.233869695926182e-07,
      "log_odds_chosen": 1.994140625,
      "log_odds_ratio": -0.320556640625,
      "logits/chosen": -1.048828125,
      "logits/rejected": -0.931640625,
      "logps/chosen": -0.610156238079071,
      "logps/rejected": -1.9894530773162842,
      "loss": 0.6862,
      "nll_loss": 0.6104491949081421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06098632887005806,
      "rewards/margins": 0.13801269233226776,
      "rewards/rejected": -0.19892577826976776,
      "step": 9440
    },
    {
      "epoch": 0.688098445407216,
      "grad_norm": 2.5940737903088014,
      "learning_rate": 8.229511997978235e-07,
      "log_odds_chosen": 1.4591796398162842,
      "log_odds_ratio": -0.4232421815395355,
      "logits/chosen": -1.032812476158142,
      "logits/rejected": -0.875781238079071,
      "logps/chosen": -0.6488281488418579,
      "logps/rejected": -1.664453148841858,
      "loss": 0.7074,
      "nll_loss": 0.644726574420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06488037109375,
      "rewards/margins": 0.10173340141773224,
      "rewards/rejected": -0.16652831435203552,
      "step": 9450
    },
    {
      "epoch": 0.6888265919102924,
      "grad_norm": 3.1690002746077846,
      "learning_rate": 8.22516121152665e-07,
      "log_odds_chosen": 1.72998046875,
      "log_odds_ratio": -0.3265136778354645,
      "logits/chosen": -1.0322265625,
      "logits/rejected": -0.908398449420929,
      "logps/chosen": -0.628710925579071,
      "logps/rejected": -1.849609375,
      "loss": 0.7049,
      "nll_loss": 0.640917956829071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06293945014476776,
      "rewards/margins": 0.12205199897289276,
      "rewards/rejected": -0.18488769233226776,
      "step": 9460
    },
    {
      "epoch": 0.6895547384133688,
      "grad_norm": 2.6302825024779404,
      "learning_rate": 8.220817318320836e-07,
      "log_odds_chosen": 1.7135741710662842,
      "log_odds_ratio": -0.35847169160842896,
      "logits/chosen": -1.016015648841858,
      "logits/rejected": -0.8828125,
      "logps/chosen": -0.6446288824081421,
      "logps/rejected": -1.8093750476837158,
      "loss": 0.6937,
      "nll_loss": 0.661425769329071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06456299126148224,
      "rewards/margins": 0.11649169772863388,
      "rewards/rejected": -0.18093261122703552,
      "step": 9470
    },
    {
      "epoch": 0.6902828849164452,
      "grad_norm": 3.356770503185186,
      "learning_rate": 8.216480300177611e-07,
      "log_odds_chosen": 1.697851538658142,
      "log_odds_ratio": -0.36469727754592896,
      "logits/chosen": -1.0144531726837158,
      "logits/rejected": -0.9214843511581421,
      "logps/chosen": -0.620800793170929,
      "logps/rejected": -1.786718726158142,
      "loss": 0.709,
      "nll_loss": 0.700976550579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06199951097369194,
      "rewards/margins": 0.11647949367761612,
      "rewards/rejected": -0.17861327528953552,
      "step": 9480
    },
    {
      "epoch": 0.6910110314195216,
      "grad_norm": 2.6712348205983973,
      "learning_rate": 8.212150138980857e-07,
      "log_odds_chosen": 1.75,
      "log_odds_ratio": -0.3794921934604645,
      "logits/chosen": -1.063085913658142,
      "logits/rejected": -0.9501953125,
      "logps/chosen": -0.5838867425918579,
      "logps/rejected": -1.8291015625,
      "loss": 0.6944,
      "nll_loss": 0.598388671875,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05838622897863388,
      "rewards/margins": 0.12451171875,
      "rewards/rejected": -0.18281249701976776,
      "step": 9490
    },
    {
      "epoch": 0.691739177922598,
      "grad_norm": 2.9765169817631247,
      "learning_rate": 8.207826816681233e-07,
      "log_odds_chosen": 1.6633789539337158,
      "log_odds_ratio": -0.3916259706020355,
      "logits/chosen": -1.0099608898162842,
      "logits/rejected": -0.9105468988418579,
      "logps/chosen": -0.6151367425918579,
      "logps/rejected": -1.752343773841858,
      "loss": 0.6952,
      "nll_loss": 0.6463867425918579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06145019456744194,
      "rewards/margins": 0.11375121772289276,
      "rewards/rejected": -0.17524413764476776,
      "step": 9500
    },
    {
      "epoch": 0.6924673244256745,
      "grad_norm": 2.9449190597971144,
      "learning_rate": 8.203510315295829e-07,
      "log_odds_chosen": 1.677343726158142,
      "log_odds_ratio": -0.38676756620407104,
      "logits/chosen": -0.940625011920929,
      "logits/rejected": -0.832812488079071,
      "logps/chosen": -0.6402343511581421,
      "logps/rejected": -1.87109375,
      "loss": 0.7145,
      "nll_loss": 0.747753918170929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06402587890625,
      "rewards/margins": 0.12319336086511612,
      "rewards/rejected": -0.18703612685203552,
      "step": 9510
    },
    {
      "epoch": 0.6931954709287509,
      "grad_norm": 2.4789089697476867,
      "learning_rate": 8.199200616907878e-07,
      "log_odds_chosen": 1.829687476158142,
      "log_odds_ratio": -0.33833009004592896,
      "logits/chosen": -0.978320300579071,
      "logits/rejected": -0.885937511920929,
      "logps/chosen": -0.58154296875,
      "logps/rejected": -1.829687476158142,
      "loss": 0.6834,
      "nll_loss": 0.651562511920929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05812988430261612,
      "rewards/margins": 0.12490234524011612,
      "rewards/rejected": -0.18293456733226776,
      "step": 9520
    },
    {
      "epoch": 0.6939236174318273,
      "grad_norm": 3.0246912137257773,
      "learning_rate": 8.194897703666421e-07,
      "log_odds_chosen": 1.9308593273162842,
      "log_odds_ratio": -0.351806640625,
      "logits/chosen": -0.964648425579071,
      "logits/rejected": -0.8941406011581421,
      "logps/chosen": -0.653613269329071,
      "logps/rejected": -2.029296875,
      "loss": 0.7184,
      "nll_loss": 0.709667980670929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06536865234375,
      "rewards/margins": 0.13748779892921448,
      "rewards/rejected": -0.20295409858226776,
      "step": 9530
    },
    {
      "epoch": 0.6946517639349037,
      "grad_norm": 3.117673309481288,
      "learning_rate": 8.190601557786015e-07,
      "log_odds_chosen": 1.430078148841858,
      "log_odds_ratio": -0.390625,
      "logits/chosen": -1.037109375,
      "logits/rejected": -0.908398449420929,
      "logps/chosen": -0.6205078363418579,
      "logps/rejected": -1.560546875,
      "loss": 0.7126,
      "nll_loss": 0.63134765625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06206054612994194,
      "rewards/margins": 0.09403076022863388,
      "rewards/rejected": -0.15607909858226776,
      "step": 9540
    },
    {
      "epoch": 0.6953799104379801,
      "grad_norm": 3.0223993937519316,
      "learning_rate": 8.186312161546413e-07,
      "log_odds_chosen": 1.736914038658142,
      "log_odds_ratio": -0.412841796875,
      "logits/chosen": -1.028710961341858,
      "logits/rejected": -0.9214843511581421,
      "logps/chosen": -0.6456054449081421,
      "logps/rejected": -1.8312499523162842,
      "loss": 0.6874,
      "nll_loss": 0.657519519329071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06456299126148224,
      "rewards/margins": 0.11858673393726349,
      "rewards/rejected": -0.18327637016773224,
      "step": 9550
    },
    {
      "epoch": 0.6961080569410565,
      "grad_norm": 2.5923581447814406,
      "learning_rate": 8.182029497292262e-07,
      "log_odds_chosen": 1.7048828601837158,
      "log_odds_ratio": -0.3844238221645355,
      "logits/chosen": -0.9722656011581421,
      "logits/rejected": -0.905468761920929,
      "logps/chosen": -0.6597656011581421,
      "logps/rejected": -1.84765625,
      "loss": 0.7023,
      "nll_loss": 0.672558605670929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06590576469898224,
      "rewards/margins": 0.1187744140625,
      "rewards/rejected": -0.18466797471046448,
      "step": 9560
    },
    {
      "epoch": 0.696836203444133,
      "grad_norm": 3.988597607226991,
      "learning_rate": 8.177753547432792e-07,
      "log_odds_chosen": 1.83203125,
      "log_odds_ratio": -0.34125977754592896,
      "logits/chosen": -0.9996093511581421,
      "logits/rejected": -0.919140636920929,
      "logps/chosen": -0.610156238079071,
      "logps/rejected": -1.9347655773162842,
      "loss": 0.7044,
      "nll_loss": 0.6761718988418579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06092529371380806,
      "rewards/margins": 0.13249512016773224,
      "rewards/rejected": -0.19340820610523224,
      "step": 9570
    },
    {
      "epoch": 0.6975643499472094,
      "grad_norm": 2.2818272516646725,
      "learning_rate": 8.173484294441524e-07,
      "log_odds_chosen": 1.5578124523162842,
      "log_odds_ratio": -0.3863281309604645,
      "logits/chosen": -1.0193359851837158,
      "logits/rejected": -0.921679675579071,
      "logps/chosen": -0.633593738079071,
      "logps/rejected": -1.691015601158142,
      "loss": 0.6947,
      "nll_loss": 0.70947265625,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06336669623851776,
      "rewards/margins": 0.10589905083179474,
      "rewards/rejected": -0.169189453125,
      "step": 9580
    },
    {
      "epoch": 0.6982924964502858,
      "grad_norm": 3.08059521939612,
      "learning_rate": 8.169221720855952e-07,
      "log_odds_chosen": 1.700781226158142,
      "log_odds_ratio": -0.3994384706020355,
      "logits/chosen": -1.015039086341858,
      "logits/rejected": -0.894726574420929,
      "logps/chosen": -0.631640613079071,
      "logps/rejected": -1.8416016101837158,
      "loss": 0.6849,
      "nll_loss": 0.646777331829071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06313476711511612,
      "rewards/margins": 0.12098999321460724,
      "rewards/rejected": -0.18415527045726776,
      "step": 9590
    },
    {
      "epoch": 0.6990206429533622,
      "grad_norm": 2.977424478391465,
      "learning_rate": 8.164965809277261e-07,
      "log_odds_chosen": 1.974511742591858,
      "log_odds_ratio": -0.3118652403354645,
      "logits/chosen": -0.989453136920929,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -0.64501953125,
      "logps/rejected": -2.043750047683716,
      "loss": 0.7102,
      "nll_loss": 0.6968749761581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06455077975988388,
      "rewards/margins": 0.13973388075828552,
      "rewards/rejected": -0.20429687201976776,
      "step": 9600
    },
    {
      "epoch": 0.6997487894564386,
      "grad_norm": 3.3630886745177877,
      "learning_rate": 8.160716542370011e-07,
      "log_odds_chosen": 1.9059569835662842,
      "log_odds_ratio": -0.3499511778354645,
      "logits/chosen": -1.0373046398162842,
      "logits/rejected": -0.9247070550918579,
      "logps/chosen": -0.6827148199081421,
      "logps/rejected": -2.080078125,
      "loss": 0.6885,
      "nll_loss": 0.643261730670929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06827392429113388,
      "rewards/margins": 0.13961181044578552,
      "rewards/rejected": -0.20795898139476776,
      "step": 9610
    },
    {
      "epoch": 0.7004769359595151,
      "grad_norm": 2.9957874328587235,
      "learning_rate": 8.156473902861856e-07,
      "log_odds_chosen": 1.702539086341858,
      "log_odds_ratio": -0.3193115293979645,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.9292968511581421,
      "logps/chosen": -0.622753918170929,
      "logps/rejected": -1.744140625,
      "loss": 0.7138,
      "nll_loss": 0.640625,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.06224365159869194,
      "rewards/margins": 0.11209716647863388,
      "rewards/rejected": -0.17436523735523224,
      "step": 9620
    },
    {
      "epoch": 0.7012050824625915,
      "grad_norm": 3.3746051653331697,
      "learning_rate": 8.152237873543241e-07,
      "log_odds_chosen": 1.5095703601837158,
      "log_odds_ratio": -0.376220703125,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.8734375238418579,
      "logps/chosen": -0.6162109375,
      "logps/rejected": -1.6181640625,
      "loss": 0.7081,
      "nll_loss": 0.680859386920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06163330003619194,
      "rewards/margins": 0.10003662109375,
      "rewards/rejected": -0.16164550185203552,
      "step": 9630
    },
    {
      "epoch": 0.7019332289656679,
      "grad_norm": 2.795115628629495,
      "learning_rate": 8.148008437267104e-07,
      "log_odds_chosen": 1.8486328125,
      "log_odds_ratio": -0.34379881620407104,
      "logits/chosen": -1.0294921398162842,
      "logits/rejected": -0.912109375,
      "logps/chosen": -0.6239258050918579,
      "logps/rejected": -1.941992163658142,
      "loss": 0.6935,
      "nll_loss": 0.666699230670929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06236572191119194,
      "rewards/margins": 0.13176269829273224,
      "rewards/rejected": -0.19399413466453552,
      "step": 9640
    },
    {
      "epoch": 0.7026613754687443,
      "grad_norm": 3.22044812190199,
      "learning_rate": 8.143785576948602e-07,
      "log_odds_chosen": 1.5945312976837158,
      "log_odds_ratio": -0.3644042909145355,
      "logits/chosen": -0.9789062738418579,
      "logits/rejected": -0.868945300579071,
      "logps/chosen": -0.62353515625,
      "logps/rejected": -1.703710913658142,
      "loss": 0.6901,
      "nll_loss": 0.6905273199081421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06239013746380806,
      "rewards/margins": 0.10805664211511612,
      "rewards/rejected": -0.17041015625,
      "step": 9650
    },
    {
      "epoch": 0.7033895219718207,
      "grad_norm": 3.1204975856335486,
      "learning_rate": 8.139569275564796e-07,
      "log_odds_chosen": 1.767578125,
      "log_odds_ratio": -0.32905274629592896,
      "logits/chosen": -1.0119140148162842,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.6249023675918579,
      "logps/rejected": -1.869140625,
      "loss": 0.6929,
      "nll_loss": 0.6380859613418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06246338039636612,
      "rewards/margins": 0.12456054985523224,
      "rewards/rejected": -0.18703612685203552,
      "step": 9660
    },
    {
      "epoch": 0.7041176684748971,
      "grad_norm": 3.220389290092767,
      "learning_rate": 8.135359516154388e-07,
      "log_odds_chosen": 1.519140601158142,
      "log_odds_ratio": -0.37041014432907104,
      "logits/chosen": -0.9515625238418579,
      "logits/rejected": -0.847460925579071,
      "logps/chosen": -0.6380859613418579,
      "logps/rejected": -1.644140601158142,
      "loss": 0.7109,
      "nll_loss": 0.743945300579071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06383056938648224,
      "rewards/margins": 0.10063476860523224,
      "rewards/rejected": -0.16435547173023224,
      "step": 9670
    },
    {
      "epoch": 0.7048458149779736,
      "grad_norm": 3.1106287863117674,
      "learning_rate": 8.131156281817418e-07,
      "log_odds_chosen": 1.5291016101837158,
      "log_odds_ratio": -0.36552733182907104,
      "logits/chosen": -0.945117175579071,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -0.663867175579071,
      "logps/rejected": -1.736914038658142,
      "loss": 0.7225,
      "nll_loss": 0.735546886920929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06639404594898224,
      "rewards/margins": 0.10743407905101776,
      "rewards/rejected": -0.17385253310203552,
      "step": 9680
    },
    {
      "epoch": 0.70557396148105,
      "grad_norm": 2.836306893317422,
      "learning_rate": 8.126959555714979e-07,
      "log_odds_chosen": 1.5950195789337158,
      "log_odds_ratio": -0.38081055879592896,
      "logits/chosen": -1.0236327648162842,
      "logits/rejected": -0.899609386920929,
      "logps/chosen": -0.651074230670929,
      "logps/rejected": -1.830078125,
      "loss": 0.7083,
      "nll_loss": 0.6421874761581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06503906100988388,
      "rewards/margins": 0.11802367866039276,
      "rewards/rejected": -0.18310546875,
      "step": 9690
    },
    {
      "epoch": 0.7063021079841264,
      "grad_norm": 2.7123613312442605,
      "learning_rate": 8.122769321068952e-07,
      "log_odds_chosen": 1.8210937976837158,
      "log_odds_ratio": -0.32416993379592896,
      "logits/chosen": -1.028906226158142,
      "logits/rejected": -0.8896484375,
      "logps/chosen": -0.606249988079071,
      "logps/rejected": -1.875,
      "loss": 0.6951,
      "nll_loss": 0.594042956829071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06065673753619194,
      "rewards/margins": 0.12674561142921448,
      "rewards/rejected": -0.18735352158546448,
      "step": 9700
    },
    {
      "epoch": 0.7070302544872028,
      "grad_norm": 2.7798662810179318,
      "learning_rate": 8.118585561161698e-07,
      "log_odds_chosen": 1.5539062023162842,
      "log_odds_ratio": -0.37109375,
      "logits/chosen": -1.0185546875,
      "logits/rejected": -0.8763672113418579,
      "logps/chosen": -0.642382800579071,
      "logps/rejected": -1.7244141101837158,
      "loss": 0.7104,
      "nll_loss": 0.64453125,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06424560397863388,
      "rewards/margins": 0.108062744140625,
      "rewards/rejected": -0.17236328125,
      "step": 9710
    },
    {
      "epoch": 0.7077584009902792,
      "grad_norm": 3.2816461749331647,
      "learning_rate": 8.114408259335793e-07,
      "log_odds_chosen": 1.536523461341858,
      "log_odds_ratio": -0.38715821504592896,
      "logits/chosen": -1.051171898841858,
      "logits/rejected": -0.9150390625,
      "logps/chosen": -0.619824230670929,
      "logps/rejected": -1.6316406726837158,
      "loss": 0.707,
      "nll_loss": 0.6680663824081421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06203613430261612,
      "rewards/margins": 0.10111694037914276,
      "rewards/rejected": -0.16311034560203552,
      "step": 9720
    },
    {
      "epoch": 0.7084865474933557,
      "grad_norm": 2.9479286948440206,
      "learning_rate": 8.110237398993754e-07,
      "log_odds_chosen": 1.746484398841858,
      "log_odds_ratio": -0.3622070252895355,
      "logits/chosen": -1.0341796875,
      "logits/rejected": -0.9306640625,
      "logps/chosen": -0.61376953125,
      "logps/rejected": -1.8298828601837158,
      "loss": 0.6803,
      "nll_loss": 0.670117199420929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.0614013671875,
      "rewards/margins": 0.12151489406824112,
      "rewards/rejected": -0.18291015923023224,
      "step": 9730
    },
    {
      "epoch": 0.7092146939964321,
      "grad_norm": 2.807038426690002,
      "learning_rate": 8.106072963597751e-07,
      "log_odds_chosen": 1.482080101966858,
      "log_odds_ratio": -0.3870849609375,
      "logits/chosen": -1.0007812976837158,
      "logits/rejected": -0.8589843511581421,
      "logps/chosen": -0.6797851324081421,
      "logps/rejected": -1.7292969226837158,
      "loss": 0.7319,
      "nll_loss": 0.695605456829071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06794433295726776,
      "rewards/margins": 0.10493774712085724,
      "rewards/rejected": -0.17287597060203552,
      "step": 9740
    },
    {
      "epoch": 0.7099428404995085,
      "grad_norm": 3.0851099928763865,
      "learning_rate": 8.101914936669332e-07,
      "log_odds_chosen": 1.6023437976837158,
      "log_odds_ratio": -0.3664794862270355,
      "logits/chosen": -1.060546875,
      "logits/rejected": -0.9029296636581421,
      "logps/chosen": -0.6551758050918579,
      "logps/rejected": -1.778906226158142,
      "loss": 0.6937,
      "nll_loss": 0.6651366949081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06553955376148224,
      "rewards/margins": 0.112457275390625,
      "rewards/rejected": -0.17788085341453552,
      "step": 9750
    },
    {
      "epoch": 0.7106709870025849,
      "grad_norm": 3.432714090532867,
      "learning_rate": 8.09776330178916e-07,
      "log_odds_chosen": 1.567285180091858,
      "log_odds_ratio": -0.3694091737270355,
      "logits/chosen": -0.9742187261581421,
      "logits/rejected": -0.8740234375,
      "logps/chosen": -0.662304699420929,
      "logps/rejected": -1.7726562023162842,
      "loss": 0.6838,
      "nll_loss": 0.678515613079071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06619872897863388,
      "rewards/margins": 0.11098632961511612,
      "rewards/rejected": -0.17727050185203552,
      "step": 9760
    },
    {
      "epoch": 0.7113991335056613,
      "grad_norm": 3.6178945942043086,
      "learning_rate": 8.093618042596727e-07,
      "log_odds_chosen": 1.846923828125,
      "log_odds_ratio": -0.35954588651657104,
      "logits/chosen": -1.0041015148162842,
      "logits/rejected": -0.8798828125,
      "logps/chosen": -0.620312511920929,
      "logps/rejected": -1.9484374523162842,
      "loss": 0.6966,
      "nll_loss": 0.6572265625,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06202392652630806,
      "rewards/margins": 0.13286133110523224,
      "rewards/rejected": -0.19482421875,
      "step": 9770
    },
    {
      "epoch": 0.7121272800087377,
      "grad_norm": 2.818566389856036,
      "learning_rate": 8.089479142790095e-07,
      "log_odds_chosen": 1.9718749523162842,
      "log_odds_ratio": -0.30314940214157104,
      "logits/chosen": -0.9794921875,
      "logits/rejected": -0.871289074420929,
      "logps/chosen": -0.5912109613418579,
      "logps/rejected": -1.952734351158142,
      "loss": 0.7198,
      "nll_loss": 0.7081054449081421,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.05911865085363388,
      "rewards/margins": 0.13630370795726776,
      "rewards/rejected": -0.19550780951976776,
      "step": 9780
    },
    {
      "epoch": 0.7128554265118142,
      "grad_norm": 3.8903842627669016,
      "learning_rate": 8.085346586125621e-07,
      "log_odds_chosen": 2.030956983566284,
      "log_odds_ratio": -0.31593018770217896,
      "logits/chosen": -1.0173828601837158,
      "logits/rejected": -0.8984375,
      "logps/chosen": -0.5767577886581421,
      "logps/rejected": -1.9832031726837158,
      "loss": 0.6948,
      "nll_loss": 0.6717773675918579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05767212063074112,
      "rewards/margins": 0.14060059189796448,
      "rewards/rejected": -0.19843749701976776,
      "step": 9790
    },
    {
      "epoch": 0.7135835730148906,
      "grad_norm": 5.058397662642896,
      "learning_rate": 8.081220356417685e-07,
      "log_odds_chosen": 1.6843750476837158,
      "log_odds_ratio": -0.34550780057907104,
      "logits/chosen": -1.0203125476837158,
      "logits/rejected": -0.8968750238418579,
      "logps/chosen": -0.606640636920929,
      "logps/rejected": -1.7863280773162842,
      "loss": 0.6892,
      "nll_loss": 0.7012695074081421,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06063232570886612,
      "rewards/margins": 0.11811523139476776,
      "rewards/rejected": -0.17873534560203552,
      "step": 9800
    },
    {
      "epoch": 0.714311719517967,
      "grad_norm": 2.7582219767243266,
      "learning_rate": 8.077100437538435e-07,
      "log_odds_chosen": 1.66796875,
      "log_odds_ratio": -0.3322509825229645,
      "logits/chosen": -1.066992163658142,
      "logits/rejected": -0.932421863079071,
      "logps/chosen": -0.5873047113418579,
      "logps/rejected": -1.685937523841858,
      "loss": 0.694,
      "nll_loss": 0.621777355670929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05878906324505806,
      "rewards/margins": 0.10987548530101776,
      "rewards/rejected": -0.16855469346046448,
      "step": 9810
    },
    {
      "epoch": 0.7150398660210434,
      "grad_norm": 2.4123695717738274,
      "learning_rate": 8.072986813417512e-07,
      "log_odds_chosen": 1.8017578125,
      "log_odds_ratio": -0.31977540254592896,
      "logits/chosen": -1.025390625,
      "logits/rejected": -0.884960949420929,
      "logps/chosen": -0.6241210699081421,
      "logps/rejected": -1.8369140625,
      "loss": 0.6975,
      "nll_loss": 0.6825195550918579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06247558444738388,
      "rewards/margins": 0.12139892578125,
      "rewards/rejected": -0.18376465141773224,
      "step": 9820
    },
    {
      "epoch": 0.7157680125241198,
      "grad_norm": 3.3164445653046775,
      "learning_rate": 8.068879468041791e-07,
      "log_odds_chosen": 1.6455078125,
      "log_odds_ratio": -0.3775634765625,
      "logits/chosen": -1.065820336341858,
      "logits/rejected": -0.953906238079071,
      "logps/chosen": -0.646484375,
      "logps/rejected": -1.818750023841858,
      "loss": 0.6678,
      "nll_loss": 0.653027355670929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06467285007238388,
      "rewards/margins": 0.11728515475988388,
      "rewards/rejected": -0.18181152641773224,
      "step": 9830
    },
    {
      "epoch": 0.7164961590271963,
      "grad_norm": 2.854329016851393,
      "learning_rate": 8.064778385455118e-07,
      "log_odds_chosen": 1.73382568359375,
      "log_odds_ratio": -0.35429686307907104,
      "logits/chosen": -0.982226550579071,
      "logits/rejected": -0.8705078363418579,
      "logps/chosen": -0.62353515625,
      "logps/rejected": -1.806249976158142,
      "loss": 0.7064,
      "nll_loss": 0.7060546875,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06231689453125,
      "rewards/margins": 0.11821594089269638,
      "rewards/rejected": -0.18056640028953552,
      "step": 9840
    },
    {
      "epoch": 0.7172243055302727,
      "grad_norm": 2.663540886954136,
      "learning_rate": 8.060683549758054e-07,
      "log_odds_chosen": 1.993554711341858,
      "log_odds_ratio": -0.3094238340854645,
      "logits/chosen": -1.041015625,
      "logits/rejected": -0.907031238079071,
      "logps/chosen": -0.5947265625,
      "logps/rejected": -1.984960913658142,
      "loss": 0.6845,
      "nll_loss": 0.622851550579071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05950927734375,
      "rewards/margins": 0.13895264267921448,
      "rewards/rejected": -0.19843749701976776,
      "step": 9850
    },
    {
      "epoch": 0.7179524520333491,
      "grad_norm": 3.382047056796238,
      "learning_rate": 8.056594945107608e-07,
      "log_odds_chosen": 1.921289086341858,
      "log_odds_ratio": -0.32783204317092896,
      "logits/chosen": -1.095117211341858,
      "logits/rejected": -0.947070300579071,
      "logps/chosen": -0.630078136920929,
      "logps/rejected": -1.99609375,
      "loss": 0.6958,
      "nll_loss": 0.6102539300918579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06304931640625,
      "rewards/margins": 0.13663330674171448,
      "rewards/rejected": -0.199462890625,
      "step": 9860
    },
    {
      "epoch": 0.7186805985364255,
      "grad_norm": 2.8615041697038377,
      "learning_rate": 8.052512555716987e-07,
      "log_odds_chosen": 1.6554687023162842,
      "log_odds_ratio": -0.3623046875,
      "logits/chosen": -1.045312523841858,
      "logits/rejected": -0.9205077886581421,
      "logps/chosen": -0.627636730670929,
      "logps/rejected": -1.7814452648162842,
      "loss": 0.707,
      "nll_loss": 0.650097668170929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06273193657398224,
      "rewards/margins": 0.115234375,
      "rewards/rejected": -0.17802734673023224,
      "step": 9870
    },
    {
      "epoch": 0.7194087450395019,
      "grad_norm": 3.2869778050983873,
      "learning_rate": 8.048436365855337e-07,
      "log_odds_chosen": 1.6599609851837158,
      "log_odds_ratio": -0.348388671875,
      "logits/chosen": -1.017968773841858,
      "logits/rejected": -0.890429675579071,
      "logps/chosen": -0.5980468988418579,
      "logps/rejected": -1.761328101158142,
      "loss": 0.7014,
      "nll_loss": 0.61474609375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05982666090130806,
      "rewards/margins": 0.11635742336511612,
      "rewards/rejected": -0.17624512314796448,
      "step": 9880
    },
    {
      "epoch": 0.7201368915425783,
      "grad_norm": 3.0100105039647405,
      "learning_rate": 8.044366359847486e-07,
      "log_odds_chosen": 1.978124976158142,
      "log_odds_ratio": -0.31718748807907104,
      "logits/chosen": -0.9839843511581421,
      "logits/rejected": -0.8636718988418579,
      "logps/chosen": -0.6357421875,
      "logps/rejected": -2.024218797683716,
      "loss": 0.729,
      "nll_loss": 0.7342773675918579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06357421725988388,
      "rewards/margins": 0.13876953721046448,
      "rewards/rejected": -0.20229491591453552,
      "step": 9890
    },
    {
      "epoch": 0.7208650380456548,
      "grad_norm": 3.15320760709319,
      "learning_rate": 8.040302522073696e-07,
      "log_odds_chosen": 1.6554687023162842,
      "log_odds_ratio": -0.39604490995407104,
      "logits/chosen": -1.028906226158142,
      "logits/rejected": -0.8890625238418579,
      "logps/chosen": -0.706835925579071,
      "logps/rejected": -1.880468726158142,
      "loss": 0.6889,
      "nll_loss": 0.6458984613418579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07070312649011612,
      "rewards/margins": 0.11751709133386612,
      "rewards/rejected": -0.18818359076976776,
      "step": 9900
    },
    {
      "epoch": 0.7215931845487312,
      "grad_norm": 3.0913105039684345,
      "learning_rate": 8.036244836969407e-07,
      "log_odds_chosen": 1.596093773841858,
      "log_odds_ratio": -0.39750975370407104,
      "logits/chosen": -0.997851550579071,
      "logits/rejected": -0.9033203125,
      "logps/chosen": -0.664746105670929,
      "logps/rejected": -1.731835961341858,
      "loss": 0.6823,
      "nll_loss": 0.6693359613418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06647948920726776,
      "rewards/margins": 0.10659179836511612,
      "rewards/rejected": -0.17312011122703552,
      "step": 9910
    },
    {
      "epoch": 0.7223213310518076,
      "grad_norm": 3.3098145143351476,
      "learning_rate": 8.032193289024989e-07,
      "log_odds_chosen": 1.848046898841858,
      "log_odds_ratio": -0.3561767637729645,
      "logits/chosen": -1.0320312976837158,
      "logits/rejected": -0.9156249761581421,
      "logps/chosen": -0.6138671636581421,
      "logps/rejected": -1.8953125476837158,
      "loss": 0.6948,
      "nll_loss": 0.652539074420929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06138915941119194,
      "rewards/margins": 0.12800292670726776,
      "rewards/rejected": -0.1893310546875,
      "step": 9920
    },
    {
      "epoch": 0.723049477554884,
      "grad_norm": 3.147623019242075,
      "learning_rate": 8.02814786278549e-07,
      "log_odds_chosen": 1.868749976158142,
      "log_odds_ratio": -0.34501951932907104,
      "logits/chosen": -1.048828125,
      "logits/rejected": -0.915820300579071,
      "logps/chosen": -0.6166015863418579,
      "logps/rejected": -1.9796874523162842,
      "loss": 0.6723,
      "nll_loss": 0.649609386920929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06169433519244194,
      "rewards/margins": 0.13633117079734802,
      "rewards/rejected": -0.19791260361671448,
      "step": 9930
    },
    {
      "epoch": 0.7237776240579604,
      "grad_norm": 3.643893080776182,
      "learning_rate": 8.024108542850394e-07,
      "log_odds_chosen": 1.8923828601837158,
      "log_odds_ratio": -0.34174805879592896,
      "logits/chosen": -1.0158202648162842,
      "logits/rejected": -0.923828125,
      "logps/chosen": -0.589160144329071,
      "logps/rejected": -1.892187476158142,
      "loss": 0.6752,
      "nll_loss": 0.6792968511581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05893554538488388,
      "rewards/margins": 0.13013915717601776,
      "rewards/rejected": -0.18911132216453552,
      "step": 9940
    },
    {
      "epoch": 0.7245057705610369,
      "grad_norm": 2.8411011058273408,
      "learning_rate": 8.020075313873367e-07,
      "log_odds_chosen": 1.943750023841858,
      "log_odds_ratio": -0.31513673067092896,
      "logits/chosen": -1.014062523841858,
      "logits/rejected": -0.9185546636581421,
      "logps/chosen": -0.6170898675918579,
      "logps/rejected": -1.962499976158142,
      "loss": 0.6767,
      "nll_loss": 0.6485351324081421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06180419772863388,
      "rewards/margins": 0.13472899794578552,
      "rewards/rejected": -0.19638672471046448,
      "step": 9950
    },
    {
      "epoch": 0.7252339170641133,
      "grad_norm": 3.730232886607255,
      "learning_rate": 8.016048160562023e-07,
      "log_odds_chosen": 2.009765625,
      "log_odds_ratio": -0.308837890625,
      "logits/chosen": -1.0109374523162842,
      "logits/rejected": -0.892773449420929,
      "logps/chosen": -0.6377929449081421,
      "logps/rejected": -2.1304688453674316,
      "loss": 0.6829,
      "nll_loss": 0.700390636920929,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06369628757238388,
      "rewards/margins": 0.14927978813648224,
      "rewards/rejected": -0.21311035752296448,
      "step": 9960
    },
    {
      "epoch": 0.7259620635671897,
      "grad_norm": 2.4113213213279057,
      "learning_rate": 8.012027067677667e-07,
      "log_odds_chosen": 1.8986327648162842,
      "log_odds_ratio": -0.37968748807907104,
      "logits/chosen": -0.9750000238418579,
      "logits/rejected": -0.8755859136581421,
      "logps/chosen": -0.6810547113418579,
      "logps/rejected": -2.049999952316284,
      "loss": 0.6783,
      "nll_loss": 0.64794921875,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06815185397863388,
      "rewards/margins": 0.13676758110523224,
      "rewards/rejected": -0.2049560546875,
      "step": 9970
    },
    {
      "epoch": 0.7266902100702661,
      "grad_norm": 3.6874383494989353,
      "learning_rate": 8.008012020035062e-07,
      "log_odds_chosen": 1.8330078125,
      "log_odds_ratio": -0.3518310487270355,
      "logits/chosen": -1.0701172351837158,
      "logits/rejected": -0.951171875,
      "logps/chosen": -0.646484375,
      "logps/rejected": -1.914453148841858,
      "loss": 0.673,
      "nll_loss": 0.628222644329071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06461181491613388,
      "rewards/margins": 0.126953125,
      "rewards/rejected": -0.19145508110523224,
      "step": 9980
    },
    {
      "epoch": 0.7274183565733425,
      "grad_norm": 2.9546987391162967,
      "learning_rate": 8.004003002502188e-07,
      "log_odds_chosen": 1.6650390625,
      "log_odds_ratio": -0.372314453125,
      "logits/chosen": -1.0056641101837158,
      "logits/rejected": -0.926953136920929,
      "logps/chosen": -0.660937488079071,
      "logps/rejected": -1.827734351158142,
      "loss": 0.7193,
      "nll_loss": 0.696582019329071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06613769382238388,
      "rewards/margins": 0.11649169772863388,
      "rewards/rejected": -0.18281249701976776,
      "step": 9990
    },
    {
      "epoch": 0.7281465030764189,
      "grad_norm": 3.233285273254317,
      "learning_rate": 8e-07,
      "log_odds_chosen": 1.5919921398162842,
      "log_odds_ratio": -0.4210449159145355,
      "logits/chosen": -1.029687523841858,
      "logits/rejected": -0.9068359136581421,
      "logps/chosen": -0.662304699420929,
      "logps/rejected": -1.790624976158142,
      "loss": 0.6869,
      "nll_loss": 0.6514648199081421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06622314453125,
      "rewards/margins": 0.11280517280101776,
      "rewards/rejected": -0.17905274033546448,
      "step": 10000
    },
    {
      "epoch": 0.7281465030764189,
      "eval_log_odds_chosen": 1.2992345094680786,
      "eval_log_odds_ratio": -0.46292415261268616,
      "eval_logits/chosen": -0.8166144490242004,
      "eval_logits/rejected": -0.7258062958717346,
      "eval_logps/chosen": -0.7621862888336182,
      "eval_logps/rejected": -1.7369604110717773,
      "eval_loss": 1.2104041576385498,
      "eval_nll_loss": 1.1603708267211914,
      "eval_rewards/accuracies": 0.7447678446769714,
      "eval_rewards/chosen": -0.07621386647224426,
      "eval_rewards/margins": 0.09745727479457855,
      "eval_rewards/rejected": -0.1736627072095871,
      "eval_runtime": 1369.6375,
      "eval_samples_per_second": 71.442,
      "eval_steps_per_second": 1.116,
      "step": 10000
    },
    {
      "epoch": 0.7288746495794954,
      "grad_norm": 3.8336332947145615,
      "learning_rate": 7.996002997502185e-07,
      "log_odds_chosen": 1.838476538658142,
      "log_odds_ratio": -0.33154296875,
      "logits/chosen": -1.0451171398162842,
      "logits/rejected": -0.921875,
      "logps/chosen": -0.6092773675918579,
      "logps/rejected": -1.8742187023162842,
      "loss": 0.6869,
      "nll_loss": 0.617968738079071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06096191331744194,
      "rewards/margins": 0.12642821669578552,
      "rewards/rejected": -0.18740233778953552,
      "step": 10010
    },
    {
      "epoch": 0.7296027960825718,
      "grad_norm": 3.1754027955359665,
      "learning_rate": 7.992011980034937e-07,
      "log_odds_chosen": 2.0546875,
      "log_odds_ratio": -0.28937989473342896,
      "logits/chosen": -1.05078125,
      "logits/rejected": -0.9339843988418579,
      "logps/chosen": -0.5772460699081421,
      "logps/rejected": -2.0355467796325684,
      "loss": 0.671,
      "nll_loss": 0.588671863079071,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.05775146558880806,
      "rewards/margins": 0.14582519233226776,
      "rewards/rejected": -0.203369140625,
      "step": 10020
    },
    {
      "epoch": 0.7303309425856482,
      "grad_norm": 3.235597594047446,
      "learning_rate": 7.98802693267671e-07,
      "log_odds_chosen": 1.823632836341858,
      "log_odds_ratio": -0.3519531190395355,
      "logits/chosen": -1.017968773841858,
      "logits/rejected": -0.8939453363418579,
      "logps/chosen": -0.629101574420929,
      "logps/rejected": -1.9249999523162842,
      "loss": 0.7025,
      "nll_loss": 0.655078113079071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06292724609375,
      "rewards/margins": 0.12943115830421448,
      "rewards/rejected": -0.19230957329273224,
      "step": 10030
    },
    {
      "epoch": 0.7310590890887246,
      "grad_norm": 2.8640129529845155,
      "learning_rate": 7.984047840557992e-07,
      "log_odds_chosen": 1.8826172351837158,
      "log_odds_ratio": -0.3016113340854645,
      "logits/chosen": -1.001562476158142,
      "logits/rejected": -0.872851550579071,
      "logps/chosen": -0.6053711175918579,
      "logps/rejected": -1.962499976158142,
      "loss": 0.684,
      "nll_loss": 0.651171863079071,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06058349460363388,
      "rewards/margins": 0.13571777939796448,
      "rewards/rejected": -0.19621582329273224,
      "step": 10040
    },
    {
      "epoch": 0.731787235591801,
      "grad_norm": 2.8459608974668344,
      "learning_rate": 7.980074688861063e-07,
      "log_odds_chosen": 1.9296875,
      "log_odds_ratio": -0.32548826932907104,
      "logits/chosen": -0.941210925579071,
      "logits/rejected": -0.8490234613418579,
      "logps/chosen": -0.606640636920929,
      "logps/rejected": -1.9617187976837158,
      "loss": 0.6797,
      "nll_loss": 0.7056640386581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.0606689453125,
      "rewards/margins": 0.13571777939796448,
      "rewards/rejected": -0.19633789360523224,
      "step": 10050
    },
    {
      "epoch": 0.7325153820948775,
      "grad_norm": 3.1647269680263443,
      "learning_rate": 7.976107462819775e-07,
      "log_odds_chosen": 1.573632836341858,
      "log_odds_ratio": -0.38041990995407104,
      "logits/chosen": -0.991406261920929,
      "logits/rejected": -0.7939453125,
      "logps/chosen": -0.6514648199081421,
      "logps/rejected": -1.7890625,
      "loss": 0.6918,
      "nll_loss": 0.6812499761581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06508789211511612,
      "rewards/margins": 0.11373291164636612,
      "rewards/rejected": -0.17888183891773224,
      "step": 10060
    },
    {
      "epoch": 0.7332435285979539,
      "grad_norm": 3.7262945005422012,
      "learning_rate": 7.97214614771931e-07,
      "log_odds_chosen": 1.905859351158142,
      "log_odds_ratio": -0.31401365995407104,
      "logits/chosen": -1.0615234375,
      "logits/rejected": -0.928515613079071,
      "logps/chosen": -0.6910156011581421,
      "logps/rejected": -2.0179686546325684,
      "loss": 0.6822,
      "nll_loss": 0.664746105670929,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06910400092601776,
      "rewards/margins": 0.13288573920726776,
      "rewards/rejected": -0.20195312798023224,
      "step": 10070
    },
    {
      "epoch": 0.7339716751010303,
      "grad_norm": 2.6841122706576614,
      "learning_rate": 7.968190728895957e-07,
      "log_odds_chosen": 1.8830077648162842,
      "log_odds_ratio": -0.32719725370407104,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.8388671875,
      "logps/chosen": -0.6202148199081421,
      "logps/rejected": -1.949609398841858,
      "loss": 0.6938,
      "nll_loss": 0.654296875,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06199951097369194,
      "rewards/margins": 0.13273926079273224,
      "rewards/rejected": -0.19482421875,
      "step": 10080
    },
    {
      "epoch": 0.7346998216041067,
      "grad_norm": 2.682741700068426,
      "learning_rate": 7.964241191736886e-07,
      "log_odds_chosen": 1.7917969226837158,
      "log_odds_ratio": -0.3451171815395355,
      "logits/chosen": -1.0439453125,
      "logits/rejected": -0.910937488079071,
      "logps/chosen": -0.62890625,
      "logps/rejected": -1.868749976158142,
      "loss": 0.6612,
      "nll_loss": 0.644726574420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06290283054113388,
      "rewards/margins": 0.12393798679113388,
      "rewards/rejected": -0.18684081733226776,
      "step": 10090
    },
    {
      "epoch": 0.7354279681071831,
      "grad_norm": 2.658729548995854,
      "learning_rate": 7.960297521679913e-07,
      "log_odds_chosen": 2.158203125,
      "log_odds_ratio": -0.303955078125,
      "logits/chosen": -1.0361328125,
      "logits/rejected": -0.9345703125,
      "logps/chosen": -0.6258789300918579,
      "logps/rejected": -2.2222657203674316,
      "loss": 0.6936,
      "nll_loss": 0.6700195074081421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06256103515625,
      "rewards/margins": 0.15974120795726776,
      "rewards/rejected": -0.22214356064796448,
      "step": 10100
    },
    {
      "epoch": 0.7361561146102595,
      "grad_norm": 3.1988104279382186,
      "learning_rate": 7.956359704213283e-07,
      "log_odds_chosen": 1.7498047351837158,
      "log_odds_ratio": -0.3436035215854645,
      "logits/chosen": -1.0304687023162842,
      "logits/rejected": -0.9359375238418579,
      "logps/chosen": -0.655566394329071,
      "logps/rejected": -1.875390648841858,
      "loss": 0.6777,
      "nll_loss": 0.684765636920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06552734225988388,
      "rewards/margins": 0.12211914360523224,
      "rewards/rejected": -0.18754883110523224,
      "step": 10110
    },
    {
      "epoch": 0.736884261113336,
      "grad_norm": 3.4194621545848065,
      "learning_rate": 7.95242772487544e-07,
      "log_odds_chosen": 2.0765624046325684,
      "log_odds_ratio": -0.3074707090854645,
      "logits/chosen": -1.0076172351837158,
      "logits/rejected": -0.8724609613418579,
      "logps/chosen": -0.62255859375,
      "logps/rejected": -2.1265625953674316,
      "loss": 0.6919,
      "nll_loss": 0.7056640386581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.062255859375,
      "rewards/margins": 0.150390625,
      "rewards/rejected": -0.21245117485523224,
      "step": 10120
    },
    {
      "epoch": 0.7376124076164124,
      "grad_norm": 2.875610915715048,
      "learning_rate": 7.94850156925481e-07,
      "log_odds_chosen": 1.817968726158142,
      "log_odds_ratio": -0.33393555879592896,
      "logits/chosen": -1.0041015148162842,
      "logits/rejected": -0.891796886920929,
      "logps/chosen": -0.575390636920929,
      "logps/rejected": -1.8308594226837158,
      "loss": 0.6735,
      "nll_loss": 0.640820324420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05755615234375,
      "rewards/margins": 0.12532958388328552,
      "rewards/rejected": -0.18303222954273224,
      "step": 10130
    },
    {
      "epoch": 0.7383405541194888,
      "grad_norm": 3.244456866089268,
      "learning_rate": 7.944581222989574e-07,
      "log_odds_chosen": 1.534765601158142,
      "log_odds_ratio": -0.4009033143520355,
      "logits/chosen": -0.997265636920929,
      "logits/rejected": -0.892773449420929,
      "logps/chosen": -0.620410144329071,
      "logps/rejected": -1.6638672351837158,
      "loss": 0.6716,
      "nll_loss": 0.6473633050918579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06207275390625,
      "rewards/margins": 0.10433349758386612,
      "rewards/rejected": -0.16635742783546448,
      "step": 10140
    },
    {
      "epoch": 0.7390687006225652,
      "grad_norm": 3.1341519597791674,
      "learning_rate": 7.940666671767441e-07,
      "log_odds_chosen": 2.0888671875,
      "log_odds_ratio": -0.2948242127895355,
      "logits/chosen": -1.090429663658142,
      "logits/rejected": -0.947265625,
      "logps/chosen": -0.6163085699081421,
      "logps/rejected": -2.1195311546325684,
      "loss": 0.6901,
      "nll_loss": 0.648730456829071,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06158447265625,
      "rewards/margins": 0.15034179389476776,
      "rewards/rejected": -0.2119140625,
      "step": 10150
    },
    {
      "epoch": 0.7397968471256416,
      "grad_norm": 3.150681355473158,
      "learning_rate": 7.936757901325451e-07,
      "log_odds_chosen": 1.736230492591858,
      "log_odds_ratio": -0.36572265625,
      "logits/chosen": -1.0556640625,
      "logits/rejected": -0.9205077886581421,
      "logps/chosen": -0.6382812261581421,
      "logps/rejected": -1.861328125,
      "loss": 0.666,
      "nll_loss": 0.633007824420929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06391601264476776,
      "rewards/margins": 0.12205810844898224,
      "rewards/rejected": -0.18603515625,
      "step": 10160
    },
    {
      "epoch": 0.7405249936287182,
      "grad_norm": 2.640249850859402,
      "learning_rate": 7.932854897449727e-07,
      "log_odds_chosen": 1.9572265148162842,
      "log_odds_ratio": -0.2979980409145355,
      "logits/chosen": -1.0244140625,
      "logits/rejected": -0.8984375,
      "logps/chosen": -0.5772460699081421,
      "logps/rejected": -1.896093726158142,
      "loss": 0.6719,
      "nll_loss": 0.6103515625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05770263820886612,
      "rewards/margins": 0.13186034560203552,
      "rewards/rejected": -0.189697265625,
      "step": 10170
    },
    {
      "epoch": 0.7412531401317946,
      "grad_norm": 3.7184220704296957,
      "learning_rate": 7.928957645975286e-07,
      "log_odds_chosen": 1.971093773841858,
      "log_odds_ratio": -0.30439454317092896,
      "logits/chosen": -1.016992211341858,
      "logits/rejected": -0.900390625,
      "logps/chosen": -0.610058605670929,
      "logps/rejected": -2.020312547683716,
      "loss": 0.6974,
      "nll_loss": 0.6937500238418579,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06093750149011612,
      "rewards/margins": 0.14107665419578552,
      "rewards/rejected": -0.20209960639476776,
      "step": 10180
    },
    {
      "epoch": 0.741981286634871,
      "grad_norm": 3.590209612142339,
      "learning_rate": 7.925066132785799e-07,
      "log_odds_chosen": 1.894921898841858,
      "log_odds_ratio": -0.334228515625,
      "logits/chosen": -0.999218761920929,
      "logits/rejected": -0.8599609136581421,
      "logps/chosen": -0.623242199420929,
      "logps/rejected": -1.979882836341858,
      "loss": 0.6781,
      "nll_loss": 0.699511706829071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06235351413488388,
      "rewards/margins": 0.13563232123851776,
      "rewards/rejected": -0.19782714545726776,
      "step": 10190
    },
    {
      "epoch": 0.7427094331379474,
      "grad_norm": 2.860217843211453,
      "learning_rate": 7.921180343813395e-07,
      "log_odds_chosen": 1.7765624523162842,
      "log_odds_ratio": -0.33442384004592896,
      "logits/chosen": -1.0048828125,
      "logits/rejected": -0.9029296636581421,
      "logps/chosen": -0.615234375,
      "logps/rejected": -1.822656273841858,
      "loss": 0.6858,
      "nll_loss": 0.634765625,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06149902194738388,
      "rewards/margins": 0.12088622897863388,
      "rewards/rejected": -0.18242187798023224,
      "step": 10200
    },
    {
      "epoch": 0.7434375796410237,
      "grad_norm": 3.6440484440436838,
      "learning_rate": 7.917300265038436e-07,
      "log_odds_chosen": 1.6484375,
      "log_odds_ratio": -0.38127440214157104,
      "logits/chosen": -1.0447266101837158,
      "logits/rejected": -0.940234363079071,
      "logps/chosen": -0.642871081829071,
      "logps/rejected": -1.745703101158142,
      "loss": 0.7101,
      "nll_loss": 0.7328125238418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06424560397863388,
      "rewards/margins": 0.11011199653148651,
      "rewards/rejected": -0.17441406846046448,
      "step": 10210
    },
    {
      "epoch": 0.7441657261441001,
      "grad_norm": 3.157320814473658,
      "learning_rate": 7.913425882489307e-07,
      "log_odds_chosen": 1.5232422351837158,
      "log_odds_ratio": -0.422119140625,
      "logits/chosen": -1.0558593273162842,
      "logits/rejected": -0.944531261920929,
      "logps/chosen": -0.6455078125,
      "logps/rejected": -1.72265625,
      "loss": 0.6993,
      "nll_loss": 0.701855480670929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06456299126148224,
      "rewards/margins": 0.10761718451976776,
      "rewards/rejected": -0.17219237983226776,
      "step": 10220
    },
    {
      "epoch": 0.7448938726471767,
      "grad_norm": 3.0388301365305863,
      "learning_rate": 7.909557182242211e-07,
      "log_odds_chosen": 1.4423828125,
      "log_odds_ratio": -0.4169555604457855,
      "logits/chosen": -1.023828148841858,
      "logits/rejected": -0.9185546636581421,
      "logps/chosen": -0.6483398675918579,
      "logps/rejected": -1.6599609851837158,
      "loss": 0.6637,
      "nll_loss": 0.6357421875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06484375149011612,
      "rewards/margins": 0.10112304985523224,
      "rewards/rejected": -0.16591796278953552,
      "step": 10230
    },
    {
      "epoch": 0.7456220191502531,
      "grad_norm": 2.7628432070565907,
      "learning_rate": 7.905694150420947e-07,
      "log_odds_chosen": 1.709814429283142,
      "log_odds_ratio": -0.3671875,
      "logits/chosen": -0.9994140863418579,
      "logits/rejected": -0.88671875,
      "logps/chosen": -0.6249023675918579,
      "logps/rejected": -1.8097655773162842,
      "loss": 0.6847,
      "nll_loss": 0.669238269329071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06247558444738388,
      "rewards/margins": 0.11849365383386612,
      "rewards/rejected": -0.18098144233226776,
      "step": 10240
    },
    {
      "epoch": 0.7463501656533295,
      "grad_norm": 3.2264028995653353,
      "learning_rate": 7.901836773196717e-07,
      "log_odds_chosen": 1.741796851158142,
      "log_odds_ratio": -0.33085936307907104,
      "logits/chosen": -1.0693359375,
      "logits/rejected": -0.9544922113418579,
      "logps/chosen": -0.631152331829071,
      "logps/rejected": -1.804296851158142,
      "loss": 0.6919,
      "nll_loss": 0.6309570074081421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06309814751148224,
      "rewards/margins": 0.11746826022863388,
      "rewards/rejected": -0.18063965439796448,
      "step": 10250
    },
    {
      "epoch": 0.7470783121564059,
      "grad_norm": 3.0308190694693327,
      "learning_rate": 7.897985036787898e-07,
      "log_odds_chosen": 1.900390625,
      "log_odds_ratio": -0.3011718690395355,
      "logits/chosen": -1.0128905773162842,
      "logits/rejected": -0.875195324420929,
      "logps/chosen": -0.5814453363418579,
      "logps/rejected": -1.844335913658142,
      "loss": 0.6828,
      "nll_loss": 0.666699230670929,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.05821533128619194,
      "rewards/margins": 0.12618407607078552,
      "rewards/rejected": -0.18437500298023224,
      "step": 10260
    },
    {
      "epoch": 0.7478064586594823,
      "grad_norm": 4.346536074544458,
      "learning_rate": 7.894138927459853e-07,
      "log_odds_chosen": 1.7158203125,
      "log_odds_ratio": -0.41120606660842896,
      "logits/chosen": -0.9654296636581421,
      "logits/rejected": -0.8814452886581421,
      "logps/chosen": -0.6151367425918579,
      "logps/rejected": -1.814843773841858,
      "loss": 0.6921,
      "nll_loss": 0.668261706829071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06149902194738388,
      "rewards/margins": 0.11979980766773224,
      "rewards/rejected": -0.181396484375,
      "step": 10270
    },
    {
      "epoch": 0.7485346051625587,
      "grad_norm": 3.5740426986865796,
      "learning_rate": 7.890298431524716e-07,
      "log_odds_chosen": 1.67529296875,
      "log_odds_ratio": -0.380615234375,
      "logits/chosen": -0.984570324420929,
      "logits/rejected": -0.889843761920929,
      "logps/chosen": -0.630566418170929,
      "logps/rejected": -1.7400391101837158,
      "loss": 0.7021,
      "nll_loss": 0.655468761920929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.0631103515625,
      "rewards/margins": 0.11115112155675888,
      "rewards/rejected": -0.1741943359375,
      "step": 10280
    },
    {
      "epoch": 0.7492627516656352,
      "grad_norm": 3.7487769558793684,
      "learning_rate": 7.88646353534119e-07,
      "log_odds_chosen": 1.701757788658142,
      "log_odds_ratio": -0.36225587129592896,
      "logits/chosen": -1.020898461341858,
      "logits/rejected": -0.896289050579071,
      "logps/chosen": -0.630175769329071,
      "logps/rejected": -1.830078125,
      "loss": 0.6733,
      "nll_loss": 0.618359386920929,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06303711235523224,
      "rewards/margins": 0.11989136040210724,
      "rewards/rejected": -0.1829833984375,
      "step": 10290
    },
    {
      "epoch": 0.7499908981687116,
      "grad_norm": 3.739333427785617,
      "learning_rate": 7.882634225314345e-07,
      "log_odds_chosen": 1.5587890148162842,
      "log_odds_ratio": -0.375732421875,
      "logits/chosen": -1.0197265148162842,
      "logits/rejected": -0.897265613079071,
      "logps/chosen": -0.625,
      "logps/rejected": -1.681249976158142,
      "loss": 0.6763,
      "nll_loss": 0.6915038824081421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06252441555261612,
      "rewards/margins": 0.10555420070886612,
      "rewards/rejected": -0.1680908203125,
      "step": 10300
    },
    {
      "epoch": 0.750719044671788,
      "grad_norm": 2.735001863887216,
      "learning_rate": 7.87881048789541e-07,
      "log_odds_chosen": 1.331445336341858,
      "log_odds_ratio": -0.426025390625,
      "logits/chosen": -1.020898461341858,
      "logits/rejected": -0.951367199420929,
      "logps/chosen": -0.6763671636581421,
      "logps/rejected": -1.568359375,
      "loss": 0.6922,
      "nll_loss": 0.692089855670929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06760253757238388,
      "rewards/margins": 0.08934326469898224,
      "rewards/rejected": -0.15695801377296448,
      "step": 10310
    },
    {
      "epoch": 0.7514471911748644,
      "grad_norm": 3.133864749724109,
      "learning_rate": 7.874992309581578e-07,
      "log_odds_chosen": 1.8466796875,
      "log_odds_ratio": -0.314208984375,
      "logits/chosen": -0.9996093511581421,
      "logits/rejected": -0.8705078363418579,
      "logps/chosen": -0.58154296875,
      "logps/rejected": -1.8546874523162842,
      "loss": 0.6839,
      "nll_loss": 0.6673828363418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05817871168255806,
      "rewards/margins": 0.12728270888328552,
      "rewards/rejected": -0.18549804389476776,
      "step": 10320
    },
    {
      "epoch": 0.7521753376779408,
      "grad_norm": 2.6677243158327872,
      "learning_rate": 7.871179676915801e-07,
      "log_odds_chosen": 1.665429711341858,
      "log_odds_ratio": -0.41572266817092896,
      "logits/chosen": -1.0402343273162842,
      "logits/rejected": -0.9466797113418579,
      "logps/chosen": -0.649609386920929,
      "logps/rejected": -1.82421875,
      "loss": 0.6641,
      "nll_loss": 0.6534179449081421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06486816704273224,
      "rewards/margins": 0.11745605617761612,
      "rewards/rejected": -0.1822509765625,
      "step": 10330
    },
    {
      "epoch": 0.7529034841810173,
      "grad_norm": 3.351078712392843,
      "learning_rate": 7.867372576486597e-07,
      "log_odds_chosen": 1.666601538658142,
      "log_odds_ratio": -0.3801513612270355,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.890625,
      "logps/chosen": -0.643750011920929,
      "logps/rejected": -1.832421898841858,
      "loss": 0.6933,
      "nll_loss": 0.653613269329071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06441650539636612,
      "rewards/margins": 0.11861572414636612,
      "rewards/rejected": -0.18303222954273224,
      "step": 10340
    },
    {
      "epoch": 0.7536316306840937,
      "grad_norm": 3.1190796881663507,
      "learning_rate": 7.863570994927847e-07,
      "log_odds_chosen": 2.02734375,
      "log_odds_ratio": -0.33637696504592896,
      "logits/chosen": -1.0193359851837158,
      "logits/rejected": -0.8863281011581421,
      "logps/chosen": -0.659472644329071,
      "logps/rejected": -2.1656250953674316,
      "loss": 0.7128,
      "nll_loss": 0.7164062261581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06600341945886612,
      "rewards/margins": 0.15047606825828552,
      "rewards/rejected": -0.21647949516773224,
      "step": 10350
    },
    {
      "epoch": 0.7543597771871701,
      "grad_norm": 3.3793143458009594,
      "learning_rate": 7.859774918918594e-07,
      "log_odds_chosen": 1.6320312023162842,
      "log_odds_ratio": -0.36406248807907104,
      "logits/chosen": -1.0212891101837158,
      "logits/rejected": -0.9078124761581421,
      "logps/chosen": -0.649707019329071,
      "logps/rejected": -1.8136718273162842,
      "loss": 0.667,
      "nll_loss": 0.6337890625,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06494140625,
      "rewards/margins": 0.11638183891773224,
      "rewards/rejected": -0.18146972358226776,
      "step": 10360
    },
    {
      "epoch": 0.7550879236902465,
      "grad_norm": 3.2542711840019463,
      "learning_rate": 7.855984335182852e-07,
      "log_odds_chosen": 1.748437523841858,
      "log_odds_ratio": -0.35664063692092896,
      "logits/chosen": -1.023046851158142,
      "logits/rejected": -0.8753906488418579,
      "logps/chosen": -0.631054699420929,
      "logps/rejected": -1.866796851158142,
      "loss": 0.6949,
      "nll_loss": 0.6739257574081421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06315918266773224,
      "rewards/margins": 0.12374267727136612,
      "rewards/rejected": -0.18681640923023224,
      "step": 10370
    },
    {
      "epoch": 0.7558160701933229,
      "grad_norm": 2.9610342485755115,
      "learning_rate": 7.852199230489422e-07,
      "log_odds_chosen": 1.9796874523162842,
      "log_odds_ratio": -0.3091796934604645,
      "logits/chosen": -1.025390625,
      "logits/rejected": -0.8724609613418579,
      "logps/chosen": -0.630859375,
      "logps/rejected": -2.00390625,
      "loss": 0.6802,
      "nll_loss": 0.67626953125,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06308593600988388,
      "rewards/margins": 0.13720703125,
      "rewards/rejected": -0.20029297471046448,
      "step": 10380
    },
    {
      "epoch": 0.7565442166963993,
      "grad_norm": 3.359948696877461,
      "learning_rate": 7.848419591651668e-07,
      "log_odds_chosen": 1.635351538658142,
      "log_odds_ratio": -0.39082032442092896,
      "logits/chosen": -1.049414038658142,
      "logits/rejected": -0.9634765386581421,
      "logps/chosen": -0.6922851800918579,
      "logps/rejected": -1.839453101158142,
      "loss": 0.6634,
      "nll_loss": 0.65283203125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0693359375,
      "rewards/margins": 0.11456909030675888,
      "rewards/rejected": -0.18398436903953552,
      "step": 10390
    },
    {
      "epoch": 0.7572723631994758,
      "grad_norm": 4.096842113553694,
      "learning_rate": 7.844645405527361e-07,
      "log_odds_chosen": 1.99755859375,
      "log_odds_ratio": -0.30517578125,
      "logits/chosen": -1.0457031726837158,
      "logits/rejected": -0.879687488079071,
      "logps/chosen": -0.6034179925918579,
      "logps/rejected": -1.996679663658142,
      "loss": 0.6873,
      "nll_loss": 0.618457019329071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06041259691119194,
      "rewards/margins": 0.13956908881664276,
      "rewards/rejected": -0.19992676377296448,
      "step": 10400
    },
    {
      "epoch": 0.7580005097025522,
      "grad_norm": 4.835982389661848,
      "learning_rate": 7.840876659018457e-07,
      "log_odds_chosen": 1.783105492591858,
      "log_odds_ratio": -0.36152344942092896,
      "logits/chosen": -1.0509765148162842,
      "logits/rejected": -0.946093738079071,
      "logps/chosen": -0.6410156488418579,
      "logps/rejected": -1.907617211341858,
      "loss": 0.6734,
      "nll_loss": 0.6219726800918579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.0640869140625,
      "rewards/margins": 0.12663574516773224,
      "rewards/rejected": -0.19072265923023224,
      "step": 10410
    },
    {
      "epoch": 0.7587286562056286,
      "grad_norm": 3.114934264636133,
      "learning_rate": 7.837113339070922e-07,
      "log_odds_chosen": 1.5351073741912842,
      "log_odds_ratio": -0.384521484375,
      "logits/chosen": -1.0505859851837158,
      "logits/rejected": -0.9183593988418579,
      "logps/chosen": -0.6376953125,
      "logps/rejected": -1.725195288658142,
      "loss": 0.6878,
      "nll_loss": 0.6241210699081421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06376953423023224,
      "rewards/margins": 0.10887451469898224,
      "rewards/rejected": -0.1727294921875,
      "step": 10420
    },
    {
      "epoch": 0.759456802708705,
      "grad_norm": 3.380205711185707,
      "learning_rate": 7.833355432674538e-07,
      "log_odds_chosen": 1.7667968273162842,
      "log_odds_ratio": -0.35078126192092896,
      "logits/chosen": -1.0021483898162842,
      "logits/rejected": -0.9095703363418579,
      "logps/chosen": -0.611621081829071,
      "logps/rejected": -1.870507836341858,
      "loss": 0.6611,
      "nll_loss": 0.6495116949081421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06120605394244194,
      "rewards/margins": 0.12598876655101776,
      "rewards/rejected": -0.1871337890625,
      "step": 10430
    },
    {
      "epoch": 0.7601849492117814,
      "grad_norm": 2.825235781578871,
      "learning_rate": 7.829602926862713e-07,
      "log_odds_chosen": 1.701757788658142,
      "log_odds_ratio": -0.3563476502895355,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.8951171636581421,
      "logps/chosen": -0.5985351800918579,
      "logps/rejected": -1.748632788658142,
      "loss": 0.6737,
      "nll_loss": 0.6493164300918579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05985107272863388,
      "rewards/margins": 0.11495361477136612,
      "rewards/rejected": -0.17475585639476776,
      "step": 10440
    },
    {
      "epoch": 0.7609130957148579,
      "grad_norm": 2.8562420390916787,
      "learning_rate": 7.825855808712296e-07,
      "log_odds_chosen": 1.783593773841858,
      "log_odds_ratio": -0.36767578125,
      "logits/chosen": -1.0066406726837158,
      "logits/rejected": -0.878125011920929,
      "logps/chosen": -0.5751953125,
      "logps/rejected": -1.816796898841858,
      "loss": 0.655,
      "nll_loss": 0.5904296636581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05753173679113388,
      "rewards/margins": 0.12429199367761612,
      "rewards/rejected": -0.18168945610523224,
      "step": 10450
    },
    {
      "epoch": 0.7616412422179343,
      "grad_norm": 2.9153950256334435,
      "learning_rate": 7.822114065343386e-07,
      "log_odds_chosen": 2.0648436546325684,
      "log_odds_ratio": -0.2813720703125,
      "logits/chosen": -1.055273413658142,
      "logits/rejected": -0.933789074420929,
      "logps/chosen": -0.594531238079071,
      "logps/rejected": -2.028515577316284,
      "loss": 0.6872,
      "nll_loss": 0.6102539300918579,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.05949706956744194,
      "rewards/margins": 0.14332275092601776,
      "rewards/rejected": -0.2027587890625,
      "step": 10460
    },
    {
      "epoch": 0.7623693887210107,
      "grad_norm": 3.091702596822546,
      "learning_rate": 7.818377683919149e-07,
      "log_odds_chosen": 1.8925292491912842,
      "log_odds_ratio": -0.3166747987270355,
      "logits/chosen": -1.048437476158142,
      "logits/rejected": -0.9150390625,
      "logps/chosen": -0.6292968988418579,
      "logps/rejected": -1.9677734375,
      "loss": 0.6783,
      "nll_loss": 0.6895507574081421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06295166164636612,
      "rewards/margins": 0.13389167189598083,
      "rewards/rejected": -0.19672851264476776,
      "step": 10470
    },
    {
      "epoch": 0.7630975352240871,
      "grad_norm": 2.8656326332918978,
      "learning_rate": 7.814646651645635e-07,
      "log_odds_chosen": 2.1402344703674316,
      "log_odds_ratio": -0.30712890625,
      "logits/chosen": -1.0341796875,
      "logits/rejected": -0.926562488079071,
      "logps/chosen": -0.6122070550918579,
      "logps/rejected": -2.112499952316284,
      "loss": 0.6779,
      "nll_loss": 0.621777355670929,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06118164211511612,
      "rewards/margins": 0.14991454780101776,
      "rewards/rejected": -0.21086426079273224,
      "step": 10480
    },
    {
      "epoch": 0.7638256817271635,
      "grad_norm": 2.928165631640627,
      "learning_rate": 7.810920955771586e-07,
      "log_odds_chosen": 1.9775879383087158,
      "log_odds_ratio": -0.3160644471645355,
      "logits/chosen": -1.0876953601837158,
      "logits/rejected": -0.941210925579071,
      "logps/chosen": -0.646484375,
      "logps/rejected": -2.036328077316284,
      "loss": 0.6547,
      "nll_loss": 0.662304699420929,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06468506157398224,
      "rewards/margins": 0.13892212510108948,
      "rewards/rejected": -0.20356445014476776,
      "step": 10490
    },
    {
      "epoch": 0.7645538282302399,
      "grad_norm": 3.1058477151021378,
      "learning_rate": 7.807200583588266e-07,
      "log_odds_chosen": 1.911523461341858,
      "log_odds_ratio": -0.3421386778354645,
      "logits/chosen": -1.0242187976837158,
      "logits/rejected": -0.881640613079071,
      "logps/chosen": -0.628710925579071,
      "logps/rejected": -2.005078077316284,
      "loss": 0.6873,
      "nll_loss": 0.636425793170929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06287841498851776,
      "rewards/margins": 0.13776855170726776,
      "rewards/rejected": -0.200439453125,
      "step": 10500
    },
    {
      "epoch": 0.7652819747333164,
      "grad_norm": 3.4534225163879753,
      "learning_rate": 7.803485522429261e-07,
      "log_odds_chosen": 1.8896973133087158,
      "log_odds_ratio": -0.3279052674770355,
      "logits/chosen": -1.021484375,
      "logits/rejected": -0.9371093511581421,
      "logps/chosen": -0.6107422113418579,
      "logps/rejected": -1.9308593273162842,
      "loss": 0.6687,
      "nll_loss": 0.655468761920929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06113281100988388,
      "rewards/margins": 0.13204650580883026,
      "rewards/rejected": -0.19301757216453552,
      "step": 10510
    },
    {
      "epoch": 0.7660101212363928,
      "grad_norm": 3.1063430615591217,
      "learning_rate": 7.799775759670318e-07,
      "log_odds_chosen": 1.6953125,
      "log_odds_ratio": -0.38896483182907104,
      "logits/chosen": -0.996289074420929,
      "logits/rejected": -0.894726574420929,
      "logps/chosen": -0.625292956829071,
      "logps/rejected": -1.78515625,
      "loss": 0.6718,
      "nll_loss": 0.618847668170929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06252441555261612,
      "rewards/margins": 0.115966796875,
      "rewards/rejected": -0.17851562798023224,
      "step": 10520
    },
    {
      "epoch": 0.7667382677394692,
      "grad_norm": 4.0512662211975465,
      "learning_rate": 7.796071282729149e-07,
      "log_odds_chosen": 1.6613280773162842,
      "log_odds_ratio": -0.40214842557907104,
      "logits/chosen": -0.9701172113418579,
      "logits/rejected": -0.8833984136581421,
      "logps/chosen": -0.634570300579071,
      "logps/rejected": -1.721289038658142,
      "loss": 0.6795,
      "nll_loss": 0.6944335699081421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06346435844898224,
      "rewards/margins": 0.10863647609949112,
      "rewards/rejected": -0.17209473252296448,
      "step": 10530
    },
    {
      "epoch": 0.7674664142425456,
      "grad_norm": 3.1976583834716723,
      "learning_rate": 7.792372079065259e-07,
      "log_odds_chosen": 1.947851538658142,
      "log_odds_ratio": -0.3319335877895355,
      "logits/chosen": -0.990234375,
      "logits/rejected": -0.852343738079071,
      "logps/chosen": -0.620800793170929,
      "logps/rejected": -2.0140624046325684,
      "loss": 0.6896,
      "nll_loss": 0.609082043170929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06208496168255806,
      "rewards/margins": 0.13924559950828552,
      "rewards/rejected": -0.20139160752296448,
      "step": 10540
    },
    {
      "epoch": 0.768194560745622,
      "grad_norm": 2.804937041073097,
      "learning_rate": 7.788678136179767e-07,
      "log_odds_chosen": 1.5956542491912842,
      "log_odds_ratio": -0.3794921934604645,
      "logits/chosen": -1.0177733898162842,
      "logits/rejected": -0.950390636920929,
      "logps/chosen": -0.6552734375,
      "logps/rejected": -1.740234375,
      "loss": 0.6971,
      "nll_loss": 0.6177734136581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0655517578125,
      "rewards/margins": 0.10842285305261612,
      "rewards/rejected": -0.17399902641773224,
      "step": 10550
    },
    {
      "epoch": 0.7689227072486985,
      "grad_norm": 3.199333955164209,
      "learning_rate": 7.78498944161523e-07,
      "log_odds_chosen": 1.958593726158142,
      "log_odds_ratio": -0.32282716035842896,
      "logits/chosen": -1.0246093273162842,
      "logits/rejected": -0.9224609136581421,
      "logps/chosen": -0.603808581829071,
      "logps/rejected": -2.014453172683716,
      "loss": 0.651,
      "nll_loss": 0.574511706829071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06040038913488388,
      "rewards/margins": 0.14108887314796448,
      "rewards/rejected": -0.20151367783546448,
      "step": 10560
    },
    {
      "epoch": 0.7696508537517749,
      "grad_norm": 2.903284940959605,
      "learning_rate": 7.781305982955459e-07,
      "log_odds_chosen": 1.933203101158142,
      "log_odds_ratio": -0.34794920682907104,
      "logits/chosen": -1.0222656726837158,
      "logits/rejected": -0.891796886920929,
      "logps/chosen": -0.597851574420929,
      "logps/rejected": -1.9773437976837158,
      "loss": 0.6577,
      "nll_loss": 0.6014648675918579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05976562574505806,
      "rewards/margins": 0.13804931938648224,
      "rewards/rejected": -0.19772949814796448,
      "step": 10570
    },
    {
      "epoch": 0.7703790002548513,
      "grad_norm": 3.1232752284633545,
      "learning_rate": 7.777627747825355e-07,
      "log_odds_chosen": 1.736230492591858,
      "log_odds_ratio": -0.36652833223342896,
      "logits/chosen": -1.0457031726837158,
      "logits/rejected": -0.89453125,
      "logps/chosen": -0.631542980670929,
      "logps/rejected": -1.878515601158142,
      "loss": 0.707,
      "nll_loss": 0.6553710699081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.0631103515625,
      "rewards/margins": 0.12469482421875,
      "rewards/rejected": -0.18776854872703552,
      "step": 10580
    },
    {
      "epoch": 0.7711071467579277,
      "grad_norm": 2.7435623556187347,
      "learning_rate": 7.773954723890725e-07,
      "log_odds_chosen": 2.032519578933716,
      "log_odds_ratio": -0.28559571504592896,
      "logits/chosen": -1.0603516101837158,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.5796874761581421,
      "logps/rejected": -2.0257811546325684,
      "loss": 0.6671,
      "nll_loss": 0.6005859375,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.0579833984375,
      "rewards/margins": 0.14462891221046448,
      "rewards/rejected": -0.2025146484375,
      "step": 10590
    },
    {
      "epoch": 0.7718352932610041,
      "grad_norm": 2.980900191731907,
      "learning_rate": 7.770286898858113e-07,
      "log_odds_chosen": 1.822265625,
      "log_odds_ratio": -0.34211426973342896,
      "logits/chosen": -1.034570336341858,
      "logits/rejected": -0.931640625,
      "logps/chosen": -0.6172851324081421,
      "logps/rejected": -1.9460937976837158,
      "loss": 0.6634,
      "nll_loss": 0.6622070074081421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06174316257238388,
      "rewards/margins": 0.13259276747703552,
      "rewards/rejected": -0.19428710639476776,
      "step": 10600
    },
    {
      "epoch": 0.7725634397640805,
      "grad_norm": 3.2376313064477378,
      "learning_rate": 7.766624260474625e-07,
      "log_odds_chosen": 1.822265625,
      "log_odds_ratio": -0.33317869901657104,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.8753906488418579,
      "logps/chosen": -0.555957019329071,
      "logps/rejected": -1.824609398841858,
      "loss": 0.6556,
      "nll_loss": 0.573535144329071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.05556640774011612,
      "rewards/margins": 0.12690429389476776,
      "rewards/rejected": -0.1824951171875,
      "step": 10610
    },
    {
      "epoch": 0.773291586267157,
      "grad_norm": 4.7298969377052,
      "learning_rate": 7.762966796527759e-07,
      "log_odds_chosen": 1.6220703125,
      "log_odds_ratio": -0.41230469942092896,
      "logits/chosen": -1.0285155773162842,
      "logits/rejected": -0.9087890386581421,
      "logps/chosen": -0.6136718988418579,
      "logps/rejected": -1.736718773841858,
      "loss": 0.6541,
      "nll_loss": 0.636035144329071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06131591647863388,
      "rewards/margins": 0.11236572265625,
      "rewards/rejected": -0.17363281548023224,
      "step": 10620
    },
    {
      "epoch": 0.7740197327702334,
      "grad_norm": 3.1097266899620064,
      "learning_rate": 7.759314494845234e-07,
      "log_odds_chosen": 1.7687499523162842,
      "log_odds_ratio": -0.34355467557907104,
      "logits/chosen": -0.9951171875,
      "logits/rejected": -0.9027343988418579,
      "logps/chosen": -0.6133788824081421,
      "logps/rejected": -1.84375,
      "loss": 0.6681,
      "nll_loss": 0.6170898675918579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06138915941119194,
      "rewards/margins": 0.12294922024011612,
      "rewards/rejected": -0.18435057997703552,
      "step": 10630
    },
    {
      "epoch": 0.7747478792733098,
      "grad_norm": 3.6755345430176196,
      "learning_rate": 7.755667343294812e-07,
      "log_odds_chosen": 2.1292967796325684,
      "log_odds_ratio": -0.2600341737270355,
      "logits/chosen": -0.9898437261581421,
      "logits/rejected": -0.8697265386581421,
      "logps/chosen": -0.5672851800918579,
      "logps/rejected": -2.056640625,
      "loss": 0.6768,
      "nll_loss": 0.620312511920929,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.05673827975988388,
      "rewards/margins": 0.1488037109375,
      "rewards/rejected": -0.20546874403953552,
      "step": 10640
    },
    {
      "epoch": 0.7754760257763862,
      "grad_norm": 3.8995674291408573,
      "learning_rate": 7.752025329784146e-07,
      "log_odds_chosen": 2.020703077316284,
      "log_odds_ratio": -0.31342774629592896,
      "logits/chosen": -1.0537109375,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.679492175579071,
      "logps/rejected": -2.1890625953674316,
      "loss": 0.6651,
      "nll_loss": 0.6878906488418579,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06787109375,
      "rewards/margins": 0.15087890625,
      "rewards/rejected": -0.21882323920726776,
      "step": 10650
    },
    {
      "epoch": 0.7762041722794626,
      "grad_norm": 3.511940604821951,
      "learning_rate": 7.748388442260596e-07,
      "log_odds_chosen": 2.096484422683716,
      "log_odds_ratio": -0.3099365234375,
      "logits/chosen": -0.988476574420929,
      "logits/rejected": -0.8726562261581421,
      "logps/chosen": -0.635546863079071,
      "logps/rejected": -2.1800780296325684,
      "loss": 0.6667,
      "nll_loss": 0.6494140625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06350097805261612,
      "rewards/margins": 0.15457764267921448,
      "rewards/rejected": -0.218017578125,
      "step": 10660
    },
    {
      "epoch": 0.7769323187825391,
      "grad_norm": 3.790383107924629,
      "learning_rate": 7.744756668711065e-07,
      "log_odds_chosen": 1.8220703601837158,
      "log_odds_ratio": -0.36396485567092896,
      "logits/chosen": -1.0148437023162842,
      "logits/rejected": -0.908984363079071,
      "logps/chosen": -0.601269543170929,
      "logps/rejected": -1.893164038658142,
      "loss": 0.676,
      "nll_loss": 0.6240234375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0601806640625,
      "rewards/margins": 0.12899169325828552,
      "rewards/rejected": -0.18901367485523224,
      "step": 10670
    },
    {
      "epoch": 0.7776604652856155,
      "grad_norm": 3.8008301031141434,
      "learning_rate": 7.741129997161835e-07,
      "log_odds_chosen": 1.5779297351837158,
      "log_odds_ratio": -0.41455078125,
      "logits/chosen": -0.9755859375,
      "logits/rejected": -0.857226550579071,
      "logps/chosen": -0.6322265863418579,
      "logps/rejected": -1.765234351158142,
      "loss": 0.6863,
      "nll_loss": 0.6572265625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06324462592601776,
      "rewards/margins": 0.11322021484375,
      "rewards/rejected": -0.17646484076976776,
      "step": 10680
    },
    {
      "epoch": 0.7783886117886919,
      "grad_norm": 2.8834180244650476,
      "learning_rate": 7.737508415678403e-07,
      "log_odds_chosen": 1.8310546875,
      "log_odds_ratio": -0.3523925840854645,
      "logits/chosen": -0.9986327886581421,
      "logits/rejected": -0.89453125,
      "logps/chosen": -0.6373046636581421,
      "logps/rejected": -1.899999976158142,
      "loss": 0.673,
      "nll_loss": 0.60498046875,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06370849907398224,
      "rewards/margins": 0.1265869140625,
      "rewards/rejected": -0.19011230766773224,
      "step": 10690
    },
    {
      "epoch": 0.7791167582917683,
      "grad_norm": 2.5661918439270455,
      "learning_rate": 7.733891912365308e-07,
      "log_odds_chosen": 1.9285156726837158,
      "log_odds_ratio": -0.3310546875,
      "logits/chosen": -1.012304663658142,
      "logits/rejected": -0.8921874761581421,
      "logps/chosen": -0.6416991949081421,
      "logps/rejected": -2.0101561546325684,
      "loss": 0.6593,
      "nll_loss": 0.6314452886581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.064208984375,
      "rewards/margins": 0.13668212294578552,
      "rewards/rejected": -0.200927734375,
      "step": 10700
    },
    {
      "epoch": 0.7798449047948447,
      "grad_norm": 3.0985432627405824,
      "learning_rate": 7.730280475365979e-07,
      "log_odds_chosen": 1.5237305164337158,
      "log_odds_ratio": -0.41191405057907104,
      "logits/chosen": -0.9593750238418579,
      "logits/rejected": -0.875,
      "logps/chosen": -0.6201171875,
      "logps/rejected": -1.678125023841858,
      "loss": 0.6642,
      "nll_loss": 0.6612304449081421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06204833835363388,
      "rewards/margins": 0.10588226467370987,
      "rewards/rejected": -0.16783447563648224,
      "step": 10710
    },
    {
      "epoch": 0.7805730512979211,
      "grad_norm": 3.6112506795538986,
      "learning_rate": 7.726674092862557e-07,
      "log_odds_chosen": 1.8369140625,
      "log_odds_ratio": -0.30195313692092896,
      "logits/chosen": -1.022851586341858,
      "logits/rejected": -0.9244140386581421,
      "logps/chosen": -0.5601562261581421,
      "logps/rejected": -1.80859375,
      "loss": 0.6711,
      "nll_loss": 0.62158203125,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0560302734375,
      "rewards/margins": 0.12478027492761612,
      "rewards/rejected": -0.180908203125,
      "step": 10720
    },
    {
      "epoch": 0.7813011978009976,
      "grad_norm": 3.9961970160394604,
      "learning_rate": 7.723072753075748e-07,
      "log_odds_chosen": 1.698632836341858,
      "log_odds_ratio": -0.337890625,
      "logits/chosen": -1.0158202648162842,
      "logits/rejected": -0.8753906488418579,
      "logps/chosen": -0.640625,
      "logps/rejected": -1.8175780773162842,
      "loss": 0.6711,
      "nll_loss": 0.636523425579071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.0640869140625,
      "rewards/margins": 0.11765136569738388,
      "rewards/rejected": -0.18173828721046448,
      "step": 10730
    },
    {
      "epoch": 0.782029344304074,
      "grad_norm": 3.0836937219444494,
      "learning_rate": 7.719476444264649e-07,
      "log_odds_chosen": 1.6482422351837158,
      "log_odds_ratio": -0.36689454317092896,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.908398449420929,
      "logps/chosen": -0.6504882574081421,
      "logps/rejected": -1.806249976158142,
      "loss": 0.6939,
      "nll_loss": 0.627734363079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06502685695886612,
      "rewards/margins": 0.11555175483226776,
      "rewards/rejected": -0.18061523139476776,
      "step": 10740
    },
    {
      "epoch": 0.7827574908071504,
      "grad_norm": 4.241898703466103,
      "learning_rate": 7.715885154726593e-07,
      "log_odds_chosen": 1.602441430091858,
      "log_odds_ratio": -0.39069825410842896,
      "logits/chosen": -1.0310547351837158,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.701953113079071,
      "logps/rejected": -1.854101538658142,
      "loss": 0.6621,
      "nll_loss": 0.6219726800918579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07021484524011612,
      "rewards/margins": 0.11528930813074112,
      "rewards/rejected": -0.18552246689796448,
      "step": 10750
    },
    {
      "epoch": 0.7834856373102268,
      "grad_norm": 3.1792118216252665,
      "learning_rate": 7.71229887279699e-07,
      "log_odds_chosen": 1.4998047351837158,
      "log_odds_ratio": -0.40874022245407104,
      "logits/chosen": -1.006445288658142,
      "logits/rejected": -0.882617175579071,
      "logps/chosen": -0.6640625,
      "logps/rejected": -1.7101562023162842,
      "loss": 0.6779,
      "nll_loss": 0.677050769329071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06636963039636612,
      "rewards/margins": 0.10465087741613388,
      "rewards/rejected": -0.17104491591453552,
      "step": 10760
    },
    {
      "epoch": 0.7842137838133032,
      "grad_norm": 2.923011701684859,
      "learning_rate": 7.708717586849164e-07,
      "log_odds_chosen": 1.5417969226837158,
      "log_odds_ratio": -0.41923826932907104,
      "logits/chosen": -1.0642578601837158,
      "logits/rejected": -0.949023425579071,
      "logps/chosen": -0.6675781011581421,
      "logps/rejected": -1.741796851158142,
      "loss": 0.6789,
      "nll_loss": 0.616894543170929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06676025688648224,
      "rewards/margins": 0.10732422024011612,
      "rewards/rejected": -0.17404785752296448,
      "step": 10770
    },
    {
      "epoch": 0.7849419303163797,
      "grad_norm": 3.240239383400665,
      "learning_rate": 7.705141285294196e-07,
      "log_odds_chosen": 1.8400390148162842,
      "log_odds_ratio": -0.3662353456020355,
      "logits/chosen": -1.0330078601837158,
      "logits/rejected": -0.9320312738418579,
      "logps/chosen": -0.673046886920929,
      "logps/rejected": -1.9777343273162842,
      "loss": 0.6672,
      "nll_loss": 0.638964831829071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06722412258386612,
      "rewards/margins": 0.13043212890625,
      "rewards/rejected": -0.19782714545726776,
      "step": 10780
    },
    {
      "epoch": 0.7856700768194561,
      "grad_norm": 3.360942581869123,
      "learning_rate": 7.701569956580767e-07,
      "log_odds_chosen": 1.9586913585662842,
      "log_odds_ratio": -0.30791014432907104,
      "logits/chosen": -1.004492163658142,
      "logits/rejected": -0.8765624761581421,
      "logps/chosen": -0.624218761920929,
      "logps/rejected": -2.004687547683716,
      "loss": 0.6897,
      "nll_loss": 0.629687488079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06237182766199112,
      "rewards/margins": 0.13790282607078552,
      "rewards/rejected": -0.20041504502296448,
      "step": 10790
    },
    {
      "epoch": 0.7863982233225325,
      "grad_norm": 3.7008965833946146,
      "learning_rate": 7.69800358919501e-07,
      "log_odds_chosen": 1.678125023841858,
      "log_odds_ratio": -0.3548828065395355,
      "logits/chosen": -1.018945336341858,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.642382800579071,
      "logps/rejected": -1.810937523841858,
      "loss": 0.6654,
      "nll_loss": 0.6336914300918579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06419678032398224,
      "rewards/margins": 0.11705322563648224,
      "rewards/rejected": -0.18115234375,
      "step": 10800
    },
    {
      "epoch": 0.7871263698256089,
      "grad_norm": 4.655001717308508,
      "learning_rate": 7.694442171660333e-07,
      "log_odds_chosen": 1.735742211341858,
      "log_odds_ratio": -0.3363281190395355,
      "logits/chosen": -0.984179675579071,
      "logits/rejected": -0.8746093511581421,
      "logps/chosen": -0.6556640863418579,
      "logps/rejected": -1.892187476158142,
      "loss": 0.6948,
      "nll_loss": 0.628125011920929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06561279296875,
      "rewards/margins": 0.12362060695886612,
      "rewards/rejected": -0.18925781548023224,
      "step": 10810
    },
    {
      "epoch": 0.7878545163286853,
      "grad_norm": 2.9394029729479794,
      "learning_rate": 7.690885692537282e-07,
      "log_odds_chosen": 1.649023413658142,
      "log_odds_ratio": -0.35393065214157104,
      "logits/chosen": -1.004296898841858,
      "logits/rejected": -0.8658202886581421,
      "logps/chosen": -0.6016601324081421,
      "logps/rejected": -1.697656273841858,
      "loss": 0.6728,
      "nll_loss": 0.713183581829071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06014404445886612,
      "rewards/margins": 0.10963134467601776,
      "rewards/rejected": -0.1697998046875,
      "step": 10820
    },
    {
      "epoch": 0.7885826628317617,
      "grad_norm": 2.671827579557402,
      "learning_rate": 7.687334140423383e-07,
      "log_odds_chosen": 1.723046898841858,
      "log_odds_ratio": -0.3456054627895355,
      "logits/chosen": -1.015039086341858,
      "logits/rejected": -0.90234375,
      "logps/chosen": -0.6373046636581421,
      "logps/rejected": -1.8429687023162842,
      "loss": 0.6429,
      "nll_loss": 0.631054699420929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06375732272863388,
      "rewards/margins": 0.12071533501148224,
      "rewards/rejected": -0.184326171875,
      "step": 10830
    },
    {
      "epoch": 0.7893108093348382,
      "grad_norm": 2.479294771218524,
      "learning_rate": 7.683787503952985e-07,
      "log_odds_chosen": 1.765039086341858,
      "log_odds_ratio": -0.36040037870407104,
      "logits/chosen": -0.985546886920929,
      "logits/rejected": -0.8896484375,
      "logps/chosen": -0.6314452886581421,
      "logps/rejected": -1.9285156726837158,
      "loss": 0.6788,
      "nll_loss": 0.644238293170929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06315918266773224,
      "rewards/margins": 0.12956543266773224,
      "rewards/rejected": -0.19272461533546448,
      "step": 10840
    },
    {
      "epoch": 0.7900389558379146,
      "grad_norm": 2.690372064814119,
      "learning_rate": 7.680245771797108e-07,
      "log_odds_chosen": 1.706445336341858,
      "log_odds_ratio": -0.4081787168979645,
      "logits/chosen": -1.000585913658142,
      "logits/rejected": -0.8910156488418579,
      "logps/chosen": -0.651562511920929,
      "logps/rejected": -1.8371093273162842,
      "loss": 0.6902,
      "nll_loss": 0.705078125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06511230766773224,
      "rewards/margins": 0.118682861328125,
      "rewards/rejected": -0.183837890625,
      "step": 10850
    },
    {
      "epoch": 0.790767102340991,
      "grad_norm": 2.9423681412064604,
      "learning_rate": 7.676708932663293e-07,
      "log_odds_chosen": 1.6101562976837158,
      "log_odds_ratio": -0.37812501192092896,
      "logits/chosen": -1.0535156726837158,
      "logits/rejected": -0.931445300579071,
      "logps/chosen": -0.59765625,
      "logps/rejected": -1.6726562976837158,
      "loss": 0.6692,
      "nll_loss": 0.649707019329071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.059814453125,
      "rewards/margins": 0.10755614936351776,
      "rewards/rejected": -0.16743163764476776,
      "step": 10860
    },
    {
      "epoch": 0.7914952488440674,
      "grad_norm": 3.585717249905652,
      "learning_rate": 7.67317697529545e-07,
      "log_odds_chosen": 1.7677733898162842,
      "log_odds_ratio": -0.34943848848342896,
      "logits/chosen": -1.056640625,
      "logits/rejected": -0.9263671636581421,
      "logps/chosen": -0.617968738079071,
      "logps/rejected": -1.8367187976837158,
      "loss": 0.6469,
      "nll_loss": 0.590136706829071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06180419772863388,
      "rewards/margins": 0.12165527045726776,
      "rewards/rejected": -0.18339844048023224,
      "step": 10870
    },
    {
      "epoch": 0.7922233953471438,
      "grad_norm": 3.3547632404345755,
      "learning_rate": 7.669649888473704e-07,
      "log_odds_chosen": 1.9904296398162842,
      "log_odds_ratio": -0.29914551973342896,
      "logits/chosen": -1.002539038658142,
      "logits/rejected": -0.8603515625,
      "logps/chosen": -0.6255859136581421,
      "logps/rejected": -2.0093750953674316,
      "loss": 0.6646,
      "nll_loss": 0.6299804449081421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06259765475988388,
      "rewards/margins": 0.13844604790210724,
      "rewards/rejected": -0.200927734375,
      "step": 10880
    },
    {
      "epoch": 0.7929515418502202,
      "grad_norm": 4.315932150874086,
      "learning_rate": 7.666127661014253e-07,
      "log_odds_chosen": 1.609960913658142,
      "log_odds_ratio": -0.383544921875,
      "logits/chosen": -1.0226562023162842,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.617871105670929,
      "logps/rejected": -1.755859375,
      "loss": 0.6648,
      "nll_loss": 0.666308581829071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06175537034869194,
      "rewards/margins": 0.11370849609375,
      "rewards/rejected": -0.17556151747703552,
      "step": 10890
    },
    {
      "epoch": 0.7936796883532967,
      "grad_norm": 3.1298498221954816,
      "learning_rate": 7.662610281769211e-07,
      "log_odds_chosen": 1.574609398841858,
      "log_odds_ratio": -0.36962890625,
      "logits/chosen": -1.0720703601837158,
      "logits/rejected": -0.915234386920929,
      "logps/chosen": -0.6112304925918579,
      "logps/rejected": -1.689062476158142,
      "loss": 0.6459,
      "nll_loss": 0.659863293170929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06110839918255806,
      "rewards/margins": 0.10781250149011612,
      "rewards/rejected": -0.1688232421875,
      "step": 10900
    },
    {
      "epoch": 0.7944078348563731,
      "grad_norm": 3.0106831790396984,
      "learning_rate": 7.659097739626465e-07,
      "log_odds_chosen": 1.7488281726837158,
      "log_odds_ratio": -0.32373046875,
      "logits/chosen": -1.001562476158142,
      "logits/rejected": -0.8662109375,
      "logps/chosen": -0.5794922113418579,
      "logps/rejected": -1.773828148841858,
      "loss": 0.6693,
      "nll_loss": 0.629101574420929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05793457105755806,
      "rewards/margins": 0.11943359673023224,
      "rewards/rejected": -0.1773681640625,
      "step": 10910
    },
    {
      "epoch": 0.7951359813594495,
      "grad_norm": 3.656946980723928,
      "learning_rate": 7.655590023509527e-07,
      "log_odds_chosen": 1.620703101158142,
      "log_odds_ratio": -0.3748535215854645,
      "logits/chosen": -0.9730468988418579,
      "logits/rejected": -0.8636718988418579,
      "logps/chosen": -0.670605480670929,
      "logps/rejected": -1.818750023841858,
      "loss": 0.6809,
      "nll_loss": 0.63232421875,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06704101711511612,
      "rewards/margins": 0.11481933295726776,
      "rewards/rejected": -0.18186035752296448,
      "step": 10920
    },
    {
      "epoch": 0.7958641278625259,
      "grad_norm": 2.8169047409018986,
      "learning_rate": 7.652087122377384e-07,
      "log_odds_chosen": 1.955078125,
      "log_odds_ratio": -0.29548341035842896,
      "logits/chosen": -1.0046875476837158,
      "logits/rejected": -0.8960937261581421,
      "logps/chosen": -0.597363293170929,
      "logps/rejected": -1.939843773841858,
      "loss": 0.6722,
      "nll_loss": 0.647167980670929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05971679836511612,
      "rewards/margins": 0.13430175185203552,
      "rewards/rejected": -0.19389648735523224,
      "step": 10930
    },
    {
      "epoch": 0.7965922743656023,
      "grad_norm": 3.19810255524397,
      "learning_rate": 7.648589025224355e-07,
      "log_odds_chosen": 1.6326172351837158,
      "log_odds_ratio": -0.403564453125,
      "logits/chosen": -1.0539062023162842,
      "logits/rejected": -0.94140625,
      "logps/chosen": -0.68017578125,
      "logps/rejected": -1.8253905773162842,
      "loss": 0.6918,
      "nll_loss": 0.696093738079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06805419921875,
      "rewards/margins": 0.11455078423023224,
      "rewards/rejected": -0.1827392578125,
      "step": 10940
    },
    {
      "epoch": 0.7973204208686788,
      "grad_norm": 2.8135726653004802,
      "learning_rate": 7.645095721079945e-07,
      "log_odds_chosen": 1.6330077648162842,
      "log_odds_ratio": -0.3629394471645355,
      "logits/chosen": -1.0412108898162842,
      "logits/rejected": -0.931640625,
      "logps/chosen": -0.6294921636581421,
      "logps/rejected": -1.740234375,
      "loss": 0.6696,
      "nll_loss": 0.605175793170929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06292114406824112,
      "rewards/margins": 0.11109618842601776,
      "rewards/rejected": -0.17409667372703552,
      "step": 10950
    },
    {
      "epoch": 0.7980485673717552,
      "grad_norm": 3.1243789300682745,
      "learning_rate": 7.641607199008701e-07,
      "log_odds_chosen": 2.167187452316284,
      "log_odds_ratio": -0.3006347715854645,
      "logits/chosen": -1.0236327648162842,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.6124023199081421,
      "logps/rejected": -2.1644530296325684,
      "loss": 0.6569,
      "nll_loss": 0.5926758050918579,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06124267727136612,
      "rewards/margins": 0.15522460639476776,
      "rewards/rejected": -0.21635742485523224,
      "step": 10960
    },
    {
      "epoch": 0.7987767138748316,
      "grad_norm": 2.9517420772961525,
      "learning_rate": 7.638123448110066e-07,
      "log_odds_chosen": 1.8957030773162842,
      "log_odds_ratio": -0.29230958223342896,
      "logits/chosen": -1.034765601158142,
      "logits/rejected": -0.9267578125,
      "logps/chosen": -0.6358398199081421,
      "logps/rejected": -1.9841797351837158,
      "loss": 0.6768,
      "nll_loss": 0.659960925579071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06358642876148224,
      "rewards/margins": 0.13474121689796448,
      "rewards/rejected": -0.19831542670726776,
      "step": 10970
    },
    {
      "epoch": 0.799504860377908,
      "grad_norm": 3.2652952357721383,
      "learning_rate": 7.634644457518242e-07,
      "log_odds_chosen": 1.9343750476837158,
      "log_odds_ratio": -0.31804198026657104,
      "logits/chosen": -1.0041015148162842,
      "logits/rejected": -0.919726550579071,
      "logps/chosen": -0.6216796636581421,
      "logps/rejected": -2.0074219703674316,
      "loss": 0.6712,
      "nll_loss": 0.589648425579071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06220703199505806,
      "rewards/margins": 0.13861083984375,
      "rewards/rejected": -0.20085449516773224,
      "step": 10980
    },
    {
      "epoch": 0.8002330068809844,
      "grad_norm": 3.1738213481998536,
      "learning_rate": 7.631170216402039e-07,
      "log_odds_chosen": 1.899023413658142,
      "log_odds_ratio": -0.35820311307907104,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.890820324420929,
      "logps/chosen": -0.6006835699081421,
      "logps/rejected": -1.953125,
      "loss": 0.666,
      "nll_loss": 0.5933593511581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06008300930261612,
      "rewards/margins": 0.13524170219898224,
      "rewards/rejected": -0.19526366889476776,
      "step": 10990
    },
    {
      "epoch": 0.8009611533840608,
      "grad_norm": 3.5810528219372513,
      "learning_rate": 7.627700713964739e-07,
      "log_odds_chosen": 1.860693335533142,
      "log_odds_ratio": -0.33317869901657104,
      "logits/chosen": -1.0505859851837158,
      "logits/rejected": -0.9302734136581421,
      "logps/chosen": -0.5865234136581421,
      "logps/rejected": -1.853124976158142,
      "loss": 0.6501,
      "nll_loss": 0.595898449420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05864257737994194,
      "rewards/margins": 0.1267745941877365,
      "rewards/rejected": -0.18552246689796448,
      "step": 11000
    },
    {
      "epoch": 0.8016892998871373,
      "grad_norm": 3.0628404995114273,
      "learning_rate": 7.624235939443953e-07,
      "log_odds_chosen": 1.4306640625,
      "log_odds_ratio": -0.42280274629592896,
      "logits/chosen": -1.041015625,
      "logits/rejected": -0.9193359613418579,
      "logps/chosen": -0.64306640625,
      "logps/rejected": -1.6892578601837158,
      "loss": 0.669,
      "nll_loss": 0.6343749761581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06430663913488388,
      "rewards/margins": 0.10476074367761612,
      "rewards/rejected": -0.16904297471046448,
      "step": 11010
    },
    {
      "epoch": 0.8024174463902137,
      "grad_norm": 3.6114063174615314,
      "learning_rate": 7.620775882111482e-07,
      "log_odds_chosen": 1.6169922351837158,
      "log_odds_ratio": -0.3812255859375,
      "logits/chosen": -0.991992175579071,
      "logits/rejected": -0.8974609375,
      "logps/chosen": -0.646289050579071,
      "logps/rejected": -1.78515625,
      "loss": 0.65,
      "nll_loss": 0.6299804449081421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06459961086511612,
      "rewards/margins": 0.1138916015625,
      "rewards/rejected": -0.1785888671875,
      "step": 11020
    },
    {
      "epoch": 0.8031455928932901,
      "grad_norm": 6.155153717871646,
      "learning_rate": 7.617320531273181e-07,
      "log_odds_chosen": 1.6474609375,
      "log_odds_ratio": -0.3651367127895355,
      "logits/chosen": -1.0232422351837158,
      "logits/rejected": -0.9072265625,
      "logps/chosen": -0.6201171875,
      "logps/rejected": -1.7341797351837158,
      "loss": 0.6528,
      "nll_loss": 0.578906238079071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06208496168255806,
      "rewards/margins": 0.11127929389476776,
      "rewards/rejected": -0.17329101264476776,
      "step": 11030
    },
    {
      "epoch": 0.8038737393963665,
      "grad_norm": 2.7970028574370036,
      "learning_rate": 7.613869876268809e-07,
      "log_odds_chosen": 1.8357422351837158,
      "log_odds_ratio": -0.359619140625,
      "logits/chosen": -1.0244140625,
      "logits/rejected": -0.9306640625,
      "logps/chosen": -0.64501953125,
      "logps/rejected": -1.956640601158142,
      "loss": 0.6764,
      "nll_loss": 0.6226562261581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06447754055261612,
      "rewards/margins": 0.131103515625,
      "rewards/rejected": -0.19560547173023224,
      "step": 11040
    },
    {
      "epoch": 0.8046018858994429,
      "grad_norm": 3.164900432781597,
      "learning_rate": 7.610423906471905e-07,
      "log_odds_chosen": 1.800195336341858,
      "log_odds_ratio": -0.3350585997104645,
      "logits/chosen": -1.0048828125,
      "logits/rejected": -0.8656250238418579,
      "logps/chosen": -0.6138671636581421,
      "logps/rejected": -1.8771483898162842,
      "loss": 0.6684,
      "nll_loss": 0.62890625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06135253980755806,
      "rewards/margins": 0.12630614638328552,
      "rewards/rejected": -0.1876220703125,
      "step": 11050
    },
    {
      "epoch": 0.8053300324025194,
      "grad_norm": 3.936710424669633,
      "learning_rate": 7.606982611289639e-07,
      "log_odds_chosen": 1.759374976158142,
      "log_odds_ratio": -0.33989256620407104,
      "logits/chosen": -1.0041015148162842,
      "logits/rejected": -0.876171886920929,
      "logps/chosen": -0.6229492425918579,
      "logps/rejected": -1.853124976158142,
      "loss": 0.6458,
      "nll_loss": 0.60693359375,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.062255859375,
      "rewards/margins": 0.12327881157398224,
      "rewards/rejected": -0.18557128310203552,
      "step": 11060
    },
    {
      "epoch": 0.8060581789055958,
      "grad_norm": 3.164371582686929,
      "learning_rate": 7.60354598016268e-07,
      "log_odds_chosen": 1.708593726158142,
      "log_odds_ratio": -0.317626953125,
      "logits/chosen": -1.052343726158142,
      "logits/rejected": -0.89453125,
      "logps/chosen": -0.591015636920929,
      "logps/rejected": -1.743749976158142,
      "loss": 0.6703,
      "nll_loss": 0.647265613079071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05909423902630806,
      "rewards/margins": 0.11538086086511612,
      "rewards/rejected": -0.17441406846046448,
      "step": 11070
    },
    {
      "epoch": 0.8067863254086722,
      "grad_norm": 3.9317529173029655,
      "learning_rate": 7.600114002565063e-07,
      "log_odds_chosen": 1.8855469226837158,
      "log_odds_ratio": -0.358154296875,
      "logits/chosen": -1.0246093273162842,
      "logits/rejected": -0.892773449420929,
      "logps/chosen": -0.584179699420929,
      "logps/rejected": -1.8781249523162842,
      "loss": 0.6515,
      "nll_loss": 0.6000000238418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05843506008386612,
      "rewards/margins": 0.12934570014476776,
      "rewards/rejected": -0.187744140625,
      "step": 11080
    },
    {
      "epoch": 0.8075144719117486,
      "grad_norm": 2.921169314358238,
      "learning_rate": 7.596686668004049e-07,
      "log_odds_chosen": 2.1058592796325684,
      "log_odds_ratio": -0.31352537870407104,
      "logits/chosen": -1.0271484851837158,
      "logits/rejected": -0.9056640863418579,
      "logps/chosen": -0.584765613079071,
      "logps/rejected": -2.0816407203674316,
      "loss": 0.6574,
      "nll_loss": 0.6629883050918579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.0584716796875,
      "rewards/margins": 0.14979247748851776,
      "rewards/rejected": -0.20827636122703552,
      "step": 11090
    },
    {
      "epoch": 0.808242618414825,
      "grad_norm": 3.1413359301150705,
      "learning_rate": 7.593263966019991e-07,
      "log_odds_chosen": 1.72314453125,
      "log_odds_ratio": -0.35053712129592896,
      "logits/chosen": -1.018945336341858,
      "logits/rejected": -0.9156249761581421,
      "logps/chosen": -0.650390625,
      "logps/rejected": -1.804296851158142,
      "loss": 0.6805,
      "nll_loss": 0.647753894329071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06503906100988388,
      "rewards/margins": 0.11531372368335724,
      "rewards/rejected": -0.18044432997703552,
      "step": 11100
    },
    {
      "epoch": 0.8089707649179014,
      "grad_norm": 3.660424685435505,
      "learning_rate": 7.589845886186201e-07,
      "log_odds_chosen": 1.9972655773162842,
      "log_odds_ratio": -0.31059569120407104,
      "logits/chosen": -1.0076172351837158,
      "logits/rejected": -0.878125011920929,
      "logps/chosen": -0.5908203125,
      "logps/rejected": -1.9296875,
      "loss": 0.6491,
      "nll_loss": 0.627734363079071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05910034105181694,
      "rewards/margins": 0.134033203125,
      "rewards/rejected": -0.19296875596046448,
      "step": 11110
    },
    {
      "epoch": 0.8096989114209779,
      "grad_norm": 3.1292878488073486,
      "learning_rate": 7.586432418108816e-07,
      "log_odds_chosen": 1.750585913658142,
      "log_odds_ratio": -0.3636474609375,
      "logits/chosen": -1.033593773841858,
      "logits/rejected": -0.9185546636581421,
      "logps/chosen": -0.6166015863418579,
      "logps/rejected": -1.877343773841858,
      "loss": 0.6562,
      "nll_loss": 0.5956054925918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06165771558880806,
      "rewards/margins": 0.12634888291358948,
      "rewards/rejected": -0.187744140625,
      "step": 11120
    },
    {
      "epoch": 0.8104270579240543,
      "grad_norm": 3.1633293981080657,
      "learning_rate": 7.583023551426664e-07,
      "log_odds_chosen": 1.945898413658142,
      "log_odds_ratio": -0.31403809785842896,
      "logits/chosen": -1.0470702648162842,
      "logits/rejected": -0.9312499761581421,
      "logps/chosen": -0.5765625238418579,
      "logps/rejected": -1.856054663658142,
      "loss": 0.6606,
      "nll_loss": 0.5693359375,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05771484225988388,
      "rewards/margins": 0.12775878608226776,
      "rewards/rejected": -0.18547363579273224,
      "step": 11130
    },
    {
      "epoch": 0.8111552044271307,
      "grad_norm": 3.265074441098985,
      "learning_rate": 7.579619275811138e-07,
      "log_odds_chosen": 1.5771484375,
      "log_odds_ratio": -0.36884766817092896,
      "logits/chosen": -1.0417969226837158,
      "logits/rejected": -0.9439452886581421,
      "logps/chosen": -0.609667956829071,
      "logps/rejected": -1.6681640148162842,
      "loss": 0.649,
      "nll_loss": 0.595507800579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06099853664636612,
      "rewards/margins": 0.10581054538488388,
      "rewards/rejected": -0.16689452528953552,
      "step": 11140
    },
    {
      "epoch": 0.8118833509302071,
      "grad_norm": 2.929257606566552,
      "learning_rate": 7.576219580966055e-07,
      "log_odds_chosen": 1.9246094226837158,
      "log_odds_ratio": -0.30927735567092896,
      "logits/chosen": -1.0498046875,
      "logits/rejected": -0.935742199420929,
      "logps/chosen": -0.615039050579071,
      "logps/rejected": -1.954687476158142,
      "loss": 0.6571,
      "nll_loss": 0.60302734375,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06157226487994194,
      "rewards/margins": 0.13405761122703552,
      "rewards/rejected": -0.19545897841453552,
      "step": 11150
    },
    {
      "epoch": 0.8126114974332835,
      "grad_norm": 3.571352611987179,
      "learning_rate": 7.57282445662753e-07,
      "log_odds_chosen": 1.843359351158142,
      "log_odds_ratio": -0.3465332090854645,
      "logits/chosen": -1.0607421398162842,
      "logits/rejected": -0.922656238079071,
      "logps/chosen": -0.6514648199081421,
      "logps/rejected": -1.9638671875,
      "loss": 0.6517,
      "nll_loss": 0.660351574420929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06514892727136612,
      "rewards/margins": 0.13126221299171448,
      "rewards/rejected": -0.19638672471046448,
      "step": 11160
    },
    {
      "epoch": 0.81333964393636,
      "grad_norm": 3.224662924766102,
      "learning_rate": 7.569433892563852e-07,
      "log_odds_chosen": 2.054492235183716,
      "log_odds_ratio": -0.3000244200229645,
      "logits/chosen": -1.011328101158142,
      "logits/rejected": -0.884570300579071,
      "logps/chosen": -0.5796874761581421,
      "logps/rejected": -2.02734375,
      "loss": 0.6611,
      "nll_loss": 0.6024414300918579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05793457105755806,
      "rewards/margins": 0.14479979872703552,
      "rewards/rejected": -0.2027587890625,
      "step": 11170
    },
    {
      "epoch": 0.8140677904394364,
      "grad_norm": 2.9240758064547756,
      "learning_rate": 7.566047878575343e-07,
      "log_odds_chosen": 1.996679663658142,
      "log_odds_ratio": -0.33232420682907104,
      "logits/chosen": -1.008203148841858,
      "logits/rejected": -0.882031261920929,
      "logps/chosen": -0.622753918170929,
      "logps/rejected": -2.0494141578674316,
      "loss": 0.6576,
      "nll_loss": 0.6650390625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06230468675494194,
      "rewards/margins": 0.14248046278953552,
      "rewards/rejected": -0.20480957627296448,
      "step": 11180
    },
    {
      "epoch": 0.8147959369425128,
      "grad_norm": 3.28743929150249,
      "learning_rate": 7.562666404494236e-07,
      "log_odds_chosen": 1.8572266101837158,
      "log_odds_ratio": -0.3323730528354645,
      "logits/chosen": -1.039648413658142,
      "logits/rejected": -0.906054675579071,
      "logps/chosen": -0.5731445550918579,
      "logps/rejected": -1.8386719226837158,
      "loss": 0.6369,
      "nll_loss": 0.6055663824081421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05727539211511612,
      "rewards/margins": 0.12684325873851776,
      "rewards/rejected": -0.1842041015625,
      "step": 11190
    },
    {
      "epoch": 0.8155240834455892,
      "grad_norm": 2.6788451286746393,
      "learning_rate": 7.559289460184543e-07,
      "log_odds_chosen": 1.5681641101837158,
      "log_odds_ratio": -0.39018553495407104,
      "logits/chosen": -0.992968738079071,
      "logits/rejected": -0.908203125,
      "logps/chosen": -0.685742199420929,
      "logps/rejected": -1.757421851158142,
      "loss": 0.6693,
      "nll_loss": 0.6328125,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06862793117761612,
      "rewards/margins": 0.10698242485523224,
      "rewards/rejected": -0.17561034858226776,
      "step": 11200
    },
    {
      "epoch": 0.8162522299486656,
      "grad_norm": 2.750034803523753,
      "learning_rate": 7.555917035541937e-07,
      "log_odds_chosen": 1.690039038658142,
      "log_odds_ratio": -0.32158201932907104,
      "logits/chosen": -1.0402343273162842,
      "logits/rejected": -0.8792968988418579,
      "logps/chosen": -0.58447265625,
      "logps/rejected": -1.7296874523162842,
      "loss": 0.6621,
      "nll_loss": 0.7041015625,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05844726413488388,
      "rewards/margins": 0.11457519233226776,
      "rewards/rejected": -0.17304687201976776,
      "step": 11210
    },
    {
      "epoch": 0.816980376451742,
      "grad_norm": 3.530553877554674,
      "learning_rate": 7.552549120493609e-07,
      "log_odds_chosen": 1.649023413658142,
      "log_odds_ratio": -0.3490234315395355,
      "logits/chosen": -1.0167968273162842,
      "logits/rejected": -0.9107421636581421,
      "logps/chosen": -0.5870116949081421,
      "logps/rejected": -1.6521484851837158,
      "loss": 0.6642,
      "nll_loss": 0.650683581829071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05869140475988388,
      "rewards/margins": 0.10640869289636612,
      "rewards/rejected": -0.16513672471046448,
      "step": 11220
    },
    {
      "epoch": 0.8177085229548186,
      "grad_norm": 3.442758701207568,
      "learning_rate": 7.549185704998158e-07,
      "log_odds_chosen": 1.8986327648162842,
      "log_odds_ratio": -0.3333496153354645,
      "logits/chosen": -0.982226550579071,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.5570312738418579,
      "logps/rejected": -1.899999976158142,
      "loss": 0.6662,
      "nll_loss": 0.6063476800918579,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.05572509765625,
      "rewards/margins": 0.13435058295726776,
      "rewards/rejected": -0.19008788466453552,
      "step": 11230
    },
    {
      "epoch": 0.818436669457895,
      "grad_norm": 2.817580634003762,
      "learning_rate": 7.545826779045449e-07,
      "log_odds_chosen": 1.8118164539337158,
      "log_odds_ratio": -0.33745115995407104,
      "logits/chosen": -1.049414038658142,
      "logits/rejected": -0.929882824420929,
      "logps/chosen": -0.633007824420929,
      "logps/rejected": -1.9025390148162842,
      "loss": 0.6677,
      "nll_loss": 0.65625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06334228813648224,
      "rewards/margins": 0.12695617973804474,
      "rewards/rejected": -0.19038085639476776,
      "step": 11240
    },
    {
      "epoch": 0.8191648159609713,
      "grad_norm": 3.0093045597107495,
      "learning_rate": 7.542472332656506e-07,
      "log_odds_chosen": 1.69677734375,
      "log_odds_ratio": -0.3783203065395355,
      "logits/chosen": -1.0080077648162842,
      "logits/rejected": -0.884765625,
      "logps/chosen": -0.632519543170929,
      "logps/rejected": -1.8029296398162842,
      "loss": 0.6681,
      "nll_loss": 0.6630859375,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06324462592601776,
      "rewards/margins": 0.11716308444738388,
      "rewards/rejected": -0.18032225966453552,
      "step": 11250
    },
    {
      "epoch": 0.8198929624640477,
      "grad_norm": 3.2660817685520596,
      "learning_rate": 7.539122355883373e-07,
      "log_odds_chosen": 2.0873045921325684,
      "log_odds_ratio": -0.3197265565395355,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.9291015863418579,
      "logps/chosen": -0.6309570074081421,
      "logps/rejected": -2.1070313453674316,
      "loss": 0.6533,
      "nll_loss": 0.649218738079071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06303711235523224,
      "rewards/margins": 0.14774170517921448,
      "rewards/rejected": -0.21064452826976776,
      "step": 11260
    },
    {
      "epoch": 0.8206211089671241,
      "grad_norm": 3.113510711762068,
      "learning_rate": 7.535776838808995e-07,
      "log_odds_chosen": 1.6960937976837158,
      "log_odds_ratio": -0.36494141817092896,
      "logits/chosen": -0.995312511920929,
      "logits/rejected": -0.891796886920929,
      "logps/chosen": -0.6373046636581421,
      "logps/rejected": -1.7898437976837158,
      "loss": 0.6538,
      "nll_loss": 0.6318359375,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.063720703125,
      "rewards/margins": 0.11517333984375,
      "rewards/rejected": -0.17905274033546448,
      "step": 11270
    },
    {
      "epoch": 0.8213492554702007,
      "grad_norm": 4.054063821239141,
      "learning_rate": 7.532435771547094e-07,
      "log_odds_chosen": 2.043164014816284,
      "log_odds_ratio": -0.33977049589157104,
      "logits/chosen": -1.0398437976837158,
      "logits/rejected": -0.936718761920929,
      "logps/chosen": -0.669726550579071,
      "logps/rejected": -2.106250047683716,
      "loss": 0.6603,
      "nll_loss": 0.6424316167831421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06688232719898224,
      "rewards/margins": 0.14375610649585724,
      "rewards/rejected": -0.21054688096046448,
      "step": 11280
    },
    {
      "epoch": 0.822077401973277,
      "grad_norm": 3.0650980401397123,
      "learning_rate": 7.52909914424205e-07,
      "log_odds_chosen": 1.875390648841858,
      "log_odds_ratio": -0.3628906309604645,
      "logits/chosen": -1.0789062976837158,
      "logits/rejected": -0.9400390386581421,
      "logps/chosen": -0.605664074420929,
      "logps/rejected": -1.913476586341858,
      "loss": 0.6664,
      "nll_loss": 0.6426757574081421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06058349460363388,
      "rewards/margins": 0.13088378310203552,
      "rewards/rejected": -0.19138184189796448,
      "step": 11290
    },
    {
      "epoch": 0.8228055484763535,
      "grad_norm": 3.2052527918292943,
      "learning_rate": 7.525766947068777e-07,
      "log_odds_chosen": 2.1314454078674316,
      "log_odds_ratio": -0.29194337129592896,
      "logits/chosen": -1.0388672351837158,
      "logits/rejected": -0.9125000238418579,
      "logps/chosen": -0.6333984136581421,
      "logps/rejected": -2.134765625,
      "loss": 0.6393,
      "nll_loss": 0.6327148675918579,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06329345703125,
      "rewards/margins": 0.15020751953125,
      "rewards/rejected": -0.21357421576976776,
      "step": 11300
    },
    {
      "epoch": 0.8235336949794299,
      "grad_norm": 3.7166027753858404,
      "learning_rate": 7.522439170232598e-07,
      "log_odds_chosen": 1.963476538658142,
      "log_odds_ratio": -0.3194824159145355,
      "logits/chosen": -1.0986328125,
      "logits/rejected": -0.9673827886581421,
      "logps/chosen": -0.6405273675918579,
      "logps/rejected": -2.059765577316284,
      "loss": 0.6689,
      "nll_loss": 0.6572265625,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06407471001148224,
      "rewards/margins": 0.14194336533546448,
      "rewards/rejected": -0.20590820908546448,
      "step": 11310
    },
    {
      "epoch": 0.8242618414825063,
      "grad_norm": 3.263157858211095,
      "learning_rate": 7.519115803969124e-07,
      "log_odds_chosen": 1.437158226966858,
      "log_odds_ratio": -0.427490234375,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.8548828363418579,
      "logps/chosen": -0.62451171875,
      "logps/rejected": -1.6505858898162842,
      "loss": 0.6625,
      "nll_loss": 0.6458984613418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06241454929113388,
      "rewards/margins": 0.10265426337718964,
      "rewards/rejected": -0.16507568955421448,
      "step": 11320
    },
    {
      "epoch": 0.8249899879855827,
      "grad_norm": 2.8644159911044405,
      "learning_rate": 7.515796838544139e-07,
      "log_odds_chosen": 2.0048828125,
      "log_odds_ratio": -0.320556640625,
      "logits/chosen": -0.987109363079071,
      "logits/rejected": -0.8980468511581421,
      "logps/chosen": -0.5897461175918579,
      "logps/rejected": -2.0169920921325684,
      "loss": 0.6594,
      "nll_loss": 0.619433581829071,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05888671800494194,
      "rewards/margins": 0.14273682236671448,
      "rewards/rejected": -0.20173339545726776,
      "step": 11330
    },
    {
      "epoch": 0.8257181344886592,
      "grad_norm": 3.1745019180009115,
      "learning_rate": 7.51248226425348e-07,
      "log_odds_chosen": 1.6224243640899658,
      "log_odds_ratio": -0.3744873106479645,
      "logits/chosen": -1.029687523841858,
      "logits/rejected": -0.907031238079071,
      "logps/chosen": -0.593945324420929,
      "logps/rejected": -1.7117187976837158,
      "loss": 0.6672,
      "nll_loss": 0.591503918170929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.05944214016199112,
      "rewards/margins": 0.11165008693933487,
      "rewards/rejected": -0.17094726860523224,
      "step": 11340
    },
    {
      "epoch": 0.8264462809917356,
      "grad_norm": 3.1631271114992776,
      "learning_rate": 7.509172071422913e-07,
      "log_odds_chosen": 1.892187476158142,
      "log_odds_ratio": -0.32636719942092896,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.9052734375,
      "logps/chosen": -0.635546863079071,
      "logps/rejected": -1.9773437976837158,
      "loss": 0.6621,
      "nll_loss": 0.6541992425918579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06356201320886612,
      "rewards/margins": 0.13419190049171448,
      "rewards/rejected": -0.19758300483226776,
      "step": 11350
    },
    {
      "epoch": 0.827174427494812,
      "grad_norm": 3.239606588287667,
      "learning_rate": 7.505866250408015e-07,
      "log_odds_chosen": 1.8581054210662842,
      "log_odds_ratio": -0.3843994140625,
      "logits/chosen": -1.0421874523162842,
      "logits/rejected": -0.9380859136581421,
      "logps/chosen": -0.612109363079071,
      "logps/rejected": -1.9128906726837158,
      "loss": 0.6495,
      "nll_loss": 0.6201171875,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06123046949505806,
      "rewards/margins": 0.13011474907398224,
      "rewards/rejected": -0.19118651747703552,
      "step": 11360
    },
    {
      "epoch": 0.8279025739978884,
      "grad_norm": 2.8763463778455978,
      "learning_rate": 7.50256479159406e-07,
      "log_odds_chosen": 1.8439452648162842,
      "log_odds_ratio": -0.34516602754592896,
      "logits/chosen": -1.017578125,
      "logits/rejected": -0.892773449420929,
      "logps/chosen": -0.5985351800918579,
      "logps/rejected": -1.884765625,
      "loss": 0.6739,
      "nll_loss": 0.6236327886581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05987548828125,
      "rewards/margins": 0.12857666611671448,
      "rewards/rejected": -0.18857422471046448,
      "step": 11370
    },
    {
      "epoch": 0.8286307205009648,
      "grad_norm": 5.024358054909604,
      "learning_rate": 7.499267685395902e-07,
      "log_odds_chosen": 1.395410180091858,
      "log_odds_ratio": -0.4173339903354645,
      "logits/chosen": -1.068359375,
      "logits/rejected": -0.9521484375,
      "logps/chosen": -0.598828136920929,
      "logps/rejected": -1.521875023841858,
      "loss": 0.6552,
      "nll_loss": 0.5604492425918579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.05989990383386612,
      "rewards/margins": 0.09227295219898224,
      "rewards/rejected": -0.15214844048023224,
      "step": 11380
    },
    {
      "epoch": 0.8293588670040413,
      "grad_norm": 2.796570911044722,
      "learning_rate": 7.495974922257845e-07,
      "log_odds_chosen": 2.035351514816284,
      "log_odds_ratio": -0.3526855409145355,
      "logits/chosen": -1.077539086341858,
      "logits/rejected": -0.953125,
      "logps/chosen": -0.637402355670929,
      "logps/rejected": -2.1292967796325684,
      "loss": 0.6703,
      "nll_loss": 0.5975586175918579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06368408352136612,
      "rewards/margins": 0.14926758408546448,
      "rewards/rejected": -0.21311035752296448,
      "step": 11390
    },
    {
      "epoch": 0.8300870135071177,
      "grad_norm": 3.393061365888499,
      "learning_rate": 7.492686492653552e-07,
      "log_odds_chosen": 1.6384766101837158,
      "log_odds_ratio": -0.43647462129592896,
      "logits/chosen": -0.998828113079071,
      "logits/rejected": -0.908398449420929,
      "logps/chosen": -0.709179699420929,
      "logps/rejected": -1.869140625,
      "loss": 0.6525,
      "nll_loss": 0.7298828363418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07087402045726776,
      "rewards/margins": 0.11611328274011612,
      "rewards/rejected": -0.18701171875,
      "step": 11400
    },
    {
      "epoch": 0.8308151600101941,
      "grad_norm": 2.793734700798136,
      "learning_rate": 7.489402387085902e-07,
      "log_odds_chosen": 2.1441407203674316,
      "log_odds_ratio": -0.29606932401657104,
      "logits/chosen": -1.059960961341858,
      "logits/rejected": -0.8740234375,
      "logps/chosen": -0.597363293170929,
      "logps/rejected": -2.174999952316284,
      "loss": 0.6329,
      "nll_loss": 0.685351550579071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.05976562574505806,
      "rewards/margins": 0.15764160454273224,
      "rewards/rejected": -0.217529296875,
      "step": 11410
    },
    {
      "epoch": 0.8315433065132705,
      "grad_norm": 3.093862894804681,
      "learning_rate": 7.486122596086891e-07,
      "log_odds_chosen": 2.0394530296325684,
      "log_odds_ratio": -0.30571287870407104,
      "logits/chosen": -0.994140625,
      "logits/rejected": -0.8999999761581421,
      "logps/chosen": -0.5669921636581421,
      "logps/rejected": -1.9933593273162842,
      "loss": 0.654,
      "nll_loss": 0.6480468511581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05673827975988388,
      "rewards/margins": 0.14259032905101776,
      "rewards/rejected": -0.19938965141773224,
      "step": 11420
    },
    {
      "epoch": 0.8322714530163469,
      "grad_norm": 4.586616041195406,
      "learning_rate": 7.482847110217516e-07,
      "log_odds_chosen": 2.056445360183716,
      "log_odds_ratio": -0.3172363340854645,
      "logits/chosen": -1.087304711341858,
      "logits/rejected": -0.930468738079071,
      "logps/chosen": -0.617382824420929,
      "logps/rejected": -2.117968797683716,
      "loss": 0.659,
      "nll_loss": 0.6014648675918579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.061767578125,
      "rewards/margins": 0.14995117485523224,
      "rewards/rejected": -0.21171875298023224,
      "step": 11430
    },
    {
      "epoch": 0.8329995995194233,
      "grad_norm": 4.741960912181229,
      "learning_rate": 7.479575920067657e-07,
      "log_odds_chosen": 1.9272949695587158,
      "log_odds_ratio": -0.35246580839157104,
      "logits/chosen": -0.9693359136581421,
      "logits/rejected": -0.8892577886581421,
      "logps/chosen": -0.6001952886581421,
      "logps/rejected": -1.931249976158142,
      "loss": 0.6628,
      "nll_loss": 0.66064453125,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06004638597369194,
      "rewards/margins": 0.13314971327781677,
      "rewards/rejected": -0.1932373046875,
      "step": 11440
    },
    {
      "epoch": 0.8337277460224998,
      "grad_norm": 3.665612314230371,
      "learning_rate": 7.476309016255964e-07,
      "log_odds_chosen": 1.7958984375,
      "log_odds_ratio": -0.3338867127895355,
      "logits/chosen": -1.016015648841858,
      "logits/rejected": -0.8990234136581421,
      "logps/chosen": -0.6195312738418579,
      "logps/rejected": -1.8488280773162842,
      "loss": 0.6553,
      "nll_loss": 0.63330078125,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06196289137005806,
      "rewards/margins": 0.12309570610523224,
      "rewards/rejected": -0.18503418564796448,
      "step": 11450
    },
    {
      "epoch": 0.8344558925255762,
      "grad_norm": 3.4192078058289246,
      "learning_rate": 7.473046389429744e-07,
      "log_odds_chosen": 1.8955078125,
      "log_odds_ratio": -0.34736329317092896,
      "logits/chosen": -1.042382836341858,
      "logits/rejected": -0.964062511920929,
      "logps/chosen": -0.633496105670929,
      "logps/rejected": -1.962890625,
      "loss": 0.6573,
      "nll_loss": 0.658398449420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.0633544921875,
      "rewards/margins": 0.133056640625,
      "rewards/rejected": -0.1962890625,
      "step": 11460
    },
    {
      "epoch": 0.8351840390286526,
      "grad_norm": 3.1673102122542742,
      "learning_rate": 7.469788030264852e-07,
      "log_odds_chosen": 1.7138671875,
      "log_odds_ratio": -0.3623046875,
      "logits/chosen": -1.0427734851837158,
      "logits/rejected": -0.9173828363418579,
      "logps/chosen": -0.6270507574081421,
      "logps/rejected": -1.8250000476837158,
      "loss": 0.6551,
      "nll_loss": 0.61328125,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06270752102136612,
      "rewards/margins": 0.11968994140625,
      "rewards/rejected": -0.18247070908546448,
      "step": 11470
    },
    {
      "epoch": 0.835912185531729,
      "grad_norm": 2.9576955746130538,
      "learning_rate": 7.466533929465574e-07,
      "log_odds_chosen": 1.7677733898162842,
      "log_odds_ratio": -0.367431640625,
      "logits/chosen": -1.005859375,
      "logits/rejected": -0.882031261920929,
      "logps/chosen": -0.6214843988418579,
      "logps/rejected": -1.783593773841858,
      "loss": 0.6571,
      "nll_loss": 0.650683581829071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06221923977136612,
      "rewards/margins": 0.11613769829273224,
      "rewards/rejected": -0.17832031846046448,
      "step": 11480
    },
    {
      "epoch": 0.8366403320348054,
      "grad_norm": 2.792897507403079,
      "learning_rate": 7.463284077764519e-07,
      "log_odds_chosen": 1.805078148841858,
      "log_odds_ratio": -0.36328125,
      "logits/chosen": -1.0447266101837158,
      "logits/rejected": -0.941601574420929,
      "logps/chosen": -0.61572265625,
      "logps/rejected": -1.845312476158142,
      "loss": 0.6538,
      "nll_loss": 0.6128906011581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.0615234375,
      "rewards/margins": 0.1229248046875,
      "rewards/rejected": -0.1844482421875,
      "step": 11490
    },
    {
      "epoch": 0.8373684785378818,
      "grad_norm": 2.970537732519707,
      "learning_rate": 7.460038465922511e-07,
      "log_odds_chosen": 1.8595702648162842,
      "log_odds_ratio": -0.33183592557907104,
      "logits/chosen": -1.061132788658142,
      "logits/rejected": -0.879101574420929,
      "logps/chosen": -0.6124023199081421,
      "logps/rejected": -1.929296851158142,
      "loss": 0.6496,
      "nll_loss": 0.601757824420929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06126708909869194,
      "rewards/margins": 0.1318359375,
      "rewards/rejected": -0.193115234375,
      "step": 11500
    },
    {
      "epoch": 0.8380966250409583,
      "grad_norm": 2.9997581784110183,
      "learning_rate": 7.456797084728466e-07,
      "log_odds_chosen": 1.7685546875,
      "log_odds_ratio": -0.32390135526657104,
      "logits/chosen": -1.0431640148162842,
      "logits/rejected": -0.911328136920929,
      "logps/chosen": -0.5870116949081421,
      "logps/rejected": -1.7873046398162842,
      "loss": 0.6851,
      "nll_loss": 0.5843750238418579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05870361253619194,
      "rewards/margins": 0.12017822265625,
      "rewards/rejected": -0.17878417670726776,
      "step": 11510
    },
    {
      "epoch": 0.8388247715440347,
      "grad_norm": 6.26084550830444,
      "learning_rate": 7.453559924999299e-07,
      "log_odds_chosen": 1.9617187976837158,
      "log_odds_ratio": -0.281982421875,
      "logits/chosen": -1.0636718273162842,
      "logits/rejected": -0.934765636920929,
      "logps/chosen": -0.5775390863418579,
      "logps/rejected": -1.929101586341858,
      "loss": 0.6397,
      "nll_loss": 0.585742175579071,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.05778808519244194,
      "rewards/margins": 0.13508300483226776,
      "rewards/rejected": -0.19304199516773224,
      "step": 11520
    },
    {
      "epoch": 0.8395529180471111,
      "grad_norm": 2.676866490369077,
      "learning_rate": 7.450326977579804e-07,
      "log_odds_chosen": 1.809960961341858,
      "log_odds_ratio": -0.3433593809604645,
      "logits/chosen": -1.101171851158142,
      "logits/rejected": -0.9580078125,
      "logps/chosen": -0.5908203125,
      "logps/rejected": -1.880859375,
      "loss": 0.6607,
      "nll_loss": 0.536914050579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.05904541164636612,
      "rewards/margins": 0.12905272841453552,
      "rewards/rejected": -0.1881103515625,
      "step": 11530
    },
    {
      "epoch": 0.8402810645501875,
      "grad_norm": 3.6346227692081166,
      "learning_rate": 7.447098233342549e-07,
      "log_odds_chosen": 1.941992163658142,
      "log_odds_ratio": -0.33143311738967896,
      "logits/chosen": -1.0126953125,
      "logits/rejected": -0.9423828125,
      "logps/chosen": -0.58837890625,
      "logps/rejected": -1.9324219226837158,
      "loss": 0.668,
      "nll_loss": 0.596386730670929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05877685546875,
      "rewards/margins": 0.13435058295726776,
      "rewards/rejected": -0.19313964247703552,
      "step": 11540
    },
    {
      "epoch": 0.8410092110532639,
      "grad_norm": 2.7334693574353994,
      "learning_rate": 7.443873683187767e-07,
      "log_odds_chosen": 1.820898413658142,
      "log_odds_ratio": -0.3306640684604645,
      "logits/chosen": -1.0056641101837158,
      "logits/rejected": -0.905468761920929,
      "logps/chosen": -0.625683605670929,
      "logps/rejected": -1.870703101158142,
      "loss": 0.6621,
      "nll_loss": 0.6841796636581421,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06253661960363388,
      "rewards/margins": 0.12452392280101776,
      "rewards/rejected": -0.18706054985523224,
      "step": 11550
    },
    {
      "epoch": 0.8417373575563404,
      "grad_norm": 4.027813470561238,
      "learning_rate": 7.440653318043245e-07,
      "log_odds_chosen": 1.616308569908142,
      "log_odds_ratio": -0.35627442598342896,
      "logits/chosen": -1.0197265148162842,
      "logits/rejected": -0.9232422113418579,
      "logps/chosen": -0.6039062738418579,
      "logps/rejected": -1.715234398841858,
      "loss": 0.6406,
      "nll_loss": 0.59619140625,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06036376953125,
      "rewards/margins": 0.1112060546875,
      "rewards/rejected": -0.1715087890625,
      "step": 11560
    },
    {
      "epoch": 0.8424655040594168,
      "grad_norm": 3.1593928837426404,
      "learning_rate": 7.437437128864224e-07,
      "log_odds_chosen": 1.640527367591858,
      "log_odds_ratio": -0.4122070372104645,
      "logits/chosen": -0.9859374761581421,
      "logits/rejected": -0.904101550579071,
      "logps/chosen": -0.6400390863418579,
      "logps/rejected": -1.7957031726837158,
      "loss": 0.6622,
      "nll_loss": 0.6231445074081421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06406249850988388,
      "rewards/margins": 0.11555175483226776,
      "rewards/rejected": -0.17966309189796448,
      "step": 11570
    },
    {
      "epoch": 0.8431936505624932,
      "grad_norm": 2.898510061298245,
      "learning_rate": 7.434225106633287e-07,
      "log_odds_chosen": 1.83984375,
      "log_odds_ratio": -0.34150391817092896,
      "logits/chosen": -1.015234351158142,
      "logits/rejected": -0.942578136920929,
      "logps/chosen": -0.5762695074081421,
      "logps/rejected": -1.873437523841858,
      "loss": 0.6623,
      "nll_loss": 0.6114257574081421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05760497972369194,
      "rewards/margins": 0.12990722060203552,
      "rewards/rejected": -0.18742676079273224,
      "step": 11580
    },
    {
      "epoch": 0.8439217970655696,
      "grad_norm": 4.519953862492492,
      "learning_rate": 7.431017242360253e-07,
      "log_odds_chosen": 1.882226586341858,
      "log_odds_ratio": -0.3495117127895355,
      "logits/chosen": -1.0281250476837158,
      "logits/rejected": -0.9183593988418579,
      "logps/chosen": -0.6568359136581421,
      "logps/rejected": -2.0240235328674316,
      "loss": 0.6199,
      "nll_loss": 0.613964855670929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06572265923023224,
      "rewards/margins": 0.13662108778953552,
      "rewards/rejected": -0.2022705078125,
      "step": 11590
    },
    {
      "epoch": 0.844649943568646,
      "grad_norm": 2.913019080399133,
      "learning_rate": 7.427813527082074e-07,
      "log_odds_chosen": 1.8662109375,
      "log_odds_ratio": -0.34619140625,
      "logits/chosen": -1.0017578601837158,
      "logits/rejected": -0.837890625,
      "logps/chosen": -0.6201171875,
      "logps/rejected": -1.951171875,
      "loss": 0.6233,
      "nll_loss": 0.6737304925918579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06203613430261612,
      "rewards/margins": 0.13289794325828552,
      "rewards/rejected": -0.19501952826976776,
      "step": 11600
    },
    {
      "epoch": 0.8453780900717224,
      "grad_norm": 3.1604149123971665,
      "learning_rate": 7.424613951862727e-07,
      "log_odds_chosen": 1.9982421398162842,
      "log_odds_ratio": -0.31108397245407104,
      "logits/chosen": -1.062109351158142,
      "logits/rejected": -0.920703113079071,
      "logps/chosen": -0.570019543170929,
      "logps/rejected": -1.9738280773162842,
      "loss": 0.6502,
      "nll_loss": 0.596386730670929,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.0570068359375,
      "rewards/margins": 0.14029541611671448,
      "rewards/rejected": -0.19741210341453552,
      "step": 11610
    },
    {
      "epoch": 0.8461062365747989,
      "grad_norm": 2.875035665510533,
      "learning_rate": 7.42141850779311e-07,
      "log_odds_chosen": 1.842187523841858,
      "log_odds_ratio": -0.32612305879592896,
      "logits/chosen": -1.078710913658142,
      "logits/rejected": -0.9214843511581421,
      "logps/chosen": -0.6201171875,
      "logps/rejected": -1.923437476158142,
      "loss": 0.6238,
      "nll_loss": 0.586718738079071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06198730319738388,
      "rewards/margins": 0.13035888969898224,
      "rewards/rejected": -0.19243164360523224,
      "step": 11620
    },
    {
      "epoch": 0.8468343830778753,
      "grad_norm": 3.534827983935677,
      "learning_rate": 7.418227185990941e-07,
      "log_odds_chosen": 1.870703101158142,
      "log_odds_ratio": -0.31257325410842896,
      "logits/chosen": -1.0652344226837158,
      "logits/rejected": -0.9033203125,
      "logps/chosen": -0.6138671636581421,
      "logps/rejected": -1.928125023841858,
      "loss": 0.6715,
      "nll_loss": 0.6151367425918579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06142578274011612,
      "rewards/margins": 0.13127441704273224,
      "rewards/rejected": -0.19265136122703552,
      "step": 11630
    },
    {
      "epoch": 0.8475625295809517,
      "grad_norm": 2.977238912891985,
      "learning_rate": 7.415039977600647e-07,
      "log_odds_chosen": 1.7019531726837158,
      "log_odds_ratio": -0.36479490995407104,
      "logits/chosen": -0.981640636920929,
      "logits/rejected": -0.90625,
      "logps/chosen": -0.6097656488418579,
      "logps/rejected": -1.769921898841858,
      "loss": 0.6443,
      "nll_loss": 0.5914062261581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06098632887005806,
      "rewards/margins": 0.11606445163488388,
      "rewards/rejected": -0.17709961533546448,
      "step": 11640
    },
    {
      "epoch": 0.8482906760840281,
      "grad_norm": 3.333171701340095,
      "learning_rate": 7.411856873793271e-07,
      "log_odds_chosen": 1.79736328125,
      "log_odds_ratio": -0.3292236328125,
      "logits/chosen": -0.991992175579071,
      "logits/rejected": -0.922656238079071,
      "logps/chosen": -0.638671875,
      "logps/rejected": -1.8810546398162842,
      "loss": 0.6795,
      "nll_loss": 0.619335949420929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06381835788488388,
      "rewards/margins": 0.12437744438648224,
      "rewards/rejected": -0.18798828125,
      "step": 11650
    },
    {
      "epoch": 0.8490188225871045,
      "grad_norm": 2.648290325363302,
      "learning_rate": 7.408677865766361e-07,
      "log_odds_chosen": 1.6769530773162842,
      "log_odds_ratio": -0.3785644471645355,
      "logits/chosen": -1.0125000476837158,
      "logits/rejected": -0.927929699420929,
      "logps/chosen": -0.5970703363418579,
      "logps/rejected": -1.7384765148162842,
      "loss": 0.6443,
      "nll_loss": 0.5936523675918579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.05972900241613388,
      "rewards/margins": 0.11397095024585724,
      "rewards/rejected": -0.17373046278953552,
      "step": 11660
    },
    {
      "epoch": 0.849746969090181,
      "grad_norm": 3.483421461174114,
      "learning_rate": 7.405502944743868e-07,
      "log_odds_chosen": 1.4861328601837158,
      "log_odds_ratio": -0.3998046815395355,
      "logits/chosen": -1.044531226158142,
      "logits/rejected": -0.94921875,
      "logps/chosen": -0.6375976800918579,
      "logps/rejected": -1.6248047351837158,
      "loss": 0.6714,
      "nll_loss": 0.5916992425918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06370849907398224,
      "rewards/margins": 0.09884033352136612,
      "rewards/rejected": -0.16257324814796448,
      "step": 11670
    },
    {
      "epoch": 0.8504751155932574,
      "grad_norm": 2.827005187709465,
      "learning_rate": 7.402332101976052e-07,
      "log_odds_chosen": 1.7529296875,
      "log_odds_ratio": -0.37211912870407104,
      "logits/chosen": -1.0392577648162842,
      "logits/rejected": -0.9056640863418579,
      "logps/chosen": -0.6041015386581421,
      "logps/rejected": -1.823828101158142,
      "loss": 0.6413,
      "nll_loss": 0.602343738079071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06038818508386612,
      "rewards/margins": 0.12193603813648224,
      "rewards/rejected": -0.18239745497703552,
      "step": 11680
    },
    {
      "epoch": 0.8512032620963338,
      "grad_norm": 3.1171854068915605,
      "learning_rate": 7.399165328739372e-07,
      "log_odds_chosen": 1.669531226158142,
      "log_odds_ratio": -0.3480468690395355,
      "logits/chosen": -1.0095703601837158,
      "logits/rejected": -0.919921875,
      "logps/chosen": -0.604199230670929,
      "logps/rejected": -1.7265625,
      "loss": 0.6699,
      "nll_loss": 0.6241210699081421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06047363206744194,
      "rewards/margins": 0.11229248344898224,
      "rewards/rejected": -0.1727294921875,
      "step": 11690
    },
    {
      "epoch": 0.8519314085994102,
      "grad_norm": 3.193969539676958,
      "learning_rate": 7.396002616336387e-07,
      "log_odds_chosen": 2.0238280296325684,
      "log_odds_ratio": -0.3099609315395355,
      "logits/chosen": -1.064453125,
      "logits/rejected": -0.957812488079071,
      "logps/chosen": -0.5956054925918579,
      "logps/rejected": -2.012890577316284,
      "loss": 0.6243,
      "nll_loss": 0.625781238079071,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.05960693210363388,
      "rewards/margins": 0.14167480170726776,
      "rewards/rejected": -0.201171875,
      "step": 11700
    },
    {
      "epoch": 0.8526595551024866,
      "grad_norm": 2.972579380245185,
      "learning_rate": 7.392843956095663e-07,
      "log_odds_chosen": 1.9113280773162842,
      "log_odds_ratio": -0.3084716796875,
      "logits/chosen": -1.040429711341858,
      "logits/rejected": -0.8896484375,
      "logps/chosen": -0.6327148675918579,
      "logps/rejected": -2.0113282203674316,
      "loss": 0.6632,
      "nll_loss": 0.6449218988418579,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06324462592601776,
      "rewards/margins": 0.13802489638328552,
      "rewards/rejected": -0.20109863579273224,
      "step": 11710
    },
    {
      "epoch": 0.853387701605563,
      "grad_norm": 2.9635600167568334,
      "learning_rate": 7.389689339371664e-07,
      "log_odds_chosen": 2.0210938453674316,
      "log_odds_ratio": -0.3182617127895355,
      "logits/chosen": -1.0041015148162842,
      "logits/rejected": -0.902148425579071,
      "logps/chosen": -0.5819336175918579,
      "logps/rejected": -2.01953125,
      "loss": 0.6444,
      "nll_loss": 0.609667956829071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05820312350988388,
      "rewards/margins": 0.14383545517921448,
      "rewards/rejected": -0.2020263671875,
      "step": 11720
    },
    {
      "epoch": 0.8541158481086395,
      "grad_norm": 3.0200406381876794,
      "learning_rate": 7.386538757544653e-07,
      "log_odds_chosen": 1.8058593273162842,
      "log_odds_ratio": -0.35332030057907104,
      "logits/chosen": -1.0705077648162842,
      "logits/rejected": -0.92578125,
      "logps/chosen": -0.614453136920929,
      "logps/rejected": -1.8787109851837158,
      "loss": 0.6643,
      "nll_loss": 0.586132824420929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06138915941119194,
      "rewards/margins": 0.12629088759422302,
      "rewards/rejected": -0.18779297173023224,
      "step": 11730
    },
    {
      "epoch": 0.8548439946117159,
      "grad_norm": 3.0075781140901974,
      "learning_rate": 7.3833922020206e-07,
      "log_odds_chosen": 1.563085913658142,
      "log_odds_ratio": -0.37944334745407104,
      "logits/chosen": -1.035546898841858,
      "logits/rejected": -0.9046875238418579,
      "logps/chosen": -0.5970703363418579,
      "logps/rejected": -1.6457030773162842,
      "loss": 0.6446,
      "nll_loss": 0.6075195074081421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05975341796875,
      "rewards/margins": 0.10491333156824112,
      "rewards/rejected": -0.16464844346046448,
      "step": 11740
    },
    {
      "epoch": 0.8555721411147923,
      "grad_norm": 3.3267755047376806,
      "learning_rate": 7.38024966423108e-07,
      "log_odds_chosen": 1.8369140625,
      "log_odds_ratio": -0.3197265565395355,
      "logits/chosen": -1.048242211341858,
      "logits/rejected": -0.8935546875,
      "logps/chosen": -0.646191418170929,
      "logps/rejected": -1.946874976158142,
      "loss": 0.6527,
      "nll_loss": 0.640917956829071,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06462402641773224,
      "rewards/margins": 0.13015136122703552,
      "rewards/rejected": -0.19482421875,
      "step": 11750
    },
    {
      "epoch": 0.8563002876178687,
      "grad_norm": 2.96219985355083,
      "learning_rate": 7.377111135633174e-07,
      "log_odds_chosen": 1.946874976158142,
      "log_odds_ratio": -0.3067382872104645,
      "logits/chosen": -1.0205078125,
      "logits/rejected": -0.9169921875,
      "logps/chosen": -0.5765625238418579,
      "logps/rejected": -1.9421875476837158,
      "loss": 0.6473,
      "nll_loss": 0.5980468988418579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05767822265625,
      "rewards/margins": 0.13671875,
      "rewards/rejected": -0.19416503608226776,
      "step": 11760
    },
    {
      "epoch": 0.8570284341209451,
      "grad_norm": 2.649287474374631,
      "learning_rate": 7.373976607709372e-07,
      "log_odds_chosen": 2.032421827316284,
      "log_odds_ratio": -0.30805665254592896,
      "logits/chosen": -0.9957031011581421,
      "logits/rejected": -0.8628906011581421,
      "logps/chosen": -0.604199230670929,
      "logps/rejected": -2.033984422683716,
      "loss": 0.6261,
      "nll_loss": 0.565625011920929,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.0604248046875,
      "rewards/margins": 0.14296874403953552,
      "rewards/rejected": -0.20341797173023224,
      "step": 11770
    },
    {
      "epoch": 0.8577565806240216,
      "grad_norm": 2.976918933155713,
      "learning_rate": 7.370846071967476e-07,
      "log_odds_chosen": 2.00732421875,
      "log_odds_ratio": -0.29484862089157104,
      "logits/chosen": -1.015234351158142,
      "logits/rejected": -0.910937488079071,
      "logps/chosen": -0.591113269329071,
      "logps/rejected": -1.9968750476837158,
      "loss": 0.651,
      "nll_loss": 0.6258789300918579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05911865085363388,
      "rewards/margins": 0.14060059189796448,
      "rewards/rejected": -0.19980469346046448,
      "step": 11780
    },
    {
      "epoch": 0.858484727127098,
      "grad_norm": 3.1251895645991894,
      "learning_rate": 7.367719519940501e-07,
      "log_odds_chosen": 2.140625,
      "log_odds_ratio": -0.24553222954273224,
      "logits/chosen": -1.014062523841858,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.539355456829071,
      "logps/rejected": -2.006640672683716,
      "loss": 0.6375,
      "nll_loss": 0.636035144329071,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.05389404296875,
      "rewards/margins": 0.14667968451976776,
      "rewards/rejected": -0.20063476264476776,
      "step": 11790
    },
    {
      "epoch": 0.8592128736301744,
      "grad_norm": 2.883316822109799,
      "learning_rate": 7.364596943186587e-07,
      "log_odds_chosen": 1.637109398841858,
      "log_odds_ratio": -0.3851562440395355,
      "logits/chosen": -1.039453148841858,
      "logits/rejected": -0.939648449420929,
      "logps/chosen": -0.6080077886581421,
      "logps/rejected": -1.74609375,
      "loss": 0.6595,
      "nll_loss": 0.5775390863418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06072998046875,
      "rewards/margins": 0.11387939751148224,
      "rewards/rejected": -0.17465820908546448,
      "step": 11800
    },
    {
      "epoch": 0.8599410201332508,
      "grad_norm": 3.333268013125844,
      "learning_rate": 7.36147833328889e-07,
      "log_odds_chosen": 1.7361328601837158,
      "log_odds_ratio": -0.33427733182907104,
      "logits/chosen": -1.0314452648162842,
      "logits/rejected": -0.905078113079071,
      "logps/chosen": -0.6044921875,
      "logps/rejected": -1.821874976158142,
      "loss": 0.6595,
      "nll_loss": 0.6724609136581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06051025539636612,
      "rewards/margins": 0.12174072116613388,
      "rewards/rejected": -0.18234863877296448,
      "step": 11810
    },
    {
      "epoch": 0.8606691666363272,
      "grad_norm": 3.139319103116092,
      "learning_rate": 7.358363681855503e-07,
      "log_odds_chosen": 1.830468773841858,
      "log_odds_ratio": -0.30610352754592896,
      "logits/chosen": -1.038671851158142,
      "logits/rejected": -0.9242187738418579,
      "logps/chosen": -0.56884765625,
      "logps/rejected": -1.8251953125,
      "loss": 0.6509,
      "nll_loss": 0.580371081829071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05686645582318306,
      "rewards/margins": 0.1256103515625,
      "rewards/rejected": -0.18266601860523224,
      "step": 11820
    },
    {
      "epoch": 0.8613973131394036,
      "grad_norm": 3.9762326733545,
      "learning_rate": 7.355252980519345e-07,
      "log_odds_chosen": 1.708398461341858,
      "log_odds_ratio": -0.36201173067092896,
      "logits/chosen": -1.0158202648162842,
      "logits/rejected": -0.9136718511581421,
      "logps/chosen": -0.656542956829071,
      "logps/rejected": -1.827539086341858,
      "loss": 0.6519,
      "nll_loss": 0.6463867425918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06566162407398224,
      "rewards/margins": 0.11705322563648224,
      "rewards/rejected": -0.18281249701976776,
      "step": 11830
    },
    {
      "epoch": 0.8621254596424801,
      "grad_norm": 3.1620133089045366,
      "learning_rate": 7.352146220938078e-07,
      "log_odds_chosen": 1.701171875,
      "log_odds_ratio": -0.3658691346645355,
      "logits/chosen": -1.024023413658142,
      "logits/rejected": -0.942187488079071,
      "logps/chosen": -0.6537109613418579,
      "logps/rejected": -1.859765648841858,
      "loss": 0.6497,
      "nll_loss": 0.586621105670929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06531982123851776,
      "rewards/margins": 0.12081298977136612,
      "rewards/rejected": -0.18613281846046448,
      "step": 11840
    },
    {
      "epoch": 0.8628536061455565,
      "grad_norm": 3.3404711207405877,
      "learning_rate": 7.349043394794005e-07,
      "log_odds_chosen": 2.1664061546325684,
      "log_odds_ratio": -0.33784180879592896,
      "logits/chosen": -1.0636718273162842,
      "logits/rejected": -0.9306640625,
      "logps/chosen": -0.5985351800918579,
      "logps/rejected": -2.109375,
      "loss": 0.6432,
      "nll_loss": 0.613574206829071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05983886867761612,
      "rewards/margins": 0.15115967392921448,
      "rewards/rejected": -0.21113280951976776,
      "step": 11850
    },
    {
      "epoch": 0.8635817526486329,
      "grad_norm": 3.013386670427959,
      "learning_rate": 7.345944493793987e-07,
      "log_odds_chosen": 2.0799803733825684,
      "log_odds_ratio": -0.2929443418979645,
      "logits/chosen": -1.021875023841858,
      "logits/rejected": -0.9169921875,
      "logps/chosen": -0.55224609375,
      "logps/rejected": -1.9658203125,
      "loss": 0.6194,
      "nll_loss": 0.531933605670929,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.05520019680261612,
      "rewards/margins": 0.14133301377296448,
      "rewards/rejected": -0.1966552734375,
      "step": 11860
    },
    {
      "epoch": 0.8643098991517093,
      "grad_norm": 3.4311166827148525,
      "learning_rate": 7.342849509669337e-07,
      "log_odds_chosen": 1.700781226158142,
      "log_odds_ratio": -0.32954102754592896,
      "logits/chosen": -0.9947265386581421,
      "logits/rejected": -0.910351574420929,
      "logps/chosen": -0.6302734613418579,
      "logps/rejected": -1.7685546875,
      "loss": 0.6679,
      "nll_loss": 0.6104491949081421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06301269680261612,
      "rewards/margins": 0.11386718600988388,
      "rewards/rejected": -0.17685547471046448,
      "step": 11870
    },
    {
      "epoch": 0.8650380456547857,
      "grad_norm": 2.9315536413642653,
      "learning_rate": 7.339758434175737e-07,
      "log_odds_chosen": 1.626367211341858,
      "log_odds_ratio": -0.3388671875,
      "logits/chosen": -1.031835913658142,
      "logits/rejected": -0.9378906488418579,
      "logps/chosen": -0.585742175579071,
      "logps/rejected": -1.646484375,
      "loss": 0.6602,
      "nll_loss": 0.582812488079071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05855713039636612,
      "rewards/margins": 0.10601806640625,
      "rewards/rejected": -0.16462402045726776,
      "step": 11880
    },
    {
      "epoch": 0.8657661921578622,
      "grad_norm": 3.302322260005392,
      "learning_rate": 7.336671259093143e-07,
      "log_odds_chosen": 2.1578125953674316,
      "log_odds_ratio": -0.2988037168979645,
      "logits/chosen": -1.0271484851837158,
      "logits/rejected": -0.897656261920929,
      "logps/chosen": -0.64111328125,
      "logps/rejected": -2.233203172683716,
      "loss": 0.6351,
      "nll_loss": 0.6444336175918579,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06409911811351776,
      "rewards/margins": 0.15932616591453552,
      "rewards/rejected": -0.22314453125,
      "step": 11890
    },
    {
      "epoch": 0.8664943386609386,
      "grad_norm": 3.7757070535828636,
      "learning_rate": 7.33358797622569e-07,
      "log_odds_chosen": 1.7175781726837158,
      "log_odds_ratio": -0.3316894471645355,
      "logits/chosen": -1.018945336341858,
      "logits/rejected": -0.915234386920929,
      "logps/chosen": -0.6258789300918579,
      "logps/rejected": -1.798828125,
      "loss": 0.6367,
      "nll_loss": 0.5953124761581421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0626220703125,
      "rewards/margins": 0.11746826022863388,
      "rewards/rejected": -0.17983397841453552,
      "step": 11900
    },
    {
      "epoch": 0.867222485164015,
      "grad_norm": 3.4719309993052625,
      "learning_rate": 7.330508577401606e-07,
      "log_odds_chosen": 1.9404296875,
      "log_odds_ratio": -0.35791015625,
      "logits/chosen": -1.023046851158142,
      "logits/rejected": -0.899121105670929,
      "logps/chosen": -0.6014648675918579,
      "logps/rejected": -1.9978516101837158,
      "loss": 0.6597,
      "nll_loss": 0.578369140625,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06010742112994194,
      "rewards/margins": 0.13980713486671448,
      "rewards/rejected": -0.19997557997703552,
      "step": 11910
    },
    {
      "epoch": 0.8679506316670914,
      "grad_norm": 3.145567629864164,
      "learning_rate": 7.327433054473117e-07,
      "log_odds_chosen": 1.682226538658142,
      "log_odds_ratio": -0.365966796875,
      "logits/chosen": -1.0564453601837158,
      "logits/rejected": -0.9359375238418579,
      "logps/chosen": -0.6050781011581421,
      "logps/rejected": -1.760351538658142,
      "loss": 0.6455,
      "nll_loss": 0.6180664300918579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06053466722369194,
      "rewards/margins": 0.11551208794116974,
      "rewards/rejected": -0.17600098252296448,
      "step": 11920
    },
    {
      "epoch": 0.8686787781701678,
      "grad_norm": 3.1652703390578125,
      "learning_rate": 7.324361399316357e-07,
      "log_odds_chosen": 1.853124976158142,
      "log_odds_ratio": -0.36420899629592896,
      "logits/chosen": -1.0486328601837158,
      "logits/rejected": -0.893750011920929,
      "logps/chosen": -0.615917980670929,
      "logps/rejected": -1.911718726158142,
      "loss": 0.6475,
      "nll_loss": 0.6219726800918579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06159668043255806,
      "rewards/margins": 0.12958984076976776,
      "rewards/rejected": -0.19123534858226776,
      "step": 11930
    },
    {
      "epoch": 0.8694069246732442,
      "grad_norm": 3.2255022767504324,
      "learning_rate": 7.321293603831281e-07,
      "log_odds_chosen": 1.6564452648162842,
      "log_odds_ratio": -0.37153321504592896,
      "logits/chosen": -1.0271484851837158,
      "logits/rejected": -0.9039062261581421,
      "logps/chosen": -0.5889648199081421,
      "logps/rejected": -1.740625023841858,
      "loss": 0.6515,
      "nll_loss": 0.6314452886581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.058868408203125,
      "rewards/margins": 0.11531982570886612,
      "rewards/rejected": -0.1741943359375,
      "step": 11940
    },
    {
      "epoch": 0.8701350711763207,
      "grad_norm": 3.2518042271716503,
      "learning_rate": 7.318229659941572e-07,
      "log_odds_chosen": 1.8292968273162842,
      "log_odds_ratio": -0.3357910215854645,
      "logits/chosen": -1.0294921398162842,
      "logits/rejected": -0.887890636920929,
      "logps/chosen": -0.605664074420929,
      "logps/rejected": -1.8439452648162842,
      "loss": 0.6409,
      "nll_loss": 0.563671886920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06059570237994194,
      "rewards/margins": 0.12396240234375,
      "rewards/rejected": -0.18452148139476776,
      "step": 11950
    },
    {
      "epoch": 0.8708632176793971,
      "grad_norm": 2.912082434896695,
      "learning_rate": 7.315169559594551e-07,
      "log_odds_chosen": 1.7268555164337158,
      "log_odds_ratio": -0.3636230528354645,
      "logits/chosen": -1.011132836341858,
      "logits/rejected": -0.885937511920929,
      "logps/chosen": -0.6064453125,
      "logps/rejected": -1.795507788658142,
      "loss": 0.6365,
      "nll_loss": 0.62890625,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06063232570886612,
      "rewards/margins": 0.118988037109375,
      "rewards/rejected": -0.17954102158546448,
      "step": 11960
    },
    {
      "epoch": 0.8715913641824735,
      "grad_norm": 2.7986971549865167,
      "learning_rate": 7.31211329476109e-07,
      "log_odds_chosen": 1.48193359375,
      "log_odds_ratio": -0.40107423067092896,
      "logits/chosen": -0.968945324420929,
      "logits/rejected": -0.8382812738418579,
      "logps/chosen": -0.5990234613418579,
      "logps/rejected": -1.5988280773162842,
      "loss": 0.6403,
      "nll_loss": 0.611621081829071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.05992431566119194,
      "rewards/margins": 0.10006103664636612,
      "rewards/rejected": -0.15998534858226776,
      "step": 11970
    },
    {
      "epoch": 0.8723195106855499,
      "grad_norm": 3.367046803003604,
      "learning_rate": 7.309060857435526e-07,
      "log_odds_chosen": 1.8224608898162842,
      "log_odds_ratio": -0.36518555879592896,
      "logits/chosen": -1.0251953601837158,
      "logits/rejected": -0.8851562738418579,
      "logps/chosen": -0.6241210699081421,
      "logps/rejected": -1.906640648841858,
      "loss": 0.6443,
      "nll_loss": 0.603515625,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.0623779296875,
      "rewards/margins": 0.12830810248851776,
      "rewards/rejected": -0.19062499701976776,
      "step": 11980
    },
    {
      "epoch": 0.8730476571886263,
      "grad_norm": 2.960880222250494,
      "learning_rate": 7.30601223963557e-07,
      "log_odds_chosen": 1.7488281726837158,
      "log_odds_ratio": -0.33281248807907104,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.919921875,
      "logps/chosen": -0.5732421875,
      "logps/rejected": -1.748046875,
      "loss": 0.626,
      "nll_loss": 0.5352538824081421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.05728759616613388,
      "rewards/margins": 0.11761474609375,
      "rewards/rejected": -0.17485351860523224,
      "step": 11990
    },
    {
      "epoch": 0.8737758036917028,
      "grad_norm": 3.4311793510699067,
      "learning_rate": 7.302967433402214e-07,
      "log_odds_chosen": 2.0687499046325684,
      "log_odds_ratio": -0.30278319120407104,
      "logits/chosen": -1.0751953125,
      "logits/rejected": -0.9443359375,
      "logps/chosen": -0.588671863079071,
      "logps/rejected": -2.0234375,
      "loss": 0.6597,
      "nll_loss": 0.5892578363418579,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.058837890625,
      "rewards/margins": 0.14348144829273224,
      "rewards/rejected": -0.20224609971046448,
      "step": 12000
    },
    {
      "epoch": 0.8745039501947792,
      "grad_norm": 3.0201559523377,
      "learning_rate": 7.299926430799657e-07,
      "log_odds_chosen": 1.6706054210662842,
      "log_odds_ratio": -0.3809570372104645,
      "logits/chosen": -1.0478515625,
      "logits/rejected": -0.936718761920929,
      "logps/chosen": -0.644824206829071,
      "logps/rejected": -1.801171898841858,
      "loss": 0.6342,
      "nll_loss": 0.559765636920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06448974460363388,
      "rewards/margins": 0.11567382514476776,
      "rewards/rejected": -0.18027344346046448,
      "step": 12010
    },
    {
      "epoch": 0.8752320966978556,
      "grad_norm": 2.806118577635346,
      "learning_rate": 7.296889223915205e-07,
      "log_odds_chosen": 2.0082030296325684,
      "log_odds_ratio": -0.3276611268520355,
      "logits/chosen": -1.060937523841858,
      "logits/rejected": -0.915234386920929,
      "logps/chosen": -0.589062511920929,
      "logps/rejected": -2.012890577316284,
      "loss": 0.6341,
      "nll_loss": 0.5723632574081421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05892334133386612,
      "rewards/margins": 0.14218750596046448,
      "rewards/rejected": -0.20107421278953552,
      "step": 12020
    },
    {
      "epoch": 0.875960243200932,
      "grad_norm": 3.7028851722327816,
      "learning_rate": 7.293855804859192e-07,
      "log_odds_chosen": 2.202929735183716,
      "log_odds_ratio": -0.3057861328125,
      "logits/chosen": -0.998828113079071,
      "logits/rejected": -0.869921863079071,
      "logps/chosen": -0.6050781011581421,
      "logps/rejected": -2.17578125,
      "loss": 0.6343,
      "nll_loss": 0.5811523199081421,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06049804762005806,
      "rewards/margins": 0.15715332329273224,
      "rewards/rejected": -0.21770019829273224,
      "step": 12030
    },
    {
      "epoch": 0.8766883897040084,
      "grad_norm": 3.6914514108839063,
      "learning_rate": 7.290826165764892e-07,
      "log_odds_chosen": 1.7667968273162842,
      "log_odds_ratio": -0.34477537870407104,
      "logits/chosen": -0.9808593988418579,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.6041015386581421,
      "logps/rejected": -1.8214843273162842,
      "loss": 0.6512,
      "nll_loss": 0.65234375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06038818508386612,
      "rewards/margins": 0.12178955227136612,
      "rewards/rejected": -0.18203124403953552,
      "step": 12040
    },
    {
      "epoch": 0.8774165362070848,
      "grad_norm": 3.574001268284298,
      "learning_rate": 7.28780029878843e-07,
      "log_odds_chosen": 2.112109422683716,
      "log_odds_ratio": -0.2900390625,
      "logits/chosen": -1.035742163658142,
      "logits/rejected": -0.927539050579071,
      "logps/chosen": -0.5966796875,
      "logps/rejected": -2.092968702316284,
      "loss": 0.6324,
      "nll_loss": 0.6181640625,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.0596923828125,
      "rewards/margins": 0.14956054091453552,
      "rewards/rejected": -0.20913085341453552,
      "step": 12050
    },
    {
      "epoch": 0.8781446827101613,
      "grad_norm": 3.2378806815823333,
      "learning_rate": 7.284778196108706e-07,
      "log_odds_chosen": 1.9220702648162842,
      "log_odds_ratio": -0.3167480528354645,
      "logits/chosen": -1.026953101158142,
      "logits/rejected": -0.9175781011581421,
      "logps/chosen": -0.6181640625,
      "logps/rejected": -1.946874976158142,
      "loss": 0.6589,
      "nll_loss": 0.5692383050918579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.061767578125,
      "rewards/margins": 0.13294677436351776,
      "rewards/rejected": -0.19472655653953552,
      "step": 12060
    },
    {
      "epoch": 0.8788728292132377,
      "grad_norm": 3.301273916031814,
      "learning_rate": 7.281759849927299e-07,
      "log_odds_chosen": 1.7568359375,
      "log_odds_ratio": -0.36528319120407104,
      "logits/chosen": -1.019921898841858,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.6366211175918579,
      "logps/rejected": -1.853906273841858,
      "loss": 0.6399,
      "nll_loss": 0.5977538824081421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06365966796875,
      "rewards/margins": 0.12153320014476776,
      "rewards/rejected": -0.18540039658546448,
      "step": 12070
    },
    {
      "epoch": 0.8796009757163141,
      "grad_norm": 3.5796654628154294,
      "learning_rate": 7.278745252468389e-07,
      "log_odds_chosen": 1.6652343273162842,
      "log_odds_ratio": -0.3746581971645355,
      "logits/chosen": -0.9712890386581421,
      "logits/rejected": -0.8609374761581421,
      "logps/chosen": -0.6275390386581421,
      "logps/rejected": -1.7685546875,
      "loss": 0.6378,
      "nll_loss": 0.6182616949081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06278076022863388,
      "rewards/margins": 0.11409302055835724,
      "rewards/rejected": -0.17685547471046448,
      "step": 12080
    },
    {
      "epoch": 0.8803291222193905,
      "grad_norm": 3.574152237658407,
      "learning_rate": 7.275734395978672e-07,
      "log_odds_chosen": 1.970117211341858,
      "log_odds_ratio": -0.32758790254592896,
      "logits/chosen": -1.0603516101837158,
      "logits/rejected": -0.900390625,
      "logps/chosen": -0.6285156011581421,
      "logps/rejected": -2.0218749046325684,
      "loss": 0.641,
      "nll_loss": 0.626171886920929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06289062649011612,
      "rewards/margins": 0.13933105766773224,
      "rewards/rejected": -0.20205077528953552,
      "step": 12090
    },
    {
      "epoch": 0.8810572687224669,
      "grad_norm": 2.7280124914794652,
      "learning_rate": 7.272727272727272e-07,
      "log_odds_chosen": 1.907128930091858,
      "log_odds_ratio": -0.35478514432907104,
      "logits/chosen": -1.018945336341858,
      "logits/rejected": -0.897265613079071,
      "logps/chosen": -0.626660168170929,
      "logps/rejected": -1.9816405773162842,
      "loss": 0.6266,
      "nll_loss": 0.632519543170929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.062744140625,
      "rewards/margins": 0.13532714545726776,
      "rewards/rejected": -0.19807128608226776,
      "step": 12100
    },
    {
      "epoch": 0.8817854152255434,
      "grad_norm": 3.516265717250135,
      "learning_rate": 7.269723875005668e-07,
      "log_odds_chosen": 2.129101514816284,
      "log_odds_ratio": -0.2860107421875,
      "logits/chosen": -1.061914086341858,
      "logits/rejected": -0.908398449420929,
      "logps/chosen": -0.6015625,
      "logps/rejected": -2.1283202171325684,
      "loss": 0.6609,
      "nll_loss": 0.6187499761581421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.0601806640625,
      "rewards/margins": 0.15275268256664276,
      "rewards/rejected": -0.21281738579273224,
      "step": 12110
    },
    {
      "epoch": 0.8825135617286198,
      "grad_norm": 3.7088353316971574,
      "learning_rate": 7.266724195127595e-07,
      "log_odds_chosen": 1.513085961341858,
      "log_odds_ratio": -0.4559082090854645,
      "logits/chosen": -0.9964843988418579,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.6532226800918579,
      "logps/rejected": -1.73046875,
      "loss": 0.6448,
      "nll_loss": 0.6514648199081421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06533203274011612,
      "rewards/margins": 0.10780028998851776,
      "rewards/rejected": -0.17307129502296448,
      "step": 12120
    },
    {
      "epoch": 0.8832417082316962,
      "grad_norm": 2.824864516703077,
      "learning_rate": 7.26372822542898e-07,
      "log_odds_chosen": 2.1568360328674316,
      "log_odds_ratio": -0.28327637910842896,
      "logits/chosen": -1.028710961341858,
      "logits/rejected": -0.8505859375,
      "logps/chosen": -0.6015625,
      "logps/rejected": -2.181640625,
      "loss": 0.6426,
      "nll_loss": 0.628222644329071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06010742112994194,
      "rewards/margins": 0.157958984375,
      "rewards/rejected": -0.2181396484375,
      "step": 12130
    },
    {
      "epoch": 0.8839698547347726,
      "grad_norm": 2.796090240363867,
      "learning_rate": 7.260735958267845e-07,
      "log_odds_chosen": 1.899023413658142,
      "log_odds_ratio": -0.3453613221645355,
      "logits/chosen": -1.0001952648162842,
      "logits/rejected": -0.908203125,
      "logps/chosen": -0.59619140625,
      "logps/rejected": -1.924218773841858,
      "loss": 0.6354,
      "nll_loss": 0.6475585699081421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05958252027630806,
      "rewards/margins": 0.13283690810203552,
      "rewards/rejected": -0.192626953125,
      "step": 12140
    },
    {
      "epoch": 0.884698001237849,
      "grad_norm": 5.079236040130026,
      "learning_rate": 7.257747386024231e-07,
      "log_odds_chosen": 1.5419921875,
      "log_odds_ratio": -0.4166503846645355,
      "logits/chosen": -1.033203125,
      "logits/rejected": -0.923632800579071,
      "logps/chosen": -0.62939453125,
      "logps/rejected": -1.678125023841858,
      "loss": 0.6595,
      "nll_loss": 0.5716797113418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06292724609375,
      "rewards/margins": 0.10493163764476776,
      "rewards/rejected": -0.16787108778953552,
      "step": 12150
    },
    {
      "epoch": 0.8854261477409254,
      "grad_norm": 3.526003054058405,
      "learning_rate": 7.254762501100117e-07,
      "log_odds_chosen": 1.8488280773162842,
      "log_odds_ratio": -0.32941895723342896,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.9068359136581421,
      "logps/chosen": -0.601855456829071,
      "logps/rejected": -1.882421851158142,
      "loss": 0.6369,
      "nll_loss": 0.6634765863418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06022949144244194,
      "rewards/margins": 0.12803344428539276,
      "rewards/rejected": -0.1881103515625,
      "step": 12160
    },
    {
      "epoch": 0.8861542942440019,
      "grad_norm": 3.2069608920770096,
      "learning_rate": 7.251781295919335e-07,
      "log_odds_chosen": 1.730126976966858,
      "log_odds_ratio": -0.37104493379592896,
      "logits/chosen": -0.9996093511581421,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.6890624761581421,
      "logps/rejected": -1.910546898841858,
      "loss": 0.6398,
      "nll_loss": 0.6703125238418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06892089545726776,
      "rewards/margins": 0.12211303412914276,
      "rewards/rejected": -0.19099120795726776,
      "step": 12170
    },
    {
      "epoch": 0.8868824407470783,
      "grad_norm": 3.6050662719276447,
      "learning_rate": 7.248803762927498e-07,
      "log_odds_chosen": 1.533105492591858,
      "log_odds_ratio": -0.3828125,
      "logits/chosen": -1.039648413658142,
      "logits/rejected": -0.938671886920929,
      "logps/chosen": -0.638476550579071,
      "logps/rejected": -1.6396484375,
      "loss": 0.659,
      "nll_loss": 0.6498047113418579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06385497748851776,
      "rewards/margins": 0.10006103664636612,
      "rewards/rejected": -0.1639404296875,
      "step": 12180
    },
    {
      "epoch": 0.8876105872501547,
      "grad_norm": 2.968252320368087,
      "learning_rate": 7.245829894591907e-07,
      "log_odds_chosen": 1.7236328125,
      "log_odds_ratio": -0.33671873807907104,
      "logits/chosen": -1.016210913658142,
      "logits/rejected": -0.9117187261581421,
      "logps/chosen": -0.570019543170929,
      "logps/rejected": -1.6896483898162842,
      "loss": 0.6383,
      "nll_loss": 0.5570312738418579,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.05701904371380806,
      "rewards/margins": 0.11204223334789276,
      "rewards/rejected": -0.16896972060203552,
      "step": 12190
    },
    {
      "epoch": 0.8883387337532311,
      "grad_norm": 3.2693250245874155,
      "learning_rate": 7.242859683401482e-07,
      "log_odds_chosen": 1.801171898841858,
      "log_odds_ratio": -0.3141113221645355,
      "logits/chosen": -1.052343726158142,
      "logits/rejected": -0.919921875,
      "logps/chosen": -0.5873047113418579,
      "logps/rejected": -1.837890625,
      "loss": 0.6281,
      "nll_loss": 0.5703125,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05875243991613388,
      "rewards/margins": 0.12495116889476776,
      "rewards/rejected": -0.183837890625,
      "step": 12200
    },
    {
      "epoch": 0.8890668802563075,
      "grad_norm": 3.0199270163322964,
      "learning_rate": 7.239893121866677e-07,
      "log_odds_chosen": 1.8811523914337158,
      "log_odds_ratio": -0.30146485567092896,
      "logits/chosen": -1.04296875,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.5757812261581421,
      "logps/rejected": -1.8449218273162842,
      "loss": 0.6422,
      "nll_loss": 0.573925793170929,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05759887769818306,
      "rewards/margins": 0.12694701552391052,
      "rewards/rejected": -0.18452148139476776,
      "step": 12210
    },
    {
      "epoch": 0.8897950267593839,
      "grad_norm": 2.935543434286194,
      "learning_rate": 7.236930202519399e-07,
      "log_odds_chosen": 1.788671851158142,
      "log_odds_ratio": -0.31391602754592896,
      "logits/chosen": -1.0701172351837158,
      "logits/rejected": -0.9300781488418579,
      "logps/chosen": -0.5401366949081421,
      "logps/rejected": -1.7195312976837158,
      "loss": 0.6133,
      "nll_loss": 0.546679675579071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05406494066119194,
      "rewards/margins": 0.11790771782398224,
      "rewards/rejected": -0.17202147841453552,
      "step": 12220
    },
    {
      "epoch": 0.8905231732624604,
      "grad_norm": 2.8150855012626286,
      "learning_rate": 7.233970917912936e-07,
      "log_odds_chosen": 1.9132812023162842,
      "log_odds_ratio": -0.3216308653354645,
      "logits/chosen": -1.0134766101837158,
      "logits/rejected": -0.8804687261581421,
      "logps/chosen": -0.5669921636581421,
      "logps/rejected": -1.892187476158142,
      "loss": 0.6188,
      "nll_loss": 0.529589831829071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05672607570886612,
      "rewards/margins": 0.1324462890625,
      "rewards/rejected": -0.189208984375,
      "step": 12230
    },
    {
      "epoch": 0.8912513197655368,
      "grad_norm": 2.7793258718983385,
      "learning_rate": 7.231015260621871e-07,
      "log_odds_chosen": 1.823632836341858,
      "log_odds_ratio": -0.32499998807907104,
      "logits/chosen": -1.0392577648162842,
      "logits/rejected": -0.928906261920929,
      "logps/chosen": -0.597851574420929,
      "logps/rejected": -1.840234398841858,
      "loss": 0.6283,
      "nll_loss": 0.5970703363418579,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05979003757238388,
      "rewards/margins": 0.12425537407398224,
      "rewards/rejected": -0.18403320014476776,
      "step": 12240
    },
    {
      "epoch": 0.8919794662686132,
      "grad_norm": 2.9303907803726394,
      "learning_rate": 7.228063223242011e-07,
      "log_odds_chosen": 1.740234375,
      "log_odds_ratio": -0.343994140625,
      "logits/chosen": -1.030859351158142,
      "logits/rejected": -0.916015625,
      "logps/chosen": -0.6358398199081421,
      "logps/rejected": -1.8406250476837158,
      "loss": 0.6367,
      "nll_loss": 0.612597644329071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06356201320886612,
      "rewards/margins": 0.12045898288488388,
      "rewards/rejected": -0.1839599609375,
      "step": 12250
    },
    {
      "epoch": 0.8927076127716896,
      "grad_norm": 3.950354819112056,
      "learning_rate": 7.225114798390295e-07,
      "log_odds_chosen": 2.011914014816284,
      "log_odds_ratio": -0.3099121153354645,
      "logits/chosen": -1.034765601158142,
      "logits/rejected": -0.91796875,
      "logps/chosen": -0.6004883050918579,
      "logps/rejected": -1.9855468273162842,
      "loss": 0.6234,
      "nll_loss": 0.599316418170929,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06002197414636612,
      "rewards/margins": 0.13872070610523224,
      "rewards/rejected": -0.19855956733226776,
      "step": 12260
    },
    {
      "epoch": 0.893435759274766,
      "grad_norm": 2.6352830066148614,
      "learning_rate": 7.222169978704737e-07,
      "log_odds_chosen": 1.640234351158142,
      "log_odds_ratio": -0.3760742247104645,
      "logits/chosen": -0.9916015863418579,
      "logits/rejected": -0.8783203363418579,
      "logps/chosen": -0.6146484613418579,
      "logps/rejected": -1.7578125,
      "loss": 0.6123,
      "nll_loss": 0.61669921875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06146240234375,
      "rewards/margins": 0.11429443210363388,
      "rewards/rejected": -0.17578125,
      "step": 12270
    },
    {
      "epoch": 0.8941639057778425,
      "grad_norm": 3.401039206144832,
      "learning_rate": 7.219228756844335e-07,
      "log_odds_chosen": 1.7668945789337158,
      "log_odds_ratio": -0.3857421875,
      "logits/chosen": -1.020898461341858,
      "logits/rejected": -0.8955078125,
      "logps/chosen": -0.5591796636581421,
      "logps/rejected": -1.772851586341858,
      "loss": 0.6355,
      "nll_loss": 0.559863269329071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.05587158352136612,
      "rewards/margins": 0.12137451022863388,
      "rewards/rejected": -0.17722168564796448,
      "step": 12280
    },
    {
      "epoch": 0.894892052280919,
      "grad_norm": 2.9959571701694188,
      "learning_rate": 7.216291125488994e-07,
      "log_odds_chosen": 1.7834961414337158,
      "log_odds_ratio": -0.3232665956020355,
      "logits/chosen": -0.990039050579071,
      "logits/rejected": -0.903515636920929,
      "logps/chosen": -0.60888671875,
      "logps/rejected": -1.8152344226837158,
      "loss": 0.6393,
      "nll_loss": 0.599316418170929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06087646633386612,
      "rewards/margins": 0.120574951171875,
      "rewards/rejected": -0.181396484375,
      "step": 12290
    },
    {
      "epoch": 0.8956201987839953,
      "grad_norm": 3.267061498901829,
      "learning_rate": 7.213357077339458e-07,
      "log_odds_chosen": 1.7607421875,
      "log_odds_ratio": -0.3316406309604645,
      "logits/chosen": -0.9994140863418579,
      "logits/rejected": -0.84375,
      "logps/chosen": -0.5838867425918579,
      "logps/rejected": -1.763085961341858,
      "loss": 0.6021,
      "nll_loss": 0.571582019329071,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.05836181715130806,
      "rewards/margins": 0.11790771782398224,
      "rewards/rejected": -0.17612305283546448,
      "step": 12300
    },
    {
      "epoch": 0.8963483452870717,
      "grad_norm": 3.322656779124618,
      "learning_rate": 7.210426605117224e-07,
      "log_odds_chosen": 2.4248046875,
      "log_odds_ratio": -0.25682371854782104,
      "logits/chosen": -1.021093726158142,
      "logits/rejected": -0.893359363079071,
      "logps/chosen": -0.5799804925918579,
      "logps/rejected": -2.338671922683716,
      "loss": 0.6264,
      "nll_loss": 0.58349609375,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.05802001804113388,
      "rewards/margins": 0.17581787705421448,
      "rewards/rejected": -0.2337646484375,
      "step": 12310
    },
    {
      "epoch": 0.8970764917901481,
      "grad_norm": 3.9152405389570375,
      "learning_rate": 7.207499701564471e-07,
      "log_odds_chosen": 2.1566405296325684,
      "log_odds_ratio": -0.2997192442417145,
      "logits/chosen": -1.0115234851837158,
      "logits/rejected": -0.880664050579071,
      "logps/chosen": -0.5975586175918579,
      "logps/rejected": -2.1246094703674316,
      "loss": 0.6422,
      "nll_loss": 0.613964855670929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05975341796875,
      "rewards/margins": 0.15260009467601776,
      "rewards/rejected": -0.21225586533546448,
      "step": 12320
    },
    {
      "epoch": 0.8978046382932245,
      "grad_norm": 3.406141605590738,
      "learning_rate": 7.204576359443989e-07,
      "log_odds_chosen": 1.836035132408142,
      "log_odds_ratio": -0.3304199278354645,
      "logits/chosen": -0.9935547113418579,
      "logits/rejected": -0.8580077886581421,
      "logps/chosen": -0.5859375,
      "logps/rejected": -1.880859375,
      "loss": 0.6357,
      "nll_loss": 0.579882800579071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05855713039636612,
      "rewards/margins": 0.129444882273674,
      "rewards/rejected": -0.18803711235523224,
      "step": 12330
    },
    {
      "epoch": 0.898532784796301,
      "grad_norm": 3.128069988641081,
      "learning_rate": 7.201656571539094e-07,
      "log_odds_chosen": 1.783593773841858,
      "log_odds_ratio": -0.32695311307907104,
      "logits/chosen": -1.0310547351837158,
      "logits/rejected": -0.9056640863418579,
      "logps/chosen": -0.57666015625,
      "logps/rejected": -1.8113281726837158,
      "loss": 0.6319,
      "nll_loss": 0.5938476324081421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05769043043255806,
      "rewards/margins": 0.12327881157398224,
      "rewards/rejected": -0.18088379502296448,
      "step": 12340
    },
    {
      "epoch": 0.8992609312993775,
      "grad_norm": 4.731959285602967,
      "learning_rate": 7.19874033065356e-07,
      "log_odds_chosen": 2.234375,
      "log_odds_ratio": -0.28459471464157104,
      "logits/chosen": -0.9755859375,
      "logits/rejected": -0.854687511920929,
      "logps/chosen": -0.59375,
      "logps/rejected": -2.212890625,
      "loss": 0.6333,
      "nll_loss": 0.626171886920929,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.05937499925494194,
      "rewards/margins": 0.1619873046875,
      "rewards/rejected": -0.22119140625,
      "step": 12350
    },
    {
      "epoch": 0.8999890778024539,
      "grad_norm": 3.1002011068578987,
      "learning_rate": 7.195827629611545e-07,
      "log_odds_chosen": 1.938085913658142,
      "log_odds_ratio": -0.3370605409145355,
      "logits/chosen": -1.0363280773162842,
      "logits/rejected": -0.9144531488418579,
      "logps/chosen": -0.6153320074081421,
      "logps/rejected": -2.005859375,
      "loss": 0.6245,
      "nll_loss": 0.5845702886581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06157226487994194,
      "rewards/margins": 0.13901367783546448,
      "rewards/rejected": -0.20046386122703552,
      "step": 12360
    },
    {
      "epoch": 0.9007172243055303,
      "grad_norm": 3.3817184831403586,
      "learning_rate": 7.19291846125751e-07,
      "log_odds_chosen": 1.9873046875,
      "log_odds_ratio": -0.315185546875,
      "logits/chosen": -1.03515625,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.59521484375,
      "logps/rejected": -2.0257811546325684,
      "loss": 0.6302,
      "nll_loss": 0.531054675579071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05955810472369194,
      "rewards/margins": 0.14309081435203552,
      "rewards/rejected": -0.20244140923023224,
      "step": 12370
    },
    {
      "epoch": 0.9014453708086066,
      "grad_norm": 2.6573564273350754,
      "learning_rate": 7.190012818456154e-07,
      "log_odds_chosen": 1.6046874523162842,
      "log_odds_ratio": -0.3658691346645355,
      "logits/chosen": -1.0304687023162842,
      "logits/rejected": -0.9267578125,
      "logps/chosen": -0.631640613079071,
      "logps/rejected": -1.7443358898162842,
      "loss": 0.6189,
      "nll_loss": 0.583203136920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06317138671875,
      "rewards/margins": 0.111236572265625,
      "rewards/rejected": -0.17438964545726776,
      "step": 12380
    },
    {
      "epoch": 0.9021735173116832,
      "grad_norm": 3.705840075615822,
      "learning_rate": 7.187110694092334e-07,
      "log_odds_chosen": 1.32666015625,
      "log_odds_ratio": -0.40498048067092896,
      "logits/chosen": -1.0275390148162842,
      "logits/rejected": -0.928515613079071,
      "logps/chosen": -0.605761706829071,
      "logps/rejected": -1.442968726158142,
      "loss": 0.6068,
      "nll_loss": 0.5902343988418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06055908277630806,
      "rewards/margins": 0.08365478366613388,
      "rewards/rejected": -0.14414063096046448,
      "step": 12390
    },
    {
      "epoch": 0.9029016638147596,
      "grad_norm": 3.5342324199017345,
      "learning_rate": 7.184212081070996e-07,
      "log_odds_chosen": 1.7727539539337158,
      "log_odds_ratio": -0.3647705018520355,
      "logits/chosen": -1.0066406726837158,
      "logits/rejected": -0.921093761920929,
      "logps/chosen": -0.5673828125,
      "logps/rejected": -1.8341796398162842,
      "loss": 0.6168,
      "nll_loss": 0.5638672113418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.0567626953125,
      "rewards/margins": 0.12658080458641052,
      "rewards/rejected": -0.18352051079273224,
      "step": 12400
    },
    {
      "epoch": 0.903629810317836,
      "grad_norm": 3.50372160457232,
      "learning_rate": 7.181316972317097e-07,
      "log_odds_chosen": 1.7201659679412842,
      "log_odds_ratio": -0.37299805879592896,
      "logits/chosen": -0.989453136920929,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.61376953125,
      "logps/rejected": -1.8009765148162842,
      "loss": 0.6624,
      "nll_loss": 0.618945300579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06141357496380806,
      "rewards/margins": 0.11880187690258026,
      "rewards/rejected": -0.18022461235523224,
      "step": 12410
    },
    {
      "epoch": 0.9043579568209124,
      "grad_norm": 4.171829104032437,
      "learning_rate": 7.17842536077554e-07,
      "log_odds_chosen": 1.558019995689392,
      "log_odds_ratio": -0.42216795682907104,
      "logits/chosen": -1.029882788658142,
      "logits/rejected": -0.9349609613418579,
      "logps/chosen": -0.629687488079071,
      "logps/rejected": -1.7109375,
      "loss": 0.6233,
      "nll_loss": 0.546191394329071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06293945014476776,
      "rewards/margins": 0.10826416313648224,
      "rewards/rejected": -0.1712646484375,
      "step": 12420
    },
    {
      "epoch": 0.9050861033239888,
      "grad_norm": 2.9162013975384937,
      "learning_rate": 7.175537239411094e-07,
      "log_odds_chosen": 2.01904296875,
      "log_odds_ratio": -0.31730955839157104,
      "logits/chosen": -1.0574219226837158,
      "logits/rejected": -0.924609363079071,
      "logps/chosen": -0.6283203363418579,
      "logps/rejected": -2.1109375953674316,
      "loss": 0.6495,
      "nll_loss": 0.5689941644668579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06290283054113388,
      "rewards/margins": 0.14827880263328552,
      "rewards/rejected": -0.21115723252296448,
      "step": 12430
    },
    {
      "epoch": 0.9058142498270652,
      "grad_norm": 3.1828345705058907,
      "learning_rate": 7.172652601208325e-07,
      "log_odds_chosen": 1.915429711341858,
      "log_odds_ratio": -0.33497315645217896,
      "logits/chosen": -1.0255858898162842,
      "logits/rejected": -0.9302734136581421,
      "logps/chosen": -0.642773449420929,
      "logps/rejected": -2.0064454078674316,
      "loss": 0.6517,
      "nll_loss": 0.652636706829071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06424560397863388,
      "rewards/margins": 0.1363525390625,
      "rewards/rejected": -0.20058593153953552,
      "step": 12440
    },
    {
      "epoch": 0.9065423963301417,
      "grad_norm": 2.7824808328019603,
      "learning_rate": 7.169771439171534e-07,
      "log_odds_chosen": 1.769921898841858,
      "log_odds_ratio": -0.3915039002895355,
      "logits/chosen": -1.0050780773162842,
      "logits/rejected": -0.9166015386581421,
      "logps/chosen": -0.6786133050918579,
      "logps/rejected": -1.9382812976837158,
      "loss": 0.6442,
      "nll_loss": 0.69384765625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06790771335363388,
      "rewards/margins": 0.12600097060203552,
      "rewards/rejected": -0.19394531846046448,
      "step": 12450
    },
    {
      "epoch": 0.9072705428332181,
      "grad_norm": 3.215576603806281,
      "learning_rate": 7.166893746324661e-07,
      "log_odds_chosen": 1.938867211341858,
      "log_odds_ratio": -0.3123779296875,
      "logits/chosen": -1.045507788658142,
      "logits/rejected": -0.9453125,
      "logps/chosen": -0.5953124761581421,
      "logps/rejected": -1.929296851158142,
      "loss": 0.6244,
      "nll_loss": 0.5560547113418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05949706956744194,
      "rewards/margins": 0.13334961235523224,
      "rewards/rejected": -0.19284668564796448,
      "step": 12460
    },
    {
      "epoch": 0.9079986893362945,
      "grad_norm": 2.7832101549716017,
      "learning_rate": 7.164019515711245e-07,
      "log_odds_chosen": 1.9480469226837158,
      "log_odds_ratio": -0.2992187440395355,
      "logits/chosen": -1.030664086341858,
      "logits/rejected": -0.863085925579071,
      "logps/chosen": -0.5499023199081421,
      "logps/rejected": -1.8923828601837158,
      "loss": 0.6289,
      "nll_loss": 0.599609375,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05499267578125,
      "rewards/margins": 0.13430175185203552,
      "rewards/rejected": -0.18930664658546448,
      "step": 12470
    },
    {
      "epoch": 0.9087268358393709,
      "grad_norm": 3.337554185786786,
      "learning_rate": 7.161148740394328e-07,
      "log_odds_chosen": 2.149218797683716,
      "log_odds_ratio": -0.2632812559604645,
      "logits/chosen": -1.045507788658142,
      "logits/rejected": -0.896484375,
      "logps/chosen": -0.583789050579071,
      "logps/rejected": -2.0863280296325684,
      "loss": 0.6016,
      "nll_loss": 0.5546875,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.05836181715130806,
      "rewards/margins": 0.1500244140625,
      "rewards/rejected": -0.20834961533546448,
      "step": 12480
    },
    {
      "epoch": 0.9094549823424473,
      "grad_norm": 2.866681952771173,
      "learning_rate": 7.158281413456402e-07,
      "log_odds_chosen": 2.020312547683716,
      "log_odds_ratio": -0.2879394590854645,
      "logits/chosen": -1.0349609851837158,
      "logits/rejected": -0.916210949420929,
      "logps/chosen": -0.5826171636581421,
      "logps/rejected": -1.93359375,
      "loss": 0.6379,
      "nll_loss": 0.60888671875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05826415866613388,
      "rewards/margins": 0.135101318359375,
      "rewards/rejected": -0.19338378310203552,
      "step": 12490
    },
    {
      "epoch": 0.9101831288455238,
      "grad_norm": 2.8885579128228143,
      "learning_rate": 7.155417527999326e-07,
      "log_odds_chosen": 2.0252928733825684,
      "log_odds_ratio": -0.33574217557907104,
      "logits/chosen": -0.9916015863418579,
      "logits/rejected": -0.852343738079071,
      "logps/chosen": -0.5992187261581421,
      "logps/rejected": -2.033984422683716,
      "loss": 0.6329,
      "nll_loss": 0.581835925579071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05989990383386612,
      "rewards/margins": 0.14330749213695526,
      "rewards/rejected": -0.20334473252296448,
      "step": 12500
    },
    {
      "epoch": 0.9109112753486002,
      "grad_norm": 3.0192660576072265,
      "learning_rate": 7.152557077144268e-07,
      "log_odds_chosen": 1.7451171875,
      "log_odds_ratio": -0.35688477754592896,
      "logits/chosen": -1.012109398841858,
      "logits/rejected": -0.8941406011581421,
      "logps/chosen": -0.64794921875,
      "logps/rejected": -1.892968773841858,
      "loss": 0.6236,
      "nll_loss": 0.614453136920929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06475830078125,
      "rewards/margins": 0.12467040866613388,
      "rewards/rejected": -0.18942871689796448,
      "step": 12510
    },
    {
      "epoch": 0.9116394218516766,
      "grad_norm": 3.092391743346633,
      "learning_rate": 7.149700054031623e-07,
      "log_odds_chosen": 1.9132812023162842,
      "log_odds_ratio": -0.3224121034145355,
      "logits/chosen": -1.0353515148162842,
      "logits/rejected": -0.9173828363418579,
      "logps/chosen": -0.6229492425918579,
      "logps/rejected": -1.989843726158142,
      "loss": 0.6234,
      "nll_loss": 0.589550793170929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06223144382238388,
      "rewards/margins": 0.13681641221046448,
      "rewards/rejected": -0.19902344048023224,
      "step": 12520
    },
    {
      "epoch": 0.912367568354753,
      "grad_norm": 3.1898174549187406,
      "learning_rate": 7.146846451820958e-07,
      "log_odds_chosen": 2.22265625,
      "log_odds_ratio": -0.2695068418979645,
      "logits/chosen": -1.061914086341858,
      "logits/rejected": -0.943554699420929,
      "logps/chosen": -0.593457043170929,
      "logps/rejected": -2.155468702316284,
      "loss": 0.6152,
      "nll_loss": 0.546191394329071,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.05933837965130806,
      "rewards/margins": 0.15607909858226776,
      "rewards/rejected": -0.21564941108226776,
      "step": 12530
    },
    {
      "epoch": 0.9130957148578294,
      "grad_norm": 6.107473602366688,
      "learning_rate": 7.14399626369093e-07,
      "log_odds_chosen": 1.8357422351837158,
      "log_odds_ratio": -0.34130859375,
      "logits/chosen": -1.0480468273162842,
      "logits/rejected": -0.923632800579071,
      "logps/chosen": -0.61962890625,
      "logps/rejected": -1.9191405773162842,
      "loss": 0.6205,
      "nll_loss": 0.609570324420929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06199951097369194,
      "rewards/margins": 0.13009032607078552,
      "rewards/rejected": -0.19196777045726776,
      "step": 12540
    },
    {
      "epoch": 0.9138238613609058,
      "grad_norm": 3.2824399161109357,
      "learning_rate": 7.141149482839228e-07,
      "log_odds_chosen": 1.795507788658142,
      "log_odds_ratio": -0.3729003965854645,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.8960937261581421,
      "logps/chosen": -0.6265624761581421,
      "logps/rejected": -1.9167969226837158,
      "loss": 0.6277,
      "nll_loss": 0.5787109136581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06267090141773224,
      "rewards/margins": 0.1290283203125,
      "rewards/rejected": -0.19167479872703552,
      "step": 12550
    },
    {
      "epoch": 0.9145520078639823,
      "grad_norm": 3.7489096027292943,
      "learning_rate": 7.138306102482496e-07,
      "log_odds_chosen": 1.7353515625,
      "log_odds_ratio": -0.353759765625,
      "logits/chosen": -1.0216796398162842,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.579394519329071,
      "logps/rejected": -1.787695288658142,
      "loss": 0.6398,
      "nll_loss": 0.609179675579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.05797119066119194,
      "rewards/margins": 0.12077637016773224,
      "rewards/rejected": -0.17866210639476776,
      "step": 12560
    },
    {
      "epoch": 0.9152801543670587,
      "grad_norm": 3.4648147987101034,
      "learning_rate": 7.135466115856274e-07,
      "log_odds_chosen": 1.7238280773162842,
      "log_odds_ratio": -0.32568359375,
      "logits/chosen": -1.024804711341858,
      "logits/rejected": -0.930859386920929,
      "logps/chosen": -0.5860351324081421,
      "logps/rejected": -1.750390648841858,
      "loss": 0.6497,
      "nll_loss": 0.6060546636581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05859375,
      "rewards/margins": 0.116455078125,
      "rewards/rejected": -0.17499999701976776,
      "step": 12570
    },
    {
      "epoch": 0.9160083008701351,
      "grad_norm": 3.2875945116325935,
      "learning_rate": 7.13262951621492e-07,
      "log_odds_chosen": 2.0667967796325684,
      "log_odds_ratio": -0.3015380799770355,
      "logits/chosen": -1.074609398841858,
      "logits/rejected": -0.9351562261581421,
      "logps/chosen": -0.5923827886581421,
      "logps/rejected": -2.046875,
      "loss": 0.6118,
      "nll_loss": 0.533496081829071,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.05925292894244194,
      "rewards/margins": 0.14533691108226776,
      "rewards/rejected": -0.20439453423023224,
      "step": 12580
    },
    {
      "epoch": 0.9167364473732115,
      "grad_norm": 5.035545395186312,
      "learning_rate": 7.129796296831554e-07,
      "log_odds_chosen": 1.7775390148162842,
      "log_odds_ratio": -0.34672850370407104,
      "logits/chosen": -1.0373046398162842,
      "logits/rejected": -0.9087890386581421,
      "logps/chosen": -0.5941406488418579,
      "logps/rejected": -1.8019530773162842,
      "loss": 0.632,
      "nll_loss": 0.604199230670929,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.05937499925494194,
      "rewards/margins": 0.12076415866613388,
      "rewards/rejected": -0.18010254204273224,
      "step": 12590
    },
    {
      "epoch": 0.9174645938762879,
      "grad_norm": 3.030934216136635,
      "learning_rate": 7.126966450997984e-07,
      "log_odds_chosen": 1.9826171398162842,
      "log_odds_ratio": -0.32429200410842896,
      "logits/chosen": -1.0380859375,
      "logits/rejected": -0.9009765386581421,
      "logps/chosen": -0.62548828125,
      "logps/rejected": -2.022265672683716,
      "loss": 0.6328,
      "nll_loss": 0.658398449420929,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06258545070886612,
      "rewards/margins": 0.13969726860523224,
      "rewards/rejected": -0.20224609971046448,
      "step": 12600
    },
    {
      "epoch": 0.9181927403793644,
      "grad_norm": 2.8646849346911782,
      "learning_rate": 7.124139972024637e-07,
      "log_odds_chosen": 2.0234375,
      "log_odds_ratio": -0.3267822265625,
      "logits/chosen": -1.0515625476837158,
      "logits/rejected": -0.897656261920929,
      "logps/chosen": -0.60888671875,
      "logps/rejected": -2.0732421875,
      "loss": 0.6284,
      "nll_loss": 0.5985351800918579,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06090087816119194,
      "rewards/margins": 0.14635619521141052,
      "rewards/rejected": -0.20737305283546448,
      "step": 12610
    },
    {
      "epoch": 0.9189208868824408,
      "grad_norm": 4.870855781497431,
      "learning_rate": 7.121316853240503e-07,
      "log_odds_chosen": 1.791406273841858,
      "log_odds_ratio": -0.3602539002895355,
      "logits/chosen": -0.9742187261581421,
      "logits/rejected": -0.88671875,
      "logps/chosen": -0.594433605670929,
      "logps/rejected": -1.805078148841858,
      "loss": 0.6444,
      "nll_loss": 0.6201171875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05942382663488388,
      "rewards/margins": 0.12113036960363388,
      "rewards/rejected": -0.18061523139476776,
      "step": 12620
    },
    {
      "epoch": 0.9196490333855172,
      "grad_norm": 3.735062330193224,
      "learning_rate": 7.118497087993057e-07,
      "log_odds_chosen": 2.2994141578674316,
      "log_odds_ratio": -0.27824705839157104,
      "logits/chosen": -1.029687523841858,
      "logits/rejected": -0.917187511920929,
      "logps/chosen": -0.5995117425918579,
      "logps/rejected": -2.24609375,
      "loss": 0.6194,
      "nll_loss": 0.622851550579071,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0599365234375,
      "rewards/margins": 0.16459961235523224,
      "rewards/rejected": -0.224609375,
      "step": 12630
    },
    {
      "epoch": 0.9203771798885936,
      "grad_norm": 2.8131241467144656,
      "learning_rate": 7.1156806696482e-07,
      "log_odds_chosen": 1.744531273841858,
      "log_odds_ratio": -0.37983399629592896,
      "logits/chosen": -0.991406261920929,
      "logits/rejected": -0.888476550579071,
      "logps/chosen": -0.611523449420929,
      "logps/rejected": -1.8601562976837158,
      "loss": 0.6407,
      "nll_loss": 0.6753906011581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06118164211511612,
      "rewards/margins": 0.12492676079273224,
      "rewards/rejected": -0.18608398735523224,
      "step": 12640
    },
    {
      "epoch": 0.92110532639167,
      "grad_norm": 3.597679873102685,
      "learning_rate": 7.112867591590192e-07,
      "log_odds_chosen": 1.809814453125,
      "log_odds_ratio": -0.3200927674770355,
      "logits/chosen": -1.0349609851837158,
      "logits/rejected": -0.920117199420929,
      "logps/chosen": -0.583691418170929,
      "logps/rejected": -1.7990233898162842,
      "loss": 0.6181,
      "nll_loss": 0.61083984375,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05836181715130806,
      "rewards/margins": 0.12155761569738388,
      "rewards/rejected": -0.17998047173023224,
      "step": 12650
    },
    {
      "epoch": 0.9218334728947464,
      "grad_norm": 3.1458014687211704,
      "learning_rate": 7.110057847221588e-07,
      "log_odds_chosen": 2.019335985183716,
      "log_odds_ratio": -0.31572264432907104,
      "logits/chosen": -1.0617187023162842,
      "logits/rejected": -0.9302734136581421,
      "logps/chosen": -0.567578136920929,
      "logps/rejected": -2.032421827316284,
      "loss": 0.6278,
      "nll_loss": 0.557421863079071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.0567626953125,
      "rewards/margins": 0.14641113579273224,
      "rewards/rejected": -0.203125,
      "step": 12660
    },
    {
      "epoch": 0.9225616193978229,
      "grad_norm": 2.732395549420326,
      "learning_rate": 7.107251429963166e-07,
      "log_odds_chosen": 2.2564454078674316,
      "log_odds_ratio": -0.25739747285842896,
      "logits/chosen": -1.046289086341858,
      "logits/rejected": -0.9095703363418579,
      "logps/chosen": -0.58740234375,
      "logps/rejected": -2.181640625,
      "loss": 0.6064,
      "nll_loss": 0.5650390386581421,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.05867920070886612,
      "rewards/margins": 0.15947265923023224,
      "rewards/rejected": -0.21818847954273224,
      "step": 12670
    },
    {
      "epoch": 0.9232897659008993,
      "grad_norm": 2.7177189205065346,
      "learning_rate": 7.104448333253878e-07,
      "log_odds_chosen": 1.8761718273162842,
      "log_odds_ratio": -0.3071044981479645,
      "logits/chosen": -1.069726586341858,
      "logits/rejected": -0.9310547113418579,
      "logps/chosen": -0.589062511920929,
      "logps/rejected": -1.9216797351837158,
      "loss": 0.6399,
      "nll_loss": 0.6011718511581421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.05893554538488388,
      "rewards/margins": 0.13328246772289276,
      "rewards/rejected": -0.1922607421875,
      "step": 12680
    },
    {
      "epoch": 0.9240179124039757,
      "grad_norm": 3.2336686227739557,
      "learning_rate": 7.101648550550766e-07,
      "log_odds_chosen": 1.84130859375,
      "log_odds_ratio": -0.3536621034145355,
      "logits/chosen": -0.991992175579071,
      "logits/rejected": -0.847851574420929,
      "logps/chosen": -0.61669921875,
      "logps/rejected": -1.954687476158142,
      "loss": 0.6264,
      "nll_loss": 0.631054699420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06166992336511612,
      "rewards/margins": 0.13383789360523224,
      "rewards/rejected": -0.19553223252296448,
      "step": 12690
    },
    {
      "epoch": 0.9247460589070521,
      "grad_norm": 4.219029937193473,
      "learning_rate": 7.098852075328911e-07,
      "log_odds_chosen": 1.788476586341858,
      "log_odds_ratio": -0.35107421875,
      "logits/chosen": -1.0359375476837158,
      "logits/rejected": -0.9273437261581421,
      "logps/chosen": -0.6236327886581421,
      "logps/rejected": -1.8367187976837158,
      "loss": 0.6319,
      "nll_loss": 0.5845702886581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06234131008386612,
      "rewards/margins": 0.121337890625,
      "rewards/rejected": -0.18369141221046448,
      "step": 12700
    },
    {
      "epoch": 0.9254742054101285,
      "grad_norm": 2.8168536148752237,
      "learning_rate": 7.096058901081364e-07,
      "log_odds_chosen": 1.823828101158142,
      "log_odds_ratio": -0.30366212129592896,
      "logits/chosen": -1.100000023841858,
      "logits/rejected": -0.951367199420929,
      "logps/chosen": -0.568554699420929,
      "logps/rejected": -1.807226538658142,
      "loss": 0.6167,
      "nll_loss": 0.537109375,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.05693359300494194,
      "rewards/margins": 0.12379150092601776,
      "rewards/rejected": -0.18068847060203552,
      "step": 12710
    },
    {
      "epoch": 0.926202351913205,
      "grad_norm": 4.401622583401404,
      "learning_rate": 7.093269021319087e-07,
      "log_odds_chosen": 1.6384766101837158,
      "log_odds_ratio": -0.3821777403354645,
      "logits/chosen": -1.070703148841858,
      "logits/rejected": -0.944531261920929,
      "logps/chosen": -0.6592773199081421,
      "logps/rejected": -1.7785155773162842,
      "loss": 0.6205,
      "nll_loss": 0.610546886920929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06591796875,
      "rewards/margins": 0.11199951171875,
      "rewards/rejected": -0.17795410752296448,
      "step": 12720
    },
    {
      "epoch": 0.9269304984162814,
      "grad_norm": 3.164911521531916,
      "learning_rate": 7.090482429570884e-07,
      "log_odds_chosen": 1.594335913658142,
      "log_odds_ratio": -0.3465332090854645,
      "logits/chosen": -1.002539038658142,
      "logits/rejected": -0.890625,
      "logps/chosen": -0.5858398675918579,
      "logps/rejected": -1.6433594226837158,
      "loss": 0.6374,
      "nll_loss": 0.579882800579071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05858154222369194,
      "rewards/margins": 0.10581054538488388,
      "rewards/rejected": -0.1644287109375,
      "step": 12730
    },
    {
      "epoch": 0.9276586449193578,
      "grad_norm": 2.9398646227652003,
      "learning_rate": 7.087699119383339e-07,
      "log_odds_chosen": 1.8564453125,
      "log_odds_ratio": -0.3157958984375,
      "logits/chosen": -1.036718726158142,
      "logits/rejected": -0.896679699420929,
      "logps/chosen": -0.606249988079071,
      "logps/rejected": -1.861718773841858,
      "loss": 0.6295,
      "nll_loss": 0.57080078125,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06055908277630806,
      "rewards/margins": 0.12548828125,
      "rewards/rejected": -0.18618163466453552,
      "step": 12740
    },
    {
      "epoch": 0.9283867914224342,
      "grad_norm": 3.777156235910574,
      "learning_rate": 7.084919084320762e-07,
      "log_odds_chosen": 2.0484375953674316,
      "log_odds_ratio": -0.3025878965854645,
      "logits/chosen": -1.0498046875,
      "logits/rejected": -0.8968750238418579,
      "logps/chosen": -0.619140625,
      "logps/rejected": -2.07421875,
      "loss": 0.645,
      "nll_loss": 0.5753418207168579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06195068359375,
      "rewards/margins": 0.14545898139476776,
      "rewards/rejected": -0.20742186903953552,
      "step": 12750
    },
    {
      "epoch": 0.9291149379255106,
      "grad_norm": 2.921323171107468,
      "learning_rate": 7.08214231796511e-07,
      "log_odds_chosen": 2.0921874046325684,
      "log_odds_ratio": -0.2861328125,
      "logits/chosen": -1.002539038658142,
      "logits/rejected": -0.913281261920929,
      "logps/chosen": -0.5594726800918579,
      "logps/rejected": -1.9765625,
      "loss": 0.6182,
      "nll_loss": 0.526074230670929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05596923828125,
      "rewards/margins": 0.14174804091453552,
      "rewards/rejected": -0.19760742783546448,
      "step": 12760
    },
    {
      "epoch": 0.929843084428587,
      "grad_norm": 4.296692084609304,
      "learning_rate": 7.079368813915939e-07,
      "log_odds_chosen": 1.540624976158142,
      "log_odds_ratio": -0.41181641817092896,
      "logits/chosen": -1.0041015148162842,
      "logits/rejected": -0.914843738079071,
      "logps/chosen": -0.6820312738418579,
      "logps/rejected": -1.767187476158142,
      "loss": 0.6371,
      "nll_loss": 0.6708984375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06813964992761612,
      "rewards/margins": 0.10853271186351776,
      "rewards/rejected": -0.17673340439796448,
      "step": 12770
    },
    {
      "epoch": 0.9305712309316635,
      "grad_norm": 3.4487894969073056,
      "learning_rate": 7.076598565790337e-07,
      "log_odds_chosen": 1.961328148841858,
      "log_odds_ratio": -0.35026854276657104,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.8802734613418579,
      "logps/chosen": -0.670117199420929,
      "logps/rejected": -2.1312499046325684,
      "loss": 0.626,
      "nll_loss": 0.6177734136581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06697998195886612,
      "rewards/margins": 0.14614257216453552,
      "rewards/rejected": -0.21308593451976776,
      "step": 12780
    },
    {
      "epoch": 0.9312993774347399,
      "grad_norm": 3.62359900286323,
      "learning_rate": 7.073831567222859e-07,
      "log_odds_chosen": 1.728515625,
      "log_odds_ratio": -0.34648436307907104,
      "logits/chosen": -1.076562523841858,
      "logits/rejected": -0.9482421875,
      "logps/chosen": -0.62109375,
      "logps/rejected": -1.8056640625,
      "loss": 0.6286,
      "nll_loss": 0.57568359375,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06212158128619194,
      "rewards/margins": 0.118408203125,
      "rewards/rejected": -0.18051758408546448,
      "step": 12790
    },
    {
      "epoch": 0.9320275239378163,
      "grad_norm": 3.9316840790034675,
      "learning_rate": 7.071067811865475e-07,
      "log_odds_chosen": 1.651953101158142,
      "log_odds_ratio": -0.384765625,
      "logits/chosen": -0.9789062738418579,
      "logits/rejected": -0.882031261920929,
      "logps/chosen": -0.675976574420929,
      "logps/rejected": -1.7863280773162842,
      "loss": 0.6218,
      "nll_loss": 0.6102539300918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06752929836511612,
      "rewards/margins": 0.11119995266199112,
      "rewards/rejected": -0.178955078125,
      "step": 12800
    },
    {
      "epoch": 0.9327556704408927,
      "grad_norm": 3.1549745871560892,
      "learning_rate": 7.068307293387497e-07,
      "log_odds_chosen": 1.8923828601837158,
      "log_odds_ratio": -0.3326171934604645,
      "logits/chosen": -1.0300781726837158,
      "logits/rejected": -0.910937488079071,
      "logps/chosen": -0.6167968511581421,
      "logps/rejected": -1.9269530773162842,
      "loss": 0.613,
      "nll_loss": 0.5545898675918579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06169433519244194,
      "rewards/margins": 0.1309814453125,
      "rewards/rejected": -0.19260254502296448,
      "step": 12810
    },
    {
      "epoch": 0.9334838169439691,
      "grad_norm": 2.9619848879206163,
      "learning_rate": 7.065550005475526e-07,
      "log_odds_chosen": 1.725488305091858,
      "log_odds_ratio": -0.37285155057907104,
      "logits/chosen": -0.987109363079071,
      "logits/rejected": -0.8687499761581421,
      "logps/chosen": -0.606249988079071,
      "logps/rejected": -1.8283202648162842,
      "loss": 0.6215,
      "nll_loss": 0.651660144329071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06058349460363388,
      "rewards/margins": 0.12214355170726776,
      "rewards/rejected": -0.18269042670726776,
      "step": 12820
    },
    {
      "epoch": 0.9342119634470455,
      "grad_norm": 3.425560993020903,
      "learning_rate": 7.062795941833388e-07,
      "log_odds_chosen": 1.876367211341858,
      "log_odds_ratio": -0.3165039122104645,
      "logits/chosen": -1.0330078601837158,
      "logits/rejected": -0.903515636920929,
      "logps/chosen": -0.5809570550918579,
      "logps/rejected": -1.8937499523162842,
      "loss": 0.6085,
      "nll_loss": 0.545214831829071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05806884914636612,
      "rewards/margins": 0.13124999403953552,
      "rewards/rejected": -0.189453125,
      "step": 12830
    },
    {
      "epoch": 0.934940109950122,
      "grad_norm": 3.7699268513005735,
      "learning_rate": 7.060045096182077e-07,
      "log_odds_chosen": 1.898828148841858,
      "log_odds_ratio": -0.330810546875,
      "logits/chosen": -0.986132800579071,
      "logits/rejected": -0.901562511920929,
      "logps/chosen": -0.6298828125,
      "logps/rejected": -1.986718773841858,
      "loss": 0.6331,
      "nll_loss": 0.629150390625,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06295166164636612,
      "rewards/margins": 0.13569335639476776,
      "rewards/rejected": -0.19868163764476776,
      "step": 12840
    },
    {
      "epoch": 0.9356682564531984,
      "grad_norm": 4.792377069747243,
      "learning_rate": 7.057297462259693e-07,
      "log_odds_chosen": 1.725195288658142,
      "log_odds_ratio": -0.367919921875,
      "logits/chosen": -0.9820312261581421,
      "logits/rejected": -0.874218761920929,
      "logps/chosen": -0.611035168170929,
      "logps/rejected": -1.7998046875,
      "loss": 0.6253,
      "nll_loss": 0.5619140863418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06102294847369194,
      "rewards/margins": 0.11896972358226776,
      "rewards/rejected": -0.18000487983226776,
      "step": 12850
    },
    {
      "epoch": 0.9363964029562748,
      "grad_norm": 2.895944959407833,
      "learning_rate": 7.05455303382138e-07,
      "log_odds_chosen": 2.117382764816284,
      "log_odds_ratio": -0.2914062440395355,
      "logits/chosen": -0.986523449420929,
      "logits/rejected": -0.8681640625,
      "logps/chosen": -0.57666015625,
      "logps/rejected": -2.0511717796325684,
      "loss": 0.6403,
      "nll_loss": 0.602832019329071,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.05765380710363388,
      "rewards/margins": 0.14757080376148224,
      "rewards/rejected": -0.20522460341453552,
      "step": 12860
    },
    {
      "epoch": 0.9371245494593512,
      "grad_norm": 3.2169591666715753,
      "learning_rate": 7.051811804639268e-07,
      "log_odds_chosen": 1.850976586341858,
      "log_odds_ratio": -0.35302734375,
      "logits/chosen": -0.9896484613418579,
      "logits/rejected": -0.8587890863418579,
      "logps/chosen": -0.6259765625,
      "logps/rejected": -1.960546851158142,
      "loss": 0.6626,
      "nll_loss": 0.598925769329071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06257323920726776,
      "rewards/margins": 0.133544921875,
      "rewards/rejected": -0.19614258408546448,
      "step": 12870
    },
    {
      "epoch": 0.9378526959624276,
      "grad_norm": 3.2083587062160257,
      "learning_rate": 7.049073768502414e-07,
      "log_odds_chosen": 2.0308594703674316,
      "log_odds_ratio": -0.3134765625,
      "logits/chosen": -0.9951171875,
      "logits/rejected": -0.8902343511581421,
      "logps/chosen": -0.599316418170929,
      "logps/rejected": -1.993749976158142,
      "loss": 0.616,
      "nll_loss": 0.537792980670929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05994873121380806,
      "rewards/margins": 0.13933105766773224,
      "rewards/rejected": -0.19921875,
      "step": 12880
    },
    {
      "epoch": 0.9385808424655041,
      "grad_norm": 3.247733355860675,
      "learning_rate": 7.046338919216742e-07,
      "log_odds_chosen": 1.8157227039337158,
      "log_odds_ratio": -0.35107421875,
      "logits/chosen": -1.0294921398162842,
      "logits/rejected": -0.8958984613418579,
      "logps/chosen": -0.597460925579071,
      "logps/rejected": -1.8298828601837158,
      "loss": 0.6265,
      "nll_loss": 0.609570324420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.0596923828125,
      "rewards/margins": 0.12343750149011612,
      "rewards/rejected": -0.18317870795726776,
      "step": 12890
    },
    {
      "epoch": 0.9393089889685805,
      "grad_norm": 3.4147507965613486,
      "learning_rate": 7.04360725060499e-07,
      "log_odds_chosen": 1.9236328601837158,
      "log_odds_ratio": -0.28754884004592896,
      "logits/chosen": -0.9677734375,
      "logits/rejected": -0.8304687738418579,
      "logps/chosen": -0.5985351800918579,
      "logps/rejected": -1.923437476158142,
      "loss": 0.6066,
      "nll_loss": 0.5457519292831421,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.05985717847943306,
      "rewards/margins": 0.13242188096046448,
      "rewards/rejected": -0.19235840439796448,
      "step": 12900
    },
    {
      "epoch": 0.9400371354716569,
      "grad_norm": 3.106899666161103,
      "learning_rate": 7.040878756506639e-07,
      "log_odds_chosen": 1.87890625,
      "log_odds_ratio": -0.33771973848342896,
      "logits/chosen": -1.054296851158142,
      "logits/rejected": -0.9140625,
      "logps/chosen": -0.617480456829071,
      "logps/rejected": -1.941015601158142,
      "loss": 0.6233,
      "nll_loss": 0.545214831829071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06170654296875,
      "rewards/margins": 0.13227538764476776,
      "rewards/rejected": -0.19401855766773224,
      "step": 12910
    },
    {
      "epoch": 0.9407652819747333,
      "grad_norm": 3.3009572299501,
      "learning_rate": 7.038153430777867e-07,
      "log_odds_chosen": 1.908203125,
      "log_odds_ratio": -0.340576171875,
      "logits/chosen": -0.9927734136581421,
      "logits/rejected": -0.8798828125,
      "logps/chosen": -0.6246093511581421,
      "logps/rejected": -1.971093773841858,
      "loss": 0.6414,
      "nll_loss": 0.623339831829071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06241454929113388,
      "rewards/margins": 0.134521484375,
      "rewards/rejected": -0.19721679389476776,
      "step": 12920
    },
    {
      "epoch": 0.9414934284778097,
      "grad_norm": 3.1944019610489405,
      "learning_rate": 7.035431267291484e-07,
      "log_odds_chosen": 2.072070360183716,
      "log_odds_ratio": -0.30467528104782104,
      "logits/chosen": -1.00390625,
      "logits/rejected": -0.9312499761581421,
      "logps/chosen": -0.5794922113418579,
      "logps/rejected": -2.033203125,
      "loss": 0.6281,
      "nll_loss": 0.607617199420929,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.05792236328125,
      "rewards/margins": 0.14544677734375,
      "rewards/rejected": -0.203369140625,
      "step": 12930
    },
    {
      "epoch": 0.9422215749808861,
      "grad_norm": 3.4322764877264644,
      "learning_rate": 7.032712259936877e-07,
      "log_odds_chosen": 2.1019530296325684,
      "log_odds_ratio": -0.3072753846645355,
      "logits/chosen": -1.0427734851837158,
      "logits/rejected": -0.9398437738418579,
      "logps/chosen": -0.5511718988418579,
      "logps/rejected": -1.964453101158142,
      "loss": 0.5922,
      "nll_loss": 0.5625,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05510864406824112,
      "rewards/margins": 0.14140625298023224,
      "rewards/rejected": -0.196533203125,
      "step": 12940
    },
    {
      "epoch": 0.9429497214839626,
      "grad_norm": 3.181966890305539,
      "learning_rate": 7.029996402619949e-07,
      "log_odds_chosen": 1.9148437976837158,
      "log_odds_ratio": -0.3274169862270355,
      "logits/chosen": -1.0603516101837158,
      "logits/rejected": -0.9515625238418579,
      "logps/chosen": -0.5811523199081421,
      "logps/rejected": -1.9035155773162842,
      "loss": 0.6203,
      "nll_loss": 0.5283203125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05812988430261612,
      "rewards/margins": 0.13218994438648224,
      "rewards/rejected": -0.19028320908546448,
      "step": 12950
    },
    {
      "epoch": 0.943677867987039,
      "grad_norm": 3.4561899602553314,
      "learning_rate": 7.027283689263065e-07,
      "log_odds_chosen": 1.82470703125,
      "log_odds_ratio": -0.3317627012729645,
      "logits/chosen": -1.028710961341858,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.610156238079071,
      "logps/rejected": -1.853124976158142,
      "loss": 0.6422,
      "nll_loss": 0.666308581829071,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.06099853664636612,
      "rewards/margins": 0.12429962307214737,
      "rewards/rejected": -0.18532714247703552,
      "step": 12960
    },
    {
      "epoch": 0.9444060144901154,
      "grad_norm": 3.939081908030912,
      "learning_rate": 7.024574113804996e-07,
      "log_odds_chosen": 1.7783203125,
      "log_odds_ratio": -0.3462890684604645,
      "logits/chosen": -1.01171875,
      "logits/rejected": -0.921093761920929,
      "logps/chosen": -0.592578113079071,
      "logps/rejected": -1.837499976158142,
      "loss": 0.6146,
      "nll_loss": 0.5821288824081421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05924072116613388,
      "rewards/margins": 0.124267578125,
      "rewards/rejected": -0.18359375,
      "step": 12970
    },
    {
      "epoch": 0.9451341609931918,
      "grad_norm": 3.435167035344221,
      "learning_rate": 7.021867670200857e-07,
      "log_odds_chosen": 1.9074218273162842,
      "log_odds_ratio": -0.3539062440395355,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.925585925579071,
      "logps/chosen": -0.5953124761581421,
      "logps/rejected": -1.9189453125,
      "loss": 0.6196,
      "nll_loss": 0.5523437261581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05952148512005806,
      "rewards/margins": 0.13236084580421448,
      "rewards/rejected": -0.19204100966453552,
      "step": 12980
    },
    {
      "epoch": 0.9458623074962682,
      "grad_norm": 3.9618887181927636,
      "learning_rate": 7.019164352422057e-07,
      "log_odds_chosen": 1.7945067882537842,
      "log_odds_ratio": -0.3578857481479645,
      "logits/chosen": -1.021875023841858,
      "logits/rejected": -0.8500000238418579,
      "logps/chosen": -0.604687511920929,
      "logps/rejected": -1.841210961341858,
      "loss": 0.6385,
      "nll_loss": 0.60986328125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06044922024011612,
      "rewards/margins": 0.12359008938074112,
      "rewards/rejected": -0.18408203125,
      "step": 12990
    },
    {
      "epoch": 0.9465904539993447,
      "grad_norm": 2.8814202144243994,
      "learning_rate": 7.016464154456234e-07,
      "log_odds_chosen": 1.5927734375,
      "log_odds_ratio": -0.36835938692092896,
      "logits/chosen": -1.068945288658142,
      "logits/rejected": -0.9371093511581421,
      "logps/chosen": -0.58984375,
      "logps/rejected": -1.6886718273162842,
      "loss": 0.6139,
      "nll_loss": 0.523242175579071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05903320387005806,
      "rewards/margins": 0.10988769680261612,
      "rewards/rejected": -0.1689453125,
      "step": 13000
    },
    {
      "epoch": 0.9473186005024211,
      "grad_norm": 3.5740618255021506,
      "learning_rate": 7.013767070307207e-07,
      "log_odds_chosen": 1.884374976158142,
      "log_odds_ratio": -0.31645506620407104,
      "logits/chosen": -1.0089843273162842,
      "logits/rejected": -0.8558593988418579,
      "logps/chosen": -0.6220703125,
      "logps/rejected": -1.9460937976837158,
      "loss": 0.6435,
      "nll_loss": 0.615917980670929,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06217040866613388,
      "rewards/margins": 0.13247069716453552,
      "rewards/rejected": -0.19455567002296448,
      "step": 13010
    },
    {
      "epoch": 0.9480467470054975,
      "grad_norm": 2.573504531773923,
      "learning_rate": 7.011073093994919e-07,
      "log_odds_chosen": 1.902734398841858,
      "log_odds_ratio": -0.31206053495407104,
      "logits/chosen": -1.0294921398162842,
      "logits/rejected": -0.884765625,
      "logps/chosen": -0.5580078363418579,
      "logps/rejected": -1.8722655773162842,
      "loss": 0.6091,
      "nll_loss": 0.5506836175918579,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.05583496019244194,
      "rewards/margins": 0.1314697265625,
      "rewards/rejected": -0.187255859375,
      "step": 13020
    },
    {
      "epoch": 0.9487748935085739,
      "grad_norm": 2.897768268838172,
      "learning_rate": 7.008382219555372e-07,
      "log_odds_chosen": 1.886816382408142,
      "log_odds_ratio": -0.3540283143520355,
      "logits/chosen": -1.020117163658142,
      "logits/rejected": -0.8921874761581421,
      "logps/chosen": -0.659375011920929,
      "logps/rejected": -2.030468702316284,
      "loss": 0.6226,
      "nll_loss": 0.592968761920929,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06591796875,
      "rewards/margins": 0.13726806640625,
      "rewards/rejected": -0.20302733778953552,
      "step": 13030
    },
    {
      "epoch": 0.9495030400116503,
      "grad_norm": 2.868919227794938,
      "learning_rate": 7.005694441040588e-07,
      "log_odds_chosen": 1.984375,
      "log_odds_ratio": -0.29743653535842896,
      "logits/chosen": -1.0236327648162842,
      "logits/rejected": -0.913281261920929,
      "logps/chosen": -0.6117187738418579,
      "logps/rejected": -2.0171875953674316,
      "loss": 0.6264,
      "nll_loss": 0.608691394329071,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0611572265625,
      "rewards/margins": 0.14047852158546448,
      "rewards/rejected": -0.20180663466453552,
      "step": 13040
    },
    {
      "epoch": 0.9502311865147267,
      "grad_norm": 3.3227751754532395,
      "learning_rate": 7.003009752518536e-07,
      "log_odds_chosen": 2.0289063453674316,
      "log_odds_ratio": -0.2757324278354645,
      "logits/chosen": -1.043359398841858,
      "logits/rejected": -0.924609363079071,
      "logps/chosen": -0.586621105670929,
      "logps/rejected": -1.9775390625,
      "loss": 0.6075,
      "nll_loss": 0.567089855670929,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.05860595777630806,
      "rewards/margins": 0.1390380859375,
      "rewards/rejected": -0.19772949814796448,
      "step": 13050
    },
    {
      "epoch": 0.9509593330178032,
      "grad_norm": 3.2082582090754603,
      "learning_rate": 7.000328148073091e-07,
      "log_odds_chosen": 1.9113280773162842,
      "log_odds_ratio": -0.3321777284145355,
      "logits/chosen": -1.031640648841858,
      "logits/rejected": -0.916796863079071,
      "logps/chosen": -0.591503918170929,
      "logps/rejected": -1.9357421398162842,
      "loss": 0.5921,
      "nll_loss": 0.5640624761581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05908203125,
      "rewards/margins": 0.13455811142921448,
      "rewards/rejected": -0.193603515625,
      "step": 13060
    },
    {
      "epoch": 0.9516874795208796,
      "grad_norm": 3.0251059483635987,
      "learning_rate": 6.997649621803973e-07,
      "log_odds_chosen": 1.4739258289337158,
      "log_odds_ratio": -0.40336912870407104,
      "logits/chosen": -1.031640648841858,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.6302734613418579,
      "logps/rejected": -1.6287109851837158,
      "loss": 0.6149,
      "nll_loss": 0.6006835699081421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06300048530101776,
      "rewards/margins": 0.09987487643957138,
      "rewards/rejected": -0.1629638671875,
      "step": 13070
    },
    {
      "epoch": 0.952415626023956,
      "grad_norm": 4.59423193896908,
      "learning_rate": 6.994974167826689e-07,
      "log_odds_chosen": 1.6886718273162842,
      "log_odds_ratio": -0.3357910215854645,
      "logits/chosen": -1.0207030773162842,
      "logits/rejected": -0.8824218511581421,
      "logps/chosen": -0.5810546875,
      "logps/rejected": -1.6951172351837158,
      "loss": 0.6421,
      "nll_loss": 0.5376952886581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05808105319738388,
      "rewards/margins": 0.11140136420726776,
      "rewards/rejected": -0.16950683295726776,
      "step": 13080
    },
    {
      "epoch": 0.9531437725270324,
      "grad_norm": 3.3379051433650564,
      "learning_rate": 6.99230178027249e-07,
      "log_odds_chosen": 1.64111328125,
      "log_odds_ratio": -0.38774412870407104,
      "logits/chosen": -0.9986327886581421,
      "logits/rejected": -0.9087890386581421,
      "logps/chosen": -0.599804699420929,
      "logps/rejected": -1.699804663658142,
      "loss": 0.6191,
      "nll_loss": 0.624316394329071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.05994873121380806,
      "rewards/margins": 0.10997924953699112,
      "rewards/rejected": -0.16997070610523224,
      "step": 13090
    },
    {
      "epoch": 0.9538719190301088,
      "grad_norm": 3.3588881283186245,
      "learning_rate": 6.989632453288303e-07,
      "log_odds_chosen": 1.9656250476837158,
      "log_odds_ratio": -0.314208984375,
      "logits/chosen": -1.0138671398162842,
      "logits/rejected": -0.866406261920929,
      "logps/chosen": -0.61083984375,
      "logps/rejected": -1.970312476158142,
      "loss": 0.6171,
      "nll_loss": 0.59326171875,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.06109619140625,
      "rewards/margins": 0.13608399033546448,
      "rewards/rejected": -0.197021484375,
      "step": 13100
    },
    {
      "epoch": 0.9546000655331853,
      "grad_norm": 3.149373440245997,
      "learning_rate": 6.98696618103669e-07,
      "log_odds_chosen": 1.891699194908142,
      "log_odds_ratio": -0.34602051973342896,
      "logits/chosen": -1.0,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -0.6455078125,
      "logps/rejected": -1.984765648841858,
      "loss": 0.6398,
      "nll_loss": 0.5462890863418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06461181491613388,
      "rewards/margins": 0.133869931101799,
      "rewards/rejected": -0.19851073622703552,
      "step": 13110
    },
    {
      "epoch": 0.9553282120362617,
      "grad_norm": 3.0556800104962947,
      "learning_rate": 6.984302957695782e-07,
      "log_odds_chosen": 1.828515648841858,
      "log_odds_ratio": -0.3318115174770355,
      "logits/chosen": -1.048437476158142,
      "logits/rejected": -0.8814452886581421,
      "logps/chosen": -0.64990234375,
      "logps/rejected": -1.951562523841858,
      "loss": 0.6411,
      "nll_loss": 0.621289074420929,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06503906100988388,
      "rewards/margins": 0.13022461533546448,
      "rewards/rejected": -0.19497069716453552,
      "step": 13120
    },
    {
      "epoch": 0.9560563585393381,
      "grad_norm": 2.8045450120533446,
      "learning_rate": 6.981642777459237e-07,
      "log_odds_chosen": 1.721337914466858,
      "log_odds_ratio": -0.3682617247104645,
      "logits/chosen": -1.048242211341858,
      "logits/rejected": -0.889453113079071,
      "logps/chosen": -0.5927734375,
      "logps/rejected": -1.7421875,
      "loss": 0.6128,
      "nll_loss": 0.5791991949081421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.0592041015625,
      "rewards/margins": 0.11490173637866974,
      "rewards/rejected": -0.1741943359375,
      "step": 13130
    },
    {
      "epoch": 0.9567845050424145,
      "grad_norm": 3.564660850010864,
      "learning_rate": 6.978985634536182e-07,
      "log_odds_chosen": 1.676171898841858,
      "log_odds_ratio": -0.39946287870407104,
      "logits/chosen": -1.015039086341858,
      "logits/rejected": -0.853710949420929,
      "logps/chosen": -0.642285168170929,
      "logps/rejected": -1.814843773841858,
      "loss": 0.6025,
      "nll_loss": 0.603515625,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.064208984375,
      "rewards/margins": 0.1171875,
      "rewards/rejected": -0.18146972358226776,
      "step": 13140
    },
    {
      "epoch": 0.9575126515454909,
      "grad_norm": 3.063190893348432,
      "learning_rate": 6.976331523151157e-07,
      "log_odds_chosen": 1.872460961341858,
      "log_odds_ratio": -0.34785157442092896,
      "logits/chosen": -1.001562476158142,
      "logits/rejected": -0.9283202886581421,
      "logps/chosen": -0.621386706829071,
      "logps/rejected": -1.8566405773162842,
      "loss": 0.6273,
      "nll_loss": 0.553515613079071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06210937350988388,
      "rewards/margins": 0.1236572265625,
      "rewards/rejected": -0.18571777641773224,
      "step": 13150
    },
    {
      "epoch": 0.9582407980485673,
      "grad_norm": 2.873520481911809,
      "learning_rate": 6.973680437544066e-07,
      "log_odds_chosen": 1.9070312976837158,
      "log_odds_ratio": -0.31499022245407104,
      "logits/chosen": -1.004492163658142,
      "logits/rejected": -0.8882812261581421,
      "logps/chosen": -0.575390636920929,
      "logps/rejected": -1.8875000476837158,
      "loss": 0.6007,
      "nll_loss": 0.5809570550918579,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.05750732496380806,
      "rewards/margins": 0.1312255859375,
      "rewards/rejected": -0.18879394233226776,
      "step": 13160
    },
    {
      "epoch": 0.9589689445516438,
      "grad_norm": 2.8076185700636236,
      "learning_rate": 6.971032371970126e-07,
      "log_odds_chosen": 1.7410156726837158,
      "log_odds_ratio": -0.35151368379592896,
      "logits/chosen": -1.002343773841858,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.5843750238418579,
      "logps/rejected": -1.766015648841858,
      "loss": 0.6302,
      "nll_loss": 0.6016601324081421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05841064453125,
      "rewards/margins": 0.11815185844898224,
      "rewards/rejected": -0.17658691108226776,
      "step": 13170
    },
    {
      "epoch": 0.9596970910547202,
      "grad_norm": 3.051015063050668,
      "learning_rate": 6.968387320699806e-07,
      "log_odds_chosen": 1.417578101158142,
      "log_odds_ratio": -0.451416015625,
      "logits/chosen": -0.9515625238418579,
      "logits/rejected": -0.880859375,
      "logps/chosen": -0.703320324420929,
      "logps/rejected": -1.66796875,
      "loss": 0.6356,
      "nll_loss": 0.664746105670929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07037353515625,
      "rewards/margins": 0.09645996242761612,
      "rewards/rejected": -0.16691894829273224,
      "step": 13180
    },
    {
      "epoch": 0.9604252375577966,
      "grad_norm": 6.165319783602804,
      "learning_rate": 6.965745278018791e-07,
      "log_odds_chosen": 2.0160155296325684,
      "log_odds_ratio": -0.3364013731479645,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.889453113079071,
      "logps/chosen": -0.650195300579071,
      "logps/rejected": -2.098437547683716,
      "loss": 0.6122,
      "nll_loss": 0.6050781011581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06500244140625,
      "rewards/margins": 0.14498290419578552,
      "rewards/rejected": -0.2099609375,
      "step": 13190
    },
    {
      "epoch": 0.961153384060873,
      "grad_norm": 2.995238378706094,
      "learning_rate": 6.963106238227914e-07,
      "log_odds_chosen": 2.033007860183716,
      "log_odds_ratio": -0.2851196229457855,
      "logits/chosen": -1.055273413658142,
      "logits/rejected": -0.9169921875,
      "logps/chosen": -0.618847668170929,
      "logps/rejected": -2.051562547683716,
      "loss": 0.5926,
      "nll_loss": 0.62158203125,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06189575046300888,
      "rewards/margins": 0.14324951171875,
      "rewards/rejected": -0.20522460341453552,
      "step": 13200
    },
    {
      "epoch": 0.9618815305639494,
      "grad_norm": 3.1735219113324136,
      "learning_rate": 6.96047019564311e-07,
      "log_odds_chosen": 1.8791015148162842,
      "log_odds_ratio": -0.37158203125,
      "logits/chosen": -0.9634765386581421,
      "logits/rejected": -0.884765625,
      "logps/chosen": -0.661816418170929,
      "logps/rejected": -2.0248045921325684,
      "loss": 0.6198,
      "nll_loss": 0.5850585699081421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06618652492761612,
      "rewards/margins": 0.13620606064796448,
      "rewards/rejected": -0.20249024033546448,
      "step": 13210
    },
    {
      "epoch": 0.9626096770670259,
      "grad_norm": 3.085411441782341,
      "learning_rate": 6.957837144595368e-07,
      "log_odds_chosen": 1.6638672351837158,
      "log_odds_ratio": -0.35834962129592896,
      "logits/chosen": -0.989062488079071,
      "logits/rejected": -0.888867199420929,
      "logps/chosen": -0.618945300579071,
      "logps/rejected": -1.7746093273162842,
      "loss": 0.617,
      "nll_loss": 0.599609375,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06191406399011612,
      "rewards/margins": 0.11551513522863388,
      "rewards/rejected": -0.17739257216453552,
      "step": 13220
    },
    {
      "epoch": 0.9633378235701023,
      "grad_norm": 5.112703814587654,
      "learning_rate": 6.955207079430681e-07,
      "log_odds_chosen": 2.197265625,
      "log_odds_ratio": -0.30156248807907104,
      "logits/chosen": -1.019921898841858,
      "logits/rejected": -0.864062488079071,
      "logps/chosen": -0.6005859375,
      "logps/rejected": -2.1929688453674316,
      "loss": 0.5992,
      "nll_loss": 0.5474609136581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06007080152630806,
      "rewards/margins": 0.159423828125,
      "rewards/rejected": -0.21943359076976776,
      "step": 13230
    },
    {
      "epoch": 0.9640659700731787,
      "grad_norm": 3.4999924397082545,
      "learning_rate": 6.952579994509982e-07,
      "log_odds_chosen": 1.771093726158142,
      "log_odds_ratio": -0.34052735567092896,
      "logits/chosen": -0.996874988079071,
      "logits/rejected": -0.841992199420929,
      "logps/chosen": -0.557324230670929,
      "logps/rejected": -1.7482421398162842,
      "loss": 0.6203,
      "nll_loss": 0.588183581829071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05571288987994194,
      "rewards/margins": 0.11916504055261612,
      "rewards/rejected": -0.1749267578125,
      "step": 13240
    },
    {
      "epoch": 0.9647941165762551,
      "grad_norm": 3.521653941656919,
      "learning_rate": 6.94995588420911e-07,
      "log_odds_chosen": 1.866601586341858,
      "log_odds_ratio": -0.3379150331020355,
      "logits/chosen": -0.992382824420929,
      "logits/rejected": -0.8734375238418579,
      "logps/chosen": -0.604296863079071,
      "logps/rejected": -1.9109375476837158,
      "loss": 0.6177,
      "nll_loss": 0.6480468511581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06038818508386612,
      "rewards/margins": 0.130615234375,
      "rewards/rejected": -0.19106444716453552,
      "step": 13250
    },
    {
      "epoch": 0.9655222630793315,
      "grad_norm": 3.5035745190229015,
      "learning_rate": 6.947334742918749e-07,
      "log_odds_chosen": 1.9854736328125,
      "log_odds_ratio": -0.3360839784145355,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.5801757574081421,
      "logps/rejected": -1.994531273841858,
      "loss": 0.6132,
      "nll_loss": 0.556445300579071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.0579833984375,
      "rewards/margins": 0.14157256484031677,
      "rewards/rejected": -0.19941405951976776,
      "step": 13260
    },
    {
      "epoch": 0.9662504095824079,
      "grad_norm": 3.208538705688946,
      "learning_rate": 6.94471656504438e-07,
      "log_odds_chosen": 2.0035157203674316,
      "log_odds_ratio": -0.3192138671875,
      "logits/chosen": -1.0388672351837158,
      "logits/rejected": -0.874218761920929,
      "logps/chosen": -0.6163085699081421,
      "logps/rejected": -2.0074219703674316,
      "loss": 0.6154,
      "nll_loss": 0.5889648199081421,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.0616455078125,
      "rewards/margins": 0.1390380859375,
      "rewards/rejected": -0.20063476264476776,
      "step": 13270
    },
    {
      "epoch": 0.9669785560854844,
      "grad_norm": 3.557398533574716,
      "learning_rate": 6.942101345006232e-07,
      "log_odds_chosen": 1.756323218345642,
      "log_odds_ratio": -0.35698240995407104,
      "logits/chosen": -0.9869140386581421,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.6285156011581421,
      "logps/rejected": -1.862695336341858,
      "loss": 0.6224,
      "nll_loss": 0.6307617425918579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06281737983226776,
      "rewards/margins": 0.12338867038488388,
      "rewards/rejected": -0.18618163466453552,
      "step": 13280
    },
    {
      "epoch": 0.9677067025885608,
      "grad_norm": 3.527534107625696,
      "learning_rate": 6.939489077239235e-07,
      "log_odds_chosen": 1.6935546398162842,
      "log_odds_ratio": -0.3567138612270355,
      "logits/chosen": -1.0037109851837158,
      "logits/rejected": -0.8931640386581421,
      "logps/chosen": -0.6338866949081421,
      "logps/rejected": -1.828125,
      "loss": 0.6065,
      "nll_loss": 0.6341797113418579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06333007663488388,
      "rewards/margins": 0.11933593451976776,
      "rewards/rejected": -0.18259277939796448,
      "step": 13290
    },
    {
      "epoch": 0.9684348490916372,
      "grad_norm": 3.1110972493101703,
      "learning_rate": 6.936879756192959e-07,
      "log_odds_chosen": 1.804296851158142,
      "log_odds_ratio": -0.31267088651657104,
      "logits/chosen": -1.0333983898162842,
      "logits/rejected": -0.9332031011581421,
      "logps/chosen": -0.55419921875,
      "logps/rejected": -1.6867187023162842,
      "loss": 0.6075,
      "nll_loss": 0.5059570074081421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.055419921875,
      "rewards/margins": 0.11320801079273224,
      "rewards/rejected": -0.16867676377296448,
      "step": 13300
    },
    {
      "epoch": 0.9691629955947136,
      "grad_norm": 3.4409253456118765,
      "learning_rate": 6.934273376331579e-07,
      "log_odds_chosen": 1.6794922351837158,
      "log_odds_ratio": -0.33833009004592896,
      "logits/chosen": -0.9917968511581421,
      "logits/rejected": -0.8773437738418579,
      "logps/chosen": -0.5570312738418579,
      "logps/rejected": -1.654296875,
      "loss": 0.5935,
      "nll_loss": 0.5926758050918579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05570068210363388,
      "rewards/margins": 0.10969237983226776,
      "rewards/rejected": -0.16538086533546448,
      "step": 13310
    },
    {
      "epoch": 0.96989114209779,
      "grad_norm": 4.02476095637564,
      "learning_rate": 6.931669932133818e-07,
      "log_odds_chosen": 2.066210985183716,
      "log_odds_ratio": -0.30290526151657104,
      "logits/chosen": -1.0173828601837158,
      "logits/rejected": -0.891406238079071,
      "logps/chosen": -0.588183581829071,
      "logps/rejected": -2.02734375,
      "loss": 0.6095,
      "nll_loss": 0.596972644329071,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05882568284869194,
      "rewards/margins": 0.14385986328125,
      "rewards/rejected": -0.20263671875,
      "step": 13320
    },
    {
      "epoch": 0.9706192886008665,
      "grad_norm": 3.433415908845924,
      "learning_rate": 6.929069418092892e-07,
      "log_odds_chosen": 1.7903320789337158,
      "log_odds_ratio": -0.330322265625,
      "logits/chosen": -1.0546875,
      "logits/rejected": -0.9447265863418579,
      "logps/chosen": -0.5997070074081421,
      "logps/rejected": -1.846289038658142,
      "loss": 0.6192,
      "nll_loss": 0.5736328363418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05998535081744194,
      "rewards/margins": 0.12459106743335724,
      "rewards/rejected": -0.18452148139476776,
      "step": 13330
    },
    {
      "epoch": 0.971347435103943,
      "grad_norm": 3.4916467618931075,
      "learning_rate": 6.926471828716478e-07,
      "log_odds_chosen": 1.8505859375,
      "log_odds_ratio": -0.355224609375,
      "logits/chosen": -1.011132836341858,
      "logits/rejected": -0.9033203125,
      "logps/chosen": -0.58251953125,
      "logps/rejected": -1.8671875,
      "loss": 0.6236,
      "nll_loss": 0.5517578125,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05826415866613388,
      "rewards/margins": 0.128662109375,
      "rewards/rejected": -0.18684081733226776,
      "step": 13340
    },
    {
      "epoch": 0.9720755816070193,
      "grad_norm": 4.273160001295136,
      "learning_rate": 6.923877158526646e-07,
      "log_odds_chosen": 1.9386718273162842,
      "log_odds_ratio": -0.2949462831020355,
      "logits/chosen": -1.0173828601837158,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.6060546636581421,
      "logps/rejected": -1.9453125,
      "loss": 0.6252,
      "nll_loss": 0.6099609136581421,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06064452975988388,
      "rewards/margins": 0.13405761122703552,
      "rewards/rejected": -0.19462890923023224,
      "step": 13350
    },
    {
      "epoch": 0.9728037281100957,
      "grad_norm": 3.551111399974384,
      "learning_rate": 6.921285402059827e-07,
      "log_odds_chosen": 1.5859375,
      "log_odds_ratio": -0.388427734375,
      "logits/chosen": -1.0662109851837158,
      "logits/rejected": -0.9046875238418579,
      "logps/chosen": -0.6513671875,
      "logps/rejected": -1.7578125,
      "loss": 0.6159,
      "nll_loss": 0.5884765386581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06512451171875,
      "rewards/margins": 0.11057128757238388,
      "rewards/rejected": -0.1756591796875,
      "step": 13360
    },
    {
      "epoch": 0.9735318746131721,
      "grad_norm": 4.022762362286783,
      "learning_rate": 6.918696553866751e-07,
      "log_odds_chosen": 1.9181640148162842,
      "log_odds_ratio": -0.2826171815395355,
      "logits/chosen": -0.976757824420929,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.563281238079071,
      "logps/rejected": -1.8488280773162842,
      "loss": 0.5902,
      "nll_loss": 0.5599609613418579,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.05633544921875,
      "rewards/margins": 0.12858887016773224,
      "rewards/rejected": -0.18496093153953552,
      "step": 13370
    },
    {
      "epoch": 0.9742600211162485,
      "grad_norm": 4.077681398502788,
      "learning_rate": 6.916110608512408e-07,
      "log_odds_chosen": 1.8445312976837158,
      "log_odds_ratio": -0.32109373807907104,
      "logits/chosen": -1.080664038658142,
      "logits/rejected": -0.9320312738418579,
      "logps/chosen": -0.603515625,
      "logps/rejected": -1.8984375,
      "loss": 0.6137,
      "nll_loss": 0.526660144329071,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.06033935397863388,
      "rewards/margins": 0.12949219346046448,
      "rewards/rejected": -0.18972167372703552,
      "step": 13380
    },
    {
      "epoch": 0.974988167619325,
      "grad_norm": 2.683833078700542,
      "learning_rate": 6.913527560575998e-07,
      "log_odds_chosen": 1.8562500476837158,
      "log_odds_ratio": -0.33488768339157104,
      "logits/chosen": -1.0261719226837158,
      "logits/rejected": -0.8984375,
      "logps/chosen": -0.6045898199081421,
      "logps/rejected": -1.8507812023162842,
      "loss": 0.6072,
      "nll_loss": 0.59521484375,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06047363206744194,
      "rewards/margins": 0.12471923977136612,
      "rewards/rejected": -0.18515625596046448,
      "step": 13390
    },
    {
      "epoch": 0.9757163141224015,
      "grad_norm": 3.242221796779325,
      "learning_rate": 6.910947404650881e-07,
      "log_odds_chosen": 1.9443359375,
      "log_odds_ratio": -0.3294433653354645,
      "logits/chosen": -1.0263671875,
      "logits/rejected": -0.891796886920929,
      "logps/chosen": -0.5782226324081421,
      "logps/rejected": -1.888281226158142,
      "loss": 0.5936,
      "nll_loss": 0.533447265625,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.05784912034869194,
      "rewards/margins": 0.13094481825828552,
      "rewards/rejected": -0.18854980170726776,
      "step": 13400
    },
    {
      "epoch": 0.9764444606254779,
      "grad_norm": 3.2773548336766316,
      "learning_rate": 6.90837013534453e-07,
      "log_odds_chosen": 1.6941406726837158,
      "log_odds_ratio": -0.3331054747104645,
      "logits/chosen": -0.9750000238418579,
      "logits/rejected": -0.8560546636581421,
      "logps/chosen": -0.602734386920929,
      "logps/rejected": -1.7472655773162842,
      "loss": 0.6105,
      "nll_loss": 0.587695300579071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06025390699505806,
      "rewards/margins": 0.11452636867761612,
      "rewards/rejected": -0.17470702528953552,
      "step": 13410
    },
    {
      "epoch": 0.9771726071285542,
      "grad_norm": 3.139404919568771,
      "learning_rate": 6.905795747278488e-07,
      "log_odds_chosen": 1.603906273841858,
      "log_odds_ratio": -0.3860839903354645,
      "logits/chosen": -1.042578101158142,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.650195300579071,
      "logps/rejected": -1.7900390625,
      "loss": 0.6171,
      "nll_loss": 0.6117187738418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06501464545726776,
      "rewards/margins": 0.11392822116613388,
      "rewards/rejected": -0.17900390923023224,
      "step": 13420
    },
    {
      "epoch": 0.9779007536316306,
      "grad_norm": 3.5566846236894616,
      "learning_rate": 6.903224235088314e-07,
      "log_odds_chosen": 1.668554663658142,
      "log_odds_ratio": -0.3659912049770355,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.8687499761581421,
      "logps/chosen": -0.593066394329071,
      "logps/rejected": -1.7179687023162842,
      "loss": 0.6232,
      "nll_loss": 0.570507824420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.05933837965130806,
      "rewards/margins": 0.11239013820886612,
      "rewards/rejected": -0.1717529296875,
      "step": 13430
    },
    {
      "epoch": 0.9786289001347072,
      "grad_norm": 2.8946882834904564,
      "learning_rate": 6.900655593423541e-07,
      "log_odds_chosen": 1.916406273841858,
      "log_odds_ratio": -0.36726075410842896,
      "logits/chosen": -1.0031249523162842,
      "logits/rejected": -0.886523425579071,
      "logps/chosen": -0.5748046636581421,
      "logps/rejected": -1.8621094226837158,
      "loss": 0.6302,
      "nll_loss": 0.586621105670929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05744628980755806,
      "rewards/margins": 0.12869873642921448,
      "rewards/rejected": -0.18605956435203552,
      "step": 13440
    },
    {
      "epoch": 0.9793570466377836,
      "grad_norm": 3.2802121052536117,
      "learning_rate": 6.89808981694763e-07,
      "log_odds_chosen": 2.0589842796325684,
      "log_odds_ratio": -0.2928466796875,
      "logits/chosen": -0.987500011920929,
      "logits/rejected": -0.833984375,
      "logps/chosen": -0.5943359136581421,
      "logps/rejected": -1.963281273841858,
      "loss": 0.5956,
      "nll_loss": 0.55908203125,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.05939941480755806,
      "rewards/margins": 0.13686522841453552,
      "rewards/rejected": -0.19626465439796448,
      "step": 13450
    },
    {
      "epoch": 0.98008519314086,
      "grad_norm": 3.343187268567383,
      "learning_rate": 6.895526900337915e-07,
      "log_odds_chosen": 1.8810546398162842,
      "log_odds_ratio": -0.2917236387729645,
      "logits/chosen": -0.960742175579071,
      "logits/rejected": -0.822265625,
      "logps/chosen": -0.571972668170929,
      "logps/rejected": -1.840234398841858,
      "loss": 0.6212,
      "nll_loss": 0.590136706829071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.05721435695886612,
      "rewards/margins": 0.12663574516773224,
      "rewards/rejected": -0.18391112983226776,
      "step": 13460
    },
    {
      "epoch": 0.9808133396439364,
      "grad_norm": 3.2396856031850274,
      "learning_rate": 6.892966838285567e-07,
      "log_odds_chosen": 1.604101538658142,
      "log_odds_ratio": -0.36376953125,
      "logits/chosen": -0.989453136920929,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.6142578125,
      "logps/rejected": -1.705078125,
      "loss": 0.6064,
      "nll_loss": 0.587597668170929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06137695163488388,
      "rewards/margins": 0.10908202826976776,
      "rewards/rejected": -0.17045898735523224,
      "step": 13470
    },
    {
      "epoch": 0.9815414861470128,
      "grad_norm": 3.6631258372607123,
      "learning_rate": 6.890409625495545e-07,
      "log_odds_chosen": 1.6316406726837158,
      "log_odds_ratio": -0.3636718690395355,
      "logits/chosen": -1.037500023841858,
      "logits/rejected": -0.909375011920929,
      "logps/chosen": -0.56298828125,
      "logps/rejected": -1.649999976158142,
      "loss": 0.61,
      "nll_loss": 0.5386718511581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05631103366613388,
      "rewards/margins": 0.10869140923023224,
      "rewards/rejected": -0.16496582329273224,
      "step": 13480
    },
    {
      "epoch": 0.9822696326500892,
      "grad_norm": 3.0446169590896295,
      "learning_rate": 6.887855256686546e-07,
      "log_odds_chosen": 1.7576172351837158,
      "log_odds_ratio": -0.3376708924770355,
      "logits/chosen": -1.004296898841858,
      "logits/rejected": -0.900390625,
      "logps/chosen": -0.6283203363418579,
      "logps/rejected": -1.836328148841858,
      "loss": 0.6399,
      "nll_loss": 0.564746081829071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0628662109375,
      "rewards/margins": 0.12073974311351776,
      "rewards/rejected": -0.18339844048023224,
      "step": 13490
    },
    {
      "epoch": 0.9829977791531657,
      "grad_norm": 3.3033649960268203,
      "learning_rate": 6.885303726590963e-07,
      "log_odds_chosen": 2.049023389816284,
      "log_odds_ratio": -0.2989746034145355,
      "logits/chosen": -1.024023413658142,
      "logits/rejected": -0.8529297113418579,
      "logps/chosen": -0.592089831829071,
      "logps/rejected": -1.994531273841858,
      "loss": 0.6189,
      "nll_loss": 0.554394543170929,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.05919189378619194,
      "rewards/margins": 0.140106201171875,
      "rewards/rejected": -0.19942626357078552,
      "step": 13500
    },
    {
      "epoch": 0.9837259256562421,
      "grad_norm": 4.347077792996884,
      "learning_rate": 6.882755029954837e-07,
      "log_odds_chosen": 1.7869141101837158,
      "log_odds_ratio": -0.3946777284145355,
      "logits/chosen": -1.0105469226837158,
      "logits/rejected": -0.892578125,
      "logps/chosen": -0.670117199420929,
      "logps/rejected": -1.9552733898162842,
      "loss": 0.6168,
      "nll_loss": 0.5728515386581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.0670166015625,
      "rewards/margins": 0.12863770127296448,
      "rewards/rejected": -0.195556640625,
      "step": 13510
    },
    {
      "epoch": 0.9844540721593185,
      "grad_norm": 4.416542416828337,
      "learning_rate": 6.880209161537815e-07,
      "log_odds_chosen": 2.019726514816284,
      "log_odds_ratio": -0.29072266817092896,
      "logits/chosen": -1.034765601158142,
      "logits/rejected": -0.9048827886581421,
      "logps/chosen": -0.558789074420929,
      "logps/rejected": -1.9435546398162842,
      "loss": 0.6102,
      "nll_loss": 0.5355468988418579,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.055908203125,
      "rewards/margins": 0.138458251953125,
      "rewards/rejected": -0.19426269829273224,
      "step": 13520
    },
    {
      "epoch": 0.9851822186623949,
      "grad_norm": 3.493486367968967,
      "learning_rate": 6.877666116113097e-07,
      "log_odds_chosen": 1.821874976158142,
      "log_odds_ratio": -0.2750488221645355,
      "logits/chosen": -1.028710961341858,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -0.534960925579071,
      "logps/rejected": -1.714453101158142,
      "loss": 0.6061,
      "nll_loss": 0.5101562738418579,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.05352783203125,
      "rewards/margins": 0.117919921875,
      "rewards/rejected": -0.17145995795726776,
      "step": 13530
    },
    {
      "epoch": 0.9859103651654713,
      "grad_norm": 3.2229954169951593,
      "learning_rate": 6.875125888467405e-07,
      "log_odds_chosen": 2.111132860183716,
      "log_odds_ratio": -0.30915528535842896,
      "logits/chosen": -1.0498046875,
      "logits/rejected": -0.926562488079071,
      "logps/chosen": -0.5538574457168579,
      "logps/rejected": -1.9666016101837158,
      "loss": 0.643,
      "nll_loss": 0.533007800579071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05535888671875,
      "rewards/margins": 0.14117431640625,
      "rewards/rejected": -0.196533203125,
      "step": 13540
    },
    {
      "epoch": 0.9866385116685477,
      "grad_norm": 3.6865871752935084,
      "learning_rate": 6.872588473400923e-07,
      "log_odds_chosen": 2.042187452316284,
      "log_odds_ratio": -0.3267578184604645,
      "logits/chosen": -0.952929675579071,
      "logits/rejected": -0.897265613079071,
      "logps/chosen": -0.589550793170929,
      "logps/rejected": -1.982812523841858,
      "loss": 0.6098,
      "nll_loss": 0.572265625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05891113355755806,
      "rewards/margins": 0.13930663466453552,
      "rewards/rejected": -0.1981201171875,
      "step": 13550
    },
    {
      "epoch": 0.9873666581716242,
      "grad_norm": 3.130040989053967,
      "learning_rate": 6.870053865727262e-07,
      "log_odds_chosen": 1.933007836341858,
      "log_odds_ratio": -0.3095947206020355,
      "logits/chosen": -0.958984375,
      "logits/rejected": -0.848828136920929,
      "logps/chosen": -0.5938476324081421,
      "logps/rejected": -1.899804711341858,
      "loss": 0.6236,
      "nll_loss": 0.594287097454071,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05937499925494194,
      "rewards/margins": 0.13041992485523224,
      "rewards/rejected": -0.18986816704273224,
      "step": 13560
    },
    {
      "epoch": 0.9880948046747006,
      "grad_norm": 3.788815560532827,
      "learning_rate": 6.867522060273408e-07,
      "log_odds_chosen": 1.954687476158142,
      "log_odds_ratio": -0.2853027284145355,
      "logits/chosen": -1.004296898841858,
      "logits/rejected": -0.8939453363418579,
      "logps/chosen": -0.5838867425918579,
      "logps/rejected": -1.908593773841858,
      "loss": 0.6225,
      "nll_loss": 0.575488269329071,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0584716796875,
      "rewards/margins": 0.13261719048023224,
      "rewards/rejected": -0.1910400390625,
      "step": 13570
    },
    {
      "epoch": 0.988822951177777,
      "grad_norm": 3.640212013140752,
      "learning_rate": 6.864993051879688e-07,
      "log_odds_chosen": 1.6278808116912842,
      "log_odds_ratio": -0.3725341856479645,
      "logits/chosen": -1.002343773841858,
      "logits/rejected": -0.8765624761581421,
      "logps/chosen": -0.6763671636581421,
      "logps/rejected": -1.790624976158142,
      "loss": 0.6044,
      "nll_loss": 0.5936523675918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06765136867761612,
      "rewards/margins": 0.11151123046875,
      "rewards/rejected": -0.17902831733226776,
      "step": 13580
    },
    {
      "epoch": 0.9895510976808534,
      "grad_norm": 3.134713728370864,
      "learning_rate": 6.862466835399716e-07,
      "log_odds_chosen": 1.4814453125,
      "log_odds_ratio": -0.39702147245407104,
      "logits/chosen": -1.0138671398162842,
      "logits/rejected": -0.9072265625,
      "logps/chosen": -0.617871105670929,
      "logps/rejected": -1.6240234375,
      "loss": 0.6169,
      "nll_loss": 0.5575195550918579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06173095852136612,
      "rewards/margins": 0.10053710639476776,
      "rewards/rejected": -0.16237792372703552,
      "step": 13590
    },
    {
      "epoch": 0.9902792441839298,
      "grad_norm": 3.3323741356883265,
      "learning_rate": 6.859943405700353e-07,
      "log_odds_chosen": 1.6941406726837158,
      "log_odds_ratio": -0.3756347596645355,
      "logits/chosen": -1.027929663658142,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.5894531011581421,
      "logps/rejected": -1.6984374523162842,
      "loss": 0.6117,
      "nll_loss": 0.5599609613418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.0589599609375,
      "rewards/margins": 0.11085204780101776,
      "rewards/rejected": -0.16984863579273224,
      "step": 13600
    },
    {
      "epoch": 0.9910073906870063,
      "grad_norm": 3.6805764917393673,
      "learning_rate": 6.857422757661664e-07,
      "log_odds_chosen": 2.1021485328674316,
      "log_odds_ratio": -0.2802734375,
      "logits/chosen": -1.017578125,
      "logits/rejected": -0.857617199420929,
      "logps/chosen": -0.5946289300918579,
      "logps/rejected": -2.100390672683716,
      "loss": 0.6075,
      "nll_loss": 0.5562499761581421,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.05947265774011612,
      "rewards/margins": 0.15043945610523224,
      "rewards/rejected": -0.20976562798023224,
      "step": 13610
    },
    {
      "epoch": 0.9917355371900827,
      "grad_norm": 3.5460980218311158,
      "learning_rate": 6.854904886176873e-07,
      "log_odds_chosen": 1.9609375,
      "log_odds_ratio": -0.2880859375,
      "logits/chosen": -0.962695300579071,
      "logits/rejected": -0.836132824420929,
      "logps/chosen": -0.5926758050918579,
      "logps/rejected": -1.916406273841858,
      "loss": 0.6235,
      "nll_loss": 0.5555664300918579,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.05928955227136612,
      "rewards/margins": 0.13236084580421448,
      "rewards/rejected": -0.19169922173023224,
      "step": 13620
    },
    {
      "epoch": 0.9924636836931591,
      "grad_norm": 3.541853849039648,
      "learning_rate": 6.85238978615232e-07,
      "log_odds_chosen": 1.7498047351837158,
      "log_odds_ratio": -0.34575194120407104,
      "logits/chosen": -1.025781273841858,
      "logits/rejected": -0.8744140863418579,
      "logps/chosen": -0.684863269329071,
      "logps/rejected": -1.900390625,
      "loss": 0.6294,
      "nll_loss": 0.6416015625,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06854248046875,
      "rewards/margins": 0.12174072116613388,
      "rewards/rejected": -0.19008788466453552,
      "step": 13630
    },
    {
      "epoch": 0.9931918301962355,
      "grad_norm": 3.4953955699190957,
      "learning_rate": 6.849877452507417e-07,
      "log_odds_chosen": 1.968164086341858,
      "log_odds_ratio": -0.3239502012729645,
      "logits/chosen": -0.9951171875,
      "logits/rejected": -0.8490234613418579,
      "logps/chosen": -0.619921863079071,
      "logps/rejected": -1.9933593273162842,
      "loss": 0.6177,
      "nll_loss": 0.5499023199081421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06203613430261612,
      "rewards/margins": 0.13729247450828552,
      "rewards/rejected": -0.19926758110523224,
      "step": 13640
    },
    {
      "epoch": 0.9939199766993119,
      "grad_norm": 3.389423388287887,
      "learning_rate": 6.847367880174605e-07,
      "log_odds_chosen": 1.8533203601837158,
      "log_odds_ratio": -0.3400634825229645,
      "logits/chosen": -0.9896484613418579,
      "logits/rejected": -0.858593761920929,
      "logps/chosen": -0.643261730670929,
      "logps/rejected": -1.932031273841858,
      "loss": 0.6311,
      "nll_loss": 0.650390625,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06431885063648224,
      "rewards/margins": 0.12879638373851776,
      "rewards/rejected": -0.19316406548023224,
      "step": 13650
    },
    {
      "epoch": 0.9946481232023883,
      "grad_norm": 3.1387312973025217,
      "learning_rate": 6.844861064099317e-07,
      "log_odds_chosen": 1.9127929210662842,
      "log_odds_ratio": -0.347900390625,
      "logits/chosen": -1.062109351158142,
      "logits/rejected": -0.945117175579071,
      "logps/chosen": -0.6075195074081421,
      "logps/rejected": -1.983007788658142,
      "loss": 0.5923,
      "nll_loss": 0.5362793207168579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06075439602136612,
      "rewards/margins": 0.13775634765625,
      "rewards/rejected": -0.19829101860523224,
      "step": 13660
    },
    {
      "epoch": 0.9953762697054648,
      "grad_norm": 3.287458671731839,
      "learning_rate": 6.842356999239922e-07,
      "log_odds_chosen": 1.8544921875,
      "log_odds_ratio": -0.3251953125,
      "logits/chosen": -1.0066406726837158,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -0.5757812261581421,
      "logps/rejected": -1.817968726158142,
      "loss": 0.621,
      "nll_loss": 0.5806640386581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.05753173679113388,
      "rewards/margins": 0.12413330376148224,
      "rewards/rejected": -0.1817626953125,
      "step": 13670
    },
    {
      "epoch": 0.9961044162085412,
      "grad_norm": 3.7784246159097554,
      "learning_rate": 6.839855680567694e-07,
      "log_odds_chosen": 1.7853515148162842,
      "log_odds_ratio": -0.3382812440395355,
      "logits/chosen": -0.964062511920929,
      "logits/rejected": -0.8349609375,
      "logps/chosen": -0.609667956829071,
      "logps/rejected": -1.845312476158142,
      "loss": 0.6282,
      "nll_loss": 0.646191418170929,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06102294847369194,
      "rewards/margins": 0.12342528998851776,
      "rewards/rejected": -0.1844482421875,
      "step": 13680
    },
    {
      "epoch": 0.9968325627116176,
      "grad_norm": 4.8560430895509725,
      "learning_rate": 6.837357103066766e-07,
      "log_odds_chosen": 1.7205078601837158,
      "log_odds_ratio": -0.3671875,
      "logits/chosen": -1.0216796398162842,
      "logits/rejected": -0.8941406011581421,
      "logps/chosen": -0.635546863079071,
      "logps/rejected": -1.820898413658142,
      "loss": 0.6114,
      "nll_loss": 0.605273425579071,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06354980170726776,
      "rewards/margins": 0.11873779445886612,
      "rewards/rejected": -0.1822509765625,
      "step": 13690
    },
    {
      "epoch": 0.997560709214694,
      "grad_norm": 2.6820041335471956,
      "learning_rate": 6.834861261734087e-07,
      "log_odds_chosen": 1.9533202648162842,
      "log_odds_ratio": -0.3153320252895355,
      "logits/chosen": -1.063085913658142,
      "logits/rejected": -0.931640625,
      "logps/chosen": -0.549609363079071,
      "logps/rejected": -1.8759765625,
      "loss": 0.6183,
      "nll_loss": 0.5541015863418579,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.05494384840130806,
      "rewards/margins": 0.13262939453125,
      "rewards/rejected": -0.18747559189796448,
      "step": 13700
    },
    {
      "epoch": 0.9982888557177704,
      "grad_norm": 2.9654373767739943,
      "learning_rate": 6.832368151579382e-07,
      "log_odds_chosen": 1.7155272960662842,
      "log_odds_ratio": -0.37114256620407104,
      "logits/chosen": -0.9925781488418579,
      "logits/rejected": -0.9068359136581421,
      "logps/chosen": -0.598828136920929,
      "logps/rejected": -1.730859398841858,
      "loss": 0.6073,
      "nll_loss": 0.589648425579071,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05988769605755806,
      "rewards/margins": 0.11319579929113388,
      "rewards/rejected": -0.17304687201976776,
      "step": 13710
    },
    {
      "epoch": 0.9990170022208469,
      "grad_norm": 3.502367178810729,
      "learning_rate": 6.829877767625106e-07,
      "log_odds_chosen": 1.738671898841858,
      "log_odds_ratio": -0.37065428495407104,
      "logits/chosen": -0.954882800579071,
      "logits/rejected": -0.8685547113418579,
      "logps/chosen": -0.601269543170929,
      "logps/rejected": -1.801367163658142,
      "loss": 0.5935,
      "nll_loss": 0.5506836175918579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06015624850988388,
      "rewards/margins": 0.11988525092601776,
      "rewards/rejected": -0.18012695014476776,
      "step": 13720
    },
    {
      "epoch": 0.9997451487239233,
      "grad_norm": 3.0377575245353396,
      "learning_rate": 6.827390104906408e-07,
      "log_odds_chosen": 1.94140625,
      "log_odds_ratio": -0.33759766817092896,
      "logits/chosen": -0.9955078363418579,
      "logits/rejected": -0.869335949420929,
      "logps/chosen": -0.5526367425918579,
      "logps/rejected": -1.859375,
      "loss": 0.5959,
      "nll_loss": 0.5707031488418579,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.05526123195886612,
      "rewards/margins": 0.13077393174171448,
      "rewards/rejected": -0.18598632514476776,
      "step": 13730
    }
  ],
  "logging_steps": 10,
  "max_steps": 13733,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 5000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}